Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.term;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.i18n.Msg;
025import ca.uhn.fhir.interceptor.model.RequestPartitionId;
026import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
027import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
028import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode;
029import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
030import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
031import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
032import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
033import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
034import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
035import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
036import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
037import ca.uhn.fhir.jpa.entity.TermCodeSystem;
038import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
039import ca.uhn.fhir.jpa.entity.TermConcept;
040import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
041import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
042import ca.uhn.fhir.jpa.entity.TermConceptProperty;
043import ca.uhn.fhir.jpa.model.dao.JpaPid;
044import ca.uhn.fhir.jpa.model.entity.ResourceTable;
045import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
046import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
047import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
048import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc;
049import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
050import ca.uhn.fhir.rest.api.Constants;
051import ca.uhn.fhir.rest.api.server.RequestDetails;
052import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
053import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
054import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
055import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
056import ca.uhn.fhir.util.ObjectUtil;
057import ca.uhn.fhir.util.UrlUtil;
058import ca.uhn.fhir.util.ValidateUtil;
059import jakarta.annotation.Nonnull;
060import jakarta.persistence.EntityManager;
061import jakarta.persistence.PersistenceContext;
062import jakarta.persistence.PersistenceContextType;
063import org.apache.commons.lang3.Validate;
064import org.hl7.fhir.instance.model.api.IIdType;
065import org.hl7.fhir.r4.model.CodeSystem;
066import org.hl7.fhir.r4.model.ConceptMap;
067import org.hl7.fhir.r4.model.ValueSet;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070import org.springframework.beans.factory.annotation.Autowired;
071import org.springframework.transaction.annotation.Propagation;
072import org.springframework.transaction.annotation.Transactional;
073import org.springframework.transaction.support.TransactionSynchronizationManager;
074
075import java.util.ArrayList;
076import java.util.Arrays;
077import java.util.Collection;
078import java.util.Collections;
079import java.util.HashMap;
080import java.util.HashSet;
081import java.util.IdentityHashMap;
082import java.util.List;
083import java.util.Map;
084import java.util.Objects;
085import java.util.Optional;
086import java.util.Set;
087import java.util.UUID;
088import java.util.concurrent.atomic.AtomicInteger;
089import java.util.stream.Collectors;
090
091import static ca.uhn.fhir.jpa.api.dao.IDao.RESOURCE_PID_KEY;
092import static org.apache.commons.lang3.StringUtils.isBlank;
093import static org.apache.commons.lang3.StringUtils.isNotBlank;
094import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
095
096public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
097        private static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemStorageSvcImpl.class);
098        private static final Object PLACEHOLDER_OBJECT = new Object();
099
100        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
101        protected EntityManager myEntityManager;
102
103        @Autowired
104        protected ITermCodeSystemDao myCodeSystemDao;
105
106        @Autowired
107        protected ITermCodeSystemVersionDao myCodeSystemVersionDao;
108
109        @Autowired
110        protected ITermConceptDao myConceptDao;
111
112        @Autowired
113        protected ITermConceptPropertyDao myConceptPropertyDao;
114
115        @Autowired
116        protected ITermConceptDesignationDao myConceptDesignationDao;
117
118        @Autowired
119        protected IIdHelperService<JpaPid> myIdHelperService;
120
121        @Autowired
122        private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
123
124        @Autowired
125        private ITermVersionAdapterSvc myTerminologyVersionAdapterSvc;
126
127        @Autowired
128        private ITermDeferredStorageSvc myDeferredStorageSvc;
129
130        @Autowired
131        private FhirContext myContext;
132
133        @Autowired
134        private ITermReadSvc myTerminologySvc;
135
136        @Autowired
137        private JpaStorageSettings myStorageSettings;
138
139        @Autowired
140        private IResourceTableDao myResourceTableDao;
141
142        @Autowired
143        private TermConceptDaoSvc myTermConceptDaoSvc;
144
145        @Transactional
146        @Override
147        public UploadStatistics applyDeltaCodeSystemsAdd(String theSystem, CustomTerminologySet theAdditions) {
148                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
149                validateDstu3OrNewer();
150                theAdditions.validateNoCycleOrThrowInvalidRequest();
151
152                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
153                if (cs == null) {
154                        CodeSystem codeSystemResource = new CodeSystem();
155                        codeSystemResource.setUrl(theSystem);
156                        codeSystemResource.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
157                        if (isBlank(codeSystemResource.getIdElement().getIdPart()) && theSystem.contains(LOINC_LOW)) {
158                                codeSystemResource.setId(LOINC_LOW);
159                        }
160                        myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(codeSystemResource);
161
162                        cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
163                }
164
165                TermCodeSystemVersion csv = cs.getCurrentVersion();
166                Validate.notNull(csv);
167
168                CodeSystem codeSystem = myTerminologySvc.fetchCanonicalCodeSystemFromCompleteContext(theSystem);
169                if (codeSystem.getContent() != CodeSystem.CodeSystemContentMode.NOTPRESENT) {
170                        throw new InvalidRequestException(
171                                        Msg.code(844) + "CodeSystem with url[" + Constants.codeSystemWithDefaultDescription(theSystem)
172                                                        + "] can not apply a delta - wrong content mode: " + codeSystem.getContent());
173                }
174
175                Validate.notNull(cs);
176                Validate.notNull(cs.getPid());
177
178                IIdType codeSystemId = cs.getResource().getIdDt();
179
180                UploadStatistics retVal = new UploadStatistics(codeSystemId);
181                HashMap<String, TermConcept> codeToConcept = new HashMap<>();
182
183                // Add root concepts
184                for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) {
185                        List<String> parentCodes = Collections.emptyList();
186                        addConceptInHierarchy(csv, parentCodes, nextRootConcept, retVal, codeToConcept, 0);
187                }
188
189                return retVal;
190        }
191
192        @Transactional
193        @Override
194        public UploadStatistics applyDeltaCodeSystemsRemove(String theSystem, CustomTerminologySet theValue) {
195                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
196                validateDstu3OrNewer();
197
198                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
199                if (cs == null) {
200                        throw new InvalidRequestException(Msg.code(845) + "Unknown code system: " + theSystem);
201                }
202                IIdType target = cs.getResource().getIdDt();
203
204                AtomicInteger removeCounter = new AtomicInteger(0);
205
206                // We need to delete all termconcepts, and their children. This stream flattens the TermConcepts and their
207                // children into a single set of TermConcept objects retrieved from the DB. Note that we have to do this because
208                // deleteById() in JPA doesnt appear to actually commit or flush a transaction until way later, and we end up
209                // iterating multiple times over the same elements, which screws up our counter.
210
211                // Grab the actual entities
212                List<TermConcept> collect = theValue.getRootConcepts().stream()
213                                .map(val -> myTerminologySvc.findCode(theSystem, val.getCode()))
214                                .filter(Optional::isPresent)
215                                .map(Optional::get)
216                                .collect(Collectors.toList());
217
218                // Iterate over the actual entities and fill out their children
219                Set<TermConcept> allFoundTermConcepts = collect.stream()
220                                .flatMap(concept -> flattenChildren(concept).stream())
221                                .map(suppliedTermConcept -> myTerminologySvc.findCode(theSystem, suppliedTermConcept.getCode()))
222                                .filter(Optional::isPresent)
223                                .map(Optional::get)
224                                .collect(Collectors.toSet());
225
226                // Delete everything about these codes.
227                for (TermConcept code : allFoundTermConcepts) {
228                        deleteEverythingRelatedToConcept(code, removeCounter);
229                }
230
231                return new UploadStatistics(removeCounter.get(), target);
232        }
233
234        private void deleteEverythingRelatedToConcept(TermConcept theConcept, AtomicInteger theRemoveCounter) {
235
236                for (TermConceptParentChildLink nextParent : theConcept.getParents()) {
237                        nextParent.getParent().getChildren().remove(nextParent);
238                        myConceptParentChildLinkDao.deleteById(nextParent.getId());
239                }
240                for (TermConceptParentChildLink nextChild : theConcept.getChildren()) {
241                        nextChild.getChild().getParents().remove(nextChild);
242                        myConceptParentChildLinkDao.deleteById(nextChild.getId());
243                }
244
245                for (TermConceptDesignation next : theConcept.getDesignations()) {
246                        myConceptDesignationDao.deleteById(next.getPid());
247                }
248                theConcept.getDesignations().clear();
249                for (TermConceptProperty next : theConcept.getProperties()) {
250                        myConceptPropertyDao.deleteById(next.getPid());
251                }
252                theConcept.getProperties().clear();
253
254                ourLog.info("Deleting concept {} - Code {}", theConcept.getId(), theConcept.getCode());
255
256                myConceptDao.deleteById(theConcept.getId());
257                //              myEntityManager.remove(theConcept);
258
259                theRemoveCounter.incrementAndGet();
260        }
261
262        private List<TermConcept> flattenChildren(TermConcept theTermConcept) {
263                if (theTermConcept.getChildren().isEmpty()) {
264                        return Arrays.asList(theTermConcept);
265                }
266
267                // Recursively flatten children
268                List<TermConcept> childTermConcepts = theTermConcept.getChildren().stream()
269                                .map(TermConceptParentChildLink::getChild)
270                                .flatMap(childConcept -> flattenChildren(childConcept).stream())
271                                .collect(Collectors.toList());
272
273                // Add itself before its list of children
274                childTermConcepts.add(0, theTermConcept);
275                return childTermConcepts;
276        }
277
278        /**
279         * Returns the number of saved concepts
280         */
281        @Override
282        public int saveConcept(TermConcept theConcept) {
283                return myTermConceptDaoSvc.saveConcept(theConcept);
284        }
285
286        @Override
287        @Transactional(propagation = Propagation.MANDATORY)
288        public void storeNewCodeSystemVersionIfNeeded(
289                        CodeSystem theCodeSystem, ResourceTable theResourceEntity, RequestDetails theRequestDetails) {
290                if (theCodeSystem != null && isNotBlank(theCodeSystem.getUrl())) {
291                        String codeSystemUrl = theCodeSystem.getUrl();
292                        if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.COMPLETE
293                                        || theCodeSystem.getContent() == null
294                                        || theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
295                                ourLog.info(
296                                                "CodeSystem {} has a status of {}, going to store concepts in terminology tables",
297                                                theResourceEntity.getIdDt().getValue(),
298                                                theCodeSystem.getContentElement().getValueAsString());
299
300                                detectDuplicatesInCodeSystem(theCodeSystem);
301
302                                Long pid = (Long) theCodeSystem.getUserData(RESOURCE_PID_KEY);
303                                assert pid != null;
304                                JpaPid codeSystemResourcePid = JpaPid.fromId(pid);
305
306                                /*
307                                 * If this is a not-present codesystem and codesystem version already exists, we don't want to
308                                 * overwrite the existing version since that will wipe out the existing concepts. We do create
309                                 * or update the TermCodeSystem table though, since that allows the DB to reject changes that would
310                                 * result in duplicate CodeSystem.url values.
311                                 */
312                                if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
313                                        TermCodeSystem termCodeSystem = myCodeSystemDao.findByCodeSystemUri(theCodeSystem.getUrl());
314                                        if (termCodeSystem != null) {
315                                                TermCodeSystemVersion codeSystemVersion =
316                                                                getExistingTermCodeSystemVersion(termCodeSystem.getPid(), theCodeSystem.getVersion());
317                                                if (codeSystemVersion != null) {
318                                                        TermCodeSystem myCodeSystemEntity = getOrCreateDistinctTermCodeSystem(
319                                                                        codeSystemResourcePid,
320                                                                        theCodeSystem.getUrl(),
321                                                                        theCodeSystem.getUrl(),
322                                                                        theCodeSystem.getVersion(),
323                                                                        theResourceEntity);
324                                                        return;
325                                                }
326                                        }
327                                }
328
329                                TermCodeSystemVersion persCs = new TermCodeSystemVersion();
330
331                                populateCodeSystemVersionProperties(persCs, theCodeSystem, theResourceEntity);
332
333                                persCs.getConcepts().addAll(TermReadSvcImpl.toPersistedConcepts(theCodeSystem.getConcept(), persCs));
334                                ourLog.debug("Code system has {} concepts", persCs.getConcepts().size());
335                                storeNewCodeSystemVersion(
336                                                codeSystemResourcePid,
337                                                codeSystemUrl,
338                                                theCodeSystem.getName(),
339                                                theCodeSystem.getVersion(),
340                                                persCs,
341                                                theResourceEntity,
342                                                theRequestDetails);
343                        }
344                }
345        }
346
347        private static void detectDuplicatesInCodeSystem(CodeSystem theCodeSystem) {
348                detectDuplicatesInCodeSystem(theCodeSystem.getConcept(), new HashSet<>());
349        }
350
351        private static void detectDuplicatesInCodeSystem(
352                        List<CodeSystem.ConceptDefinitionComponent> theCodeList, Set<String> theFoundCodesBuffer) {
353                for (var next : theCodeList) {
354                        if (isNotBlank(next.getCode())) {
355                                if (!theFoundCodesBuffer.add(next.getCode())) {
356                                        /*
357                                         * Note: We could possibly modify this behaviour to be forgiving, and just
358                                         * ignore duplicates. The only issue is that concepts can have properties,
359                                         * designations, etc. and it could be dangerous to just pick one and ignore the
360                                         * other. So the safer thing seems to be to just throw an error.
361                                         */
362                                        throw new PreconditionFailedException(Msg.code(2528) + "Duplicate concept detected in CodeSystem: "
363                                                        + UrlUtil.sanitizeUrlPart(next.getCode()));
364                                }
365                        }
366                        // Test child concepts within the parent concept
367                        detectDuplicatesInCodeSystem(next.getConcept(), theFoundCodesBuffer);
368                }
369        }
370
371        @Override
372        @Transactional
373        public IIdType storeNewCodeSystemVersion(
374                        CodeSystem theCodeSystemResource,
375                        TermCodeSystemVersion theCodeSystemVersion,
376                        RequestDetails theRequest,
377                        List<ValueSet> theValueSets,
378                        List<ConceptMap> theConceptMaps) {
379                assert TransactionSynchronizationManager.isActualTransactionActive();
380
381                Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
382
383                // Note that this creates the TermCodeSystem and TermCodeSystemVersion entities if needed
384                IIdType csId = myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(theCodeSystemResource, theRequest);
385
386                JpaPid codeSystemResourcePid = myIdHelperService.resolveResourceIdentityPid(
387                                RequestPartitionId.allPartitions(),
388                                csId.getResourceType(),
389                                csId.getIdPart(),
390                                ResolveIdentityMode.includeDeleted().cacheOk());
391                ResourceTable resource = myResourceTableDao.getOne(codeSystemResourcePid.getId());
392
393                ourLog.info("CodeSystem resource has ID: {}", csId.getValue());
394
395                populateCodeSystemVersionProperties(theCodeSystemVersion, theCodeSystemResource, resource);
396
397                storeNewCodeSystemVersion(
398                                codeSystemResourcePid,
399                                theCodeSystemResource.getUrl(),
400                                theCodeSystemResource.getName(),
401                                theCodeSystemResource.getVersion(),
402                                theCodeSystemVersion,
403                                resource,
404                                theRequest);
405
406                myDeferredStorageSvc.addConceptMapsToStorageQueue(theConceptMaps);
407                myDeferredStorageSvc.addValueSetsToStorageQueue(theValueSets);
408
409                return csId;
410        }
411
412        @Override
413        @Transactional
414        public void storeNewCodeSystemVersion(
415                        IResourcePersistentId theCodeSystemResourcePid,
416                        String theSystemUri,
417                        String theSystemName,
418                        String theCodeSystemVersionId,
419                        TermCodeSystemVersion theCodeSystemVersion,
420                        ResourceTable theCodeSystemResourceTable,
421                        RequestDetails theRequestDetails) {
422                assert TransactionSynchronizationManager.isActualTransactionActive();
423
424                ourLog.debug("Storing code system");
425
426                TermCodeSystemVersion codeSystemToStore = theCodeSystemVersion;
427                ValidateUtil.isTrueOrThrowInvalidRequest(codeSystemToStore.getResource() != null, "No resource supplied");
428                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied");
429
430                TermCodeSystem codeSystem = getOrCreateDistinctTermCodeSystem(
431                                theCodeSystemResourcePid,
432                                theSystemUri,
433                                theSystemName,
434                                theCodeSystemVersionId,
435                                theCodeSystemResourceTable);
436
437                List<TermCodeSystemVersion> existing =
438                                myCodeSystemVersionDao.findByCodeSystemResourcePid(((JpaPid) theCodeSystemResourcePid).getId());
439                for (TermCodeSystemVersion next : existing) {
440                        if (Objects.equals(next.getCodeSystemVersionId(), theCodeSystemVersionId)
441                                        && myConceptDao.countByCodeSystemVersion(next.getPid()) == 0) {
442
443                                /*
444                                 * If we already have a CodeSystemVersion that matches the version we're storing, we
445                                 * can reuse it.
446                                 */
447                                next.setCodeSystemDisplayName(theSystemName);
448                                codeSystemToStore = next;
449
450                        } else {
451
452                                /*
453                                 * If we already have a TermCodeSystemVersion that corresponds to the FHIR Resource ID we're
454                                 * adding a version to, we will mark it for deletion. For any one resource there can only
455                                 * be one TermCodeSystemVersion entity in the DB. Multiple versions of a codesystem uses
456                                 * multiple CodeSystem resources with CodeSystem.version set differently (as opposed to
457                                 * multiple versions of the same CodeSystem, where CodeSystem.meta.versionId is different)
458                                 */
459                                next.setCodeSystemVersionId("DELETED_" + UUID.randomUUID().toString());
460                                myCodeSystemVersionDao.saveAndFlush(next);
461                                myDeferredStorageSvc.deleteCodeSystemVersion(next);
462                        }
463                }
464
465                /*
466                 * Do the upload
467                 */
468
469                codeSystemToStore.setCodeSystem(codeSystem);
470                codeSystemToStore.setCodeSystemDisplayName(theSystemName);
471                codeSystemToStore.setCodeSystemVersionId(theCodeSystemVersionId);
472
473                ourLog.debug("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
474
475                // Validate the code system
476                ArrayList<String> conceptsStack = new ArrayList<>();
477                IdentityHashMap<TermConcept, Object> allConcepts = new IdentityHashMap<>();
478                int totalCodeCount = 0;
479                Collection<TermConcept> conceptsToSave = theCodeSystemVersion.getConcepts();
480                for (TermConcept next : conceptsToSave) {
481                        totalCodeCount += validateConceptForStorage(next, codeSystemToStore, conceptsStack, allConcepts);
482                }
483
484                ourLog.debug("Saving version containing {} concepts", totalCodeCount);
485                if (codeSystemToStore.getPid() == null) {
486                        codeSystemToStore = myCodeSystemVersionDao.saveAndFlush(codeSystemToStore);
487                }
488
489                boolean isMakeVersionCurrent = ITermCodeSystemStorageSvc.isMakeVersionCurrent(theRequestDetails);
490                if (isMakeVersionCurrent) {
491                        codeSystem.setCurrentVersion(codeSystemToStore);
492                        if (codeSystem.getPid() == null) {
493                                codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
494                        }
495                }
496
497                ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
498                for (TermConcept next : conceptsToSave) {
499                        populateVersion(next, codeSystemToStore);
500                }
501
502                ourLog.debug("Saving {} concepts...", totalCodeCount);
503                IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
504                for (TermConcept next : conceptsToSave) {
505                        persistChildren(next, codeSystemToStore, conceptsStack2, totalCodeCount);
506                }
507
508                ourLog.debug("Done saving concepts, flushing to database");
509                if (!myDeferredStorageSvc.isStorageQueueEmpty(true)) {
510                        ourLog.info("Note that some concept saving has been deferred");
511                }
512        }
513
514        private TermCodeSystemVersion getExistingTermCodeSystemVersion(
515                        Long theCodeSystemVersionPid, String theCodeSystemVersion) {
516                TermCodeSystemVersion existing;
517                if (theCodeSystemVersion == null) {
518                        existing = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystemVersionPid);
519                } else {
520                        existing =
521                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystemVersionPid, theCodeSystemVersion);
522                }
523
524                return existing;
525        }
526
527        private void validateDstu3OrNewer() {
528                Validate.isTrue(
529                                myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3),
530                                "Terminology operations only supported in DSTU3+ mode");
531        }
532
533        private void addConceptInHierarchy(
534                        TermCodeSystemVersion theCsv,
535                        Collection<String> theParentCodes,
536                        TermConcept theConceptToAdd,
537                        UploadStatistics theStatisticsTracker,
538                        Map<String, TermConcept> theCodeToConcept,
539                        int theSequence) {
540                TermConcept conceptToAdd = theConceptToAdd;
541                List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren();
542
543                String nextCodeToAdd = conceptToAdd.getCode();
544                String parentDescription = "(root concept)";
545
546                ourLog.info(
547                                "Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription);
548
549                Optional<TermConcept> existingCodeOpt = myConceptDao.findByCodeSystemAndCode(theCsv.getPid(), nextCodeToAdd);
550                List<TermConceptParentChildLink> existingParentLinks;
551                if (existingCodeOpt.isPresent()) {
552                        TermConcept existingCode = existingCodeOpt.get();
553                        existingCode.setIndexStatus(null);
554                        existingCode.setDisplay(conceptToAdd.getDisplay());
555                        conceptToAdd = existingCode;
556                        existingParentLinks = conceptToAdd.getParents();
557                } else {
558                        existingParentLinks = Collections.emptyList();
559                }
560
561                Set<TermConcept> parentConceptsWeShouldLinkTo = new HashSet<>();
562                for (String nextParentCode : theParentCodes) {
563
564                        // Don't add parent links that already exist for the code
565                        if (existingParentLinks.stream()
566                                        .anyMatch(t -> t.getParent().getCode().equals(nextParentCode))) {
567                                continue;
568                        }
569
570                        TermConcept nextParentOpt = theCodeToConcept.get(nextParentCode);
571                        if (nextParentOpt == null) {
572                                nextParentOpt = myConceptDao
573                                                .findByCodeSystemAndCode(theCsv.getPid(), nextParentCode)
574                                                .orElse(null);
575                        }
576                        if (nextParentOpt == null) {
577                                throw new InvalidRequestException(Msg.code(846) + "Unable to add code \"" + nextCodeToAdd
578                                                + "\" to unknown parent: " + nextParentCode);
579                        }
580                        parentConceptsWeShouldLinkTo.add(nextParentOpt);
581                }
582
583                if (conceptToAdd.getSequence() == null) {
584                        conceptToAdd.setSequence(theSequence);
585                }
586
587                // Null out the hierarchy PIDs for this concept always. We do this because we're going to
588                // force a reindex, and it'll be regenerated then
589                conceptToAdd.setParentPids(null);
590                conceptToAdd.setCodeSystemVersion(theCsv);
591
592                if (conceptToAdd.getProperties() != null)
593                        conceptToAdd.getProperties().forEach(termConceptProperty -> {
594                                termConceptProperty.setConcept(theConceptToAdd);
595                                termConceptProperty.setCodeSystemVersion(theCsv);
596                        });
597                if (theStatisticsTracker.getUpdatedConceptCount() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
598                        saveConcept(conceptToAdd);
599                        Long nextConceptPid = conceptToAdd.getId();
600                        Objects.requireNonNull(nextConceptPid);
601                } else {
602                        myDeferredStorageSvc.addConceptToStorageQueue(conceptToAdd);
603                }
604
605                theCodeToConcept.put(conceptToAdd.getCode(), conceptToAdd);
606
607                theStatisticsTracker.incrementUpdatedConceptCount();
608
609                // Add link to new child to the parent
610                for (TermConcept nextParentConcept : parentConceptsWeShouldLinkTo) {
611                        TermConceptParentChildLink parentLink = new TermConceptParentChildLink();
612                        parentLink.setParent(nextParentConcept);
613                        parentLink.setChild(conceptToAdd);
614                        parentLink.setCodeSystem(theCsv);
615                        parentLink.setRelationshipType(TermConceptParentChildLink.RelationshipTypeEnum.ISA);
616                        nextParentConcept.getChildren().add(parentLink);
617                        conceptToAdd.getParents().add(parentLink);
618                        ourLog.info(
619                                        "Saving parent/child link - Parent[{}] Child[{}]",
620                                        parentLink.getParent().getCode(),
621                                        parentLink.getChild().getCode());
622
623                        if (theStatisticsTracker.getUpdatedConceptCount()
624                                        <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
625                                myConceptParentChildLinkDao.save(parentLink);
626                        } else {
627                                myDeferredStorageSvc.addConceptLinkToStorageQueue(parentLink);
628                        }
629                }
630
631                ourLog.trace("About to save parent-child links");
632
633                // Save children recursively
634                int childIndex = 0;
635                for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) {
636
637                        TermConcept nextChild = nextChildConceptLink.getChild();
638
639                        for (int i = 0; i < nextChild.getParents().size(); i++) {
640                                if (nextChild.getParents().get(i).getId() == null) {
641                                        String parentCode =
642                                                        nextChild.getParents().get(i).getParent().getCode();
643                                        TermConcept parentConcept = theCodeToConcept.get(parentCode);
644                                        if (parentConcept == null) {
645                                                parentConcept = myConceptDao
646                                                                .findByCodeSystemAndCode(theCsv.getPid(), parentCode)
647                                                                .orElse(null);
648                                        }
649                                        if (parentConcept == null) {
650                                                throw new IllegalArgumentException(Msg.code(847) + "Unknown parent code: " + parentCode);
651                                        }
652
653                                        nextChild.getParents().get(i).setParent(parentConcept);
654                                }
655                        }
656
657                        Collection<String> parentCodes = nextChild.getParents().stream()
658                                        .map(t -> t.getParent().getCode())
659                                        .collect(Collectors.toList());
660                        addConceptInHierarchy(theCsv, parentCodes, nextChild, theStatisticsTracker, theCodeToConcept, childIndex);
661
662                        childIndex++;
663                }
664        }
665
666        private void persistChildren(
667                        TermConcept theConcept,
668                        TermCodeSystemVersion theCodeSystem,
669                        IdentityHashMap<TermConcept, Object> theConceptsStack,
670                        int theTotalConcepts) {
671                if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) {
672                        return;
673                }
674
675                if ((theConceptsStack.size() + 1) % 10000 == 0) {
676                        float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
677                        ourLog.info(
678                                        "Have processed {}/{} concepts ({}%)",
679                                        theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
680                }
681
682                theConcept.setCodeSystemVersion(theCodeSystem);
683                theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);
684
685                if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
686                        saveConcept(theConcept);
687                } else {
688                        myDeferredStorageSvc.addConceptToStorageQueue(theConcept);
689                }
690
691                for (TermConceptParentChildLink next : theConcept.getChildren()) {
692                        persistChildren(next.getChild(), theCodeSystem, theConceptsStack, theTotalConcepts);
693                }
694
695                for (TermConceptParentChildLink next : theConcept.getChildren()) {
696                        if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
697                                saveConceptLink(next);
698                        } else {
699                                myDeferredStorageSvc.addConceptLinkToStorageQueue(next);
700                        }
701                }
702        }
703
704        private void populateVersion(TermConcept theNext, TermCodeSystemVersion theCodeSystemVersion) {
705                theNext.setCodeSystemVersion(theCodeSystemVersion);
706                for (TermConceptParentChildLink next : theNext.getChildren()) {
707                        populateVersion(next.getChild(), theCodeSystemVersion);
708                }
709                theNext.getProperties().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
710                theNext.getDesignations().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
711        }
712
713        private void saveConceptLink(TermConceptParentChildLink next) {
714                if (next.getId() == null) {
715                        myConceptParentChildLinkDao.save(next);
716                }
717        }
718
719        @Nonnull
720        private TermCodeSystem getOrCreateDistinctTermCodeSystem(
721                        IResourcePersistentId theCodeSystemResourcePid,
722                        String theSystemUri,
723                        String theSystemName,
724                        String theSystemVersionId,
725                        ResourceTable theCodeSystemResourceTable) {
726                TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(theSystemUri);
727                if (codeSystem == null) {
728                        codeSystem = myCodeSystemDao.findByResourcePid(((JpaPid) theCodeSystemResourcePid).getId());
729                        if (codeSystem == null) {
730                                codeSystem = new TermCodeSystem();
731                        }
732                } else {
733                        checkForCodeSystemVersionDuplicate(
734                                        codeSystem, theSystemUri, theSystemVersionId, theCodeSystemResourceTable);
735                }
736
737                codeSystem.setResource(theCodeSystemResourceTable);
738                codeSystem.setCodeSystemUri(theSystemUri);
739                codeSystem.setName(theSystemName);
740                codeSystem = myCodeSystemDao.save(codeSystem);
741                return codeSystem;
742        }
743
744        private void checkForCodeSystemVersionDuplicate(
745                        TermCodeSystem theCodeSystem,
746                        String theSystemUri,
747                        String theSystemVersionId,
748                        ResourceTable theCodeSystemResourceTable) {
749                TermCodeSystemVersion codeSystemVersionEntity;
750                String msg = null;
751                if (theSystemVersionId == null) {
752                        // Check if a non-versioned TermCodeSystemVersion entity already exists for this TermCodeSystem.
753                        codeSystemVersionEntity = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystem.getPid());
754                        if (codeSystemVersionEntity != null) {
755                                msg = myContext
756                                                .getLocalizer()
757                                                .getMessage(
758                                                                TermReadSvcImpl.class,
759                                                                "cannotCreateDuplicateCodeSystemUrl",
760                                                                theSystemUri,
761                                                                codeSystemVersionEntity
762                                                                                .getResource()
763                                                                                .getIdDt()
764                                                                                .toUnqualifiedVersionless()
765                                                                                .getValue());
766                        }
767                } else {
768                        // Check if a TermCodeSystemVersion entity already exists for this TermCodeSystem and version.
769                        codeSystemVersionEntity =
770                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystem.getPid(), theSystemVersionId);
771                        if (codeSystemVersionEntity != null) {
772                                msg = myContext
773                                                .getLocalizer()
774                                                .getMessage(
775                                                                TermReadSvcImpl.class,
776                                                                "cannotCreateDuplicateCodeSystemUrlAndVersion",
777                                                                theSystemUri,
778                                                                theSystemVersionId,
779                                                                codeSystemVersionEntity
780                                                                                .getResource()
781                                                                                .getIdDt()
782                                                                                .toUnqualifiedVersionless()
783                                                                                .getValue());
784                        }
785                }
786                // Throw exception if the TermCodeSystemVersion is being duplicated.
787                if (codeSystemVersionEntity != null) {
788                        if (!ObjectUtil.equals(codeSystemVersionEntity.getResource().getId(), theCodeSystemResourceTable.getId())) {
789                                throw new UnprocessableEntityException(Msg.code(848) + msg);
790                        }
791                }
792        }
793
794        private void populateCodeSystemVersionProperties(
795                        TermCodeSystemVersion theCodeSystemVersion,
796                        CodeSystem theCodeSystemResource,
797                        ResourceTable theResourceTable) {
798                theCodeSystemVersion.setResource(theResourceTable);
799                theCodeSystemVersion.setCodeSystemDisplayName(theCodeSystemResource.getName());
800                theCodeSystemVersion.setCodeSystemVersionId(theCodeSystemResource.getVersion());
801        }
802
803        private int validateConceptForStorage(
804                        TermConcept theConcept,
805                        TermCodeSystemVersion theCodeSystemVersion,
806                        ArrayList<String> theConceptsStack,
807                        IdentityHashMap<TermConcept, Object> theAllConcepts) {
808                ValidateUtil.isTrueOrThrowInvalidRequest(
809                                theConcept.getCodeSystemVersion() != null, "CodeSystemVersion is null");
810                ValidateUtil.isNotBlankOrThrowInvalidRequest(
811                                theConcept.getCode(), "CodeSystem contains a code with no code value");
812
813                theConcept.setCodeSystemVersion(theCodeSystemVersion);
814                if (theConceptsStack.contains(theConcept.getCode())) {
815                        throw new InvalidRequestException(
816                                        Msg.code(849) + "CodeSystem contains circular reference around code " + theConcept.getCode());
817                }
818                theConceptsStack.add(theConcept.getCode());
819
820                int retVal = 0;
821                if (theAllConcepts.put(theConcept, theAllConcepts) == null) {
822                        if (theAllConcepts.size() % 1000 == 0) {
823                                ourLog.info("Have validated {} concepts", theAllConcepts.size());
824                        }
825                        retVal = 1;
826                }
827
828                for (TermConceptParentChildLink next : theConcept.getChildren()) {
829                        next.setCodeSystem(theCodeSystemVersion);
830                        retVal +=
831                                        validateConceptForStorage(next.getChild(), theCodeSystemVersion, theConceptsStack, theAllConcepts);
832                }
833
834                theConceptsStack.remove(theConceptsStack.size() - 1);
835
836                return retVal;
837        }
838}