Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.term;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.i18n.Msg;
025import ca.uhn.fhir.interceptor.model.RequestPartitionId;
026import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
027import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
028import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
029import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
030import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
031import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
032import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
033import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
034import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
035import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
036import ca.uhn.fhir.jpa.entity.TermCodeSystem;
037import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
038import ca.uhn.fhir.jpa.entity.TermConcept;
039import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
040import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
041import ca.uhn.fhir.jpa.entity.TermConceptProperty;
042import ca.uhn.fhir.jpa.model.dao.JpaPid;
043import ca.uhn.fhir.jpa.model.entity.ResourceTable;
044import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
045import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
046import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
047import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc;
048import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
049import ca.uhn.fhir.rest.api.Constants;
050import ca.uhn.fhir.rest.api.server.RequestDetails;
051import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId;
052import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
053import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
054import ca.uhn.fhir.util.ObjectUtil;
055import ca.uhn.fhir.util.ValidateUtil;
056import jakarta.annotation.Nonnull;
057import jakarta.persistence.EntityManager;
058import jakarta.persistence.PersistenceContext;
059import jakarta.persistence.PersistenceContextType;
060import org.apache.commons.lang3.Validate;
061import org.hl7.fhir.instance.model.api.IIdType;
062import org.hl7.fhir.r4.model.CodeSystem;
063import org.hl7.fhir.r4.model.ConceptMap;
064import org.hl7.fhir.r4.model.ValueSet;
065import org.slf4j.Logger;
066import org.slf4j.LoggerFactory;
067import org.springframework.beans.factory.annotation.Autowired;
068import org.springframework.transaction.annotation.Propagation;
069import org.springframework.transaction.annotation.Transactional;
070import org.springframework.transaction.support.TransactionSynchronizationManager;
071
072import java.util.ArrayList;
073import java.util.Arrays;
074import java.util.Collection;
075import java.util.Collections;
076import java.util.Date;
077import java.util.HashMap;
078import java.util.HashSet;
079import java.util.IdentityHashMap;
080import java.util.List;
081import java.util.Map;
082import java.util.Objects;
083import java.util.Optional;
084import java.util.Set;
085import java.util.UUID;
086import java.util.concurrent.atomic.AtomicInteger;
087import java.util.stream.Collectors;
088
089import static ca.uhn.fhir.jpa.api.dao.IDao.RESOURCE_PID_KEY;
090import static org.apache.commons.lang3.StringUtils.isBlank;
091import static org.apache.commons.lang3.StringUtils.isNotBlank;
092import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
093
094public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
095        private static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemStorageSvcImpl.class);
096        private static final Object PLACEHOLDER_OBJECT = new Object();
097
098        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
099        protected EntityManager myEntityManager;
100
101        @Autowired
102        protected ITermCodeSystemDao myCodeSystemDao;
103
104        @Autowired
105        protected ITermCodeSystemVersionDao myCodeSystemVersionDao;
106
107        @Autowired
108        protected ITermConceptDao myConceptDao;
109
110        @Autowired
111        protected ITermConceptPropertyDao myConceptPropertyDao;
112
113        @Autowired
114        protected ITermConceptDesignationDao myConceptDesignationDao;
115
116        @Autowired
117        protected IIdHelperService<JpaPid> myIdHelperService;
118
119        @Autowired
120        private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
121
122        @Autowired
123        private ITermVersionAdapterSvc myTerminologyVersionAdapterSvc;
124
125        @Autowired
126        private ITermDeferredStorageSvc myDeferredStorageSvc;
127
128        @Autowired
129        private FhirContext myContext;
130
131        @Autowired
132        private ITermReadSvc myTerminologySvc;
133
134        @Autowired
135        private JpaStorageSettings myStorageSettings;
136
137        @Autowired
138        private IResourceTableDao myResourceTableDao;
139
140        @Autowired
141        private TermConceptDaoSvc myTermConceptDaoSvc;
142
143        @Transactional
144        @Override
145        public UploadStatistics applyDeltaCodeSystemsAdd(String theSystem, CustomTerminologySet theAdditions) {
146                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
147                validateDstu3OrNewer();
148                theAdditions.validateNoCycleOrThrowInvalidRequest();
149
150                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
151                if (cs == null) {
152                        CodeSystem codeSystemResource = new CodeSystem();
153                        codeSystemResource.setUrl(theSystem);
154                        codeSystemResource.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
155                        if (isBlank(codeSystemResource.getIdElement().getIdPart()) && theSystem.contains(LOINC_LOW)) {
156                                codeSystemResource.setId(LOINC_LOW);
157                        }
158                        myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(codeSystemResource);
159
160                        cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
161                }
162
163                TermCodeSystemVersion csv = cs.getCurrentVersion();
164                Validate.notNull(csv);
165
166                CodeSystem codeSystem = myTerminologySvc.fetchCanonicalCodeSystemFromCompleteContext(theSystem);
167                if (codeSystem.getContent() != CodeSystem.CodeSystemContentMode.NOTPRESENT) {
168                        throw new InvalidRequestException(
169                                        Msg.code(844) + "CodeSystem with url[" + Constants.codeSystemWithDefaultDescription(theSystem)
170                                                        + "] can not apply a delta - wrong content mode: " + codeSystem.getContent());
171                }
172
173                Validate.notNull(cs);
174                Validate.notNull(cs.getPid());
175
176                IIdType codeSystemId = cs.getResource().getIdDt();
177
178                UploadStatistics retVal = new UploadStatistics(codeSystemId);
179                HashMap<String, TermConcept> codeToConcept = new HashMap<>();
180
181                // Add root concepts
182                for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) {
183                        List<String> parentCodes = Collections.emptyList();
184                        addConceptInHierarchy(csv, parentCodes, nextRootConcept, retVal, codeToConcept, 0);
185                }
186
187                return retVal;
188        }
189
190        @Transactional
191        @Override
192        public UploadStatistics applyDeltaCodeSystemsRemove(String theSystem, CustomTerminologySet theValue) {
193                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
194                validateDstu3OrNewer();
195
196                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
197                if (cs == null) {
198                        throw new InvalidRequestException(Msg.code(845) + "Unknown code system: " + theSystem);
199                }
200                IIdType target = cs.getResource().getIdDt();
201
202                AtomicInteger removeCounter = new AtomicInteger(0);
203
204                // We need to delete all termconcepts, and their children. This stream flattens the TermConcepts and their
205                // children into a single set of TermConcept objects retrieved from the DB. Note that we have to do this because
206                // deleteById() in JPA doesnt appear to actually commit or flush a transaction until way later, and we end up
207                // iterating multiple times over the same elements, which screws up our counter.
208
209                // Grab the actual entities
210                List<TermConcept> collect = theValue.getRootConcepts().stream()
211                                .map(val -> myTerminologySvc.findCode(theSystem, val.getCode()))
212                                .filter(Optional::isPresent)
213                                .map(Optional::get)
214                                .collect(Collectors.toList());
215
216                // Iterate over the actual entities and fill out their children
217                Set<TermConcept> allFoundTermConcepts = collect.stream()
218                                .flatMap(concept -> flattenChildren(concept).stream())
219                                .map(suppliedTermConcept -> myTerminologySvc.findCode(theSystem, suppliedTermConcept.getCode()))
220                                .filter(Optional::isPresent)
221                                .map(Optional::get)
222                                .collect(Collectors.toSet());
223
224                // Delete everything about these codes.
225                for (TermConcept code : allFoundTermConcepts) {
226                        deleteEverythingRelatedToConcept(code, removeCounter);
227                }
228
229                return new UploadStatistics(removeCounter.get(), target);
230        }
231
232        private void deleteEverythingRelatedToConcept(TermConcept theConcept, AtomicInteger theRemoveCounter) {
233
234                for (TermConceptParentChildLink nextParent : theConcept.getParents()) {
235                        nextParent.getParent().getChildren().remove(nextParent);
236                        myConceptParentChildLinkDao.deleteById(nextParent.getId());
237                }
238                for (TermConceptParentChildLink nextChild : theConcept.getChildren()) {
239                        nextChild.getChild().getParents().remove(nextChild);
240                        myConceptParentChildLinkDao.deleteById(nextChild.getId());
241                }
242
243                for (TermConceptDesignation next : theConcept.getDesignations()) {
244                        myConceptDesignationDao.deleteById(next.getPid());
245                }
246                theConcept.getDesignations().clear();
247                for (TermConceptProperty next : theConcept.getProperties()) {
248                        myConceptPropertyDao.deleteById(next.getPid());
249                }
250                theConcept.getProperties().clear();
251
252                ourLog.info("Deleting concept {} - Code {}", theConcept.getId(), theConcept.getCode());
253
254                myConceptDao.deleteById(theConcept.getId());
255                //              myEntityManager.remove(theConcept);
256
257                theRemoveCounter.incrementAndGet();
258        }
259
260        private List<TermConcept> flattenChildren(TermConcept theTermConcept) {
261                if (theTermConcept.getChildren().isEmpty()) {
262                        return Arrays.asList(theTermConcept);
263                }
264
265                // Recursively flatten children
266                List<TermConcept> childTermConcepts = theTermConcept.getChildren().stream()
267                                .map(TermConceptParentChildLink::getChild)
268                                .flatMap(childConcept -> flattenChildren(childConcept).stream())
269                                .collect(Collectors.toList());
270
271                // Add itself before its list of children
272                childTermConcepts.add(0, theTermConcept);
273                return childTermConcepts;
274        }
275
276        /**
277         * Returns the number of saved concepts
278         */
279        @Override
280        public int saveConcept(TermConcept theConcept) {
281                return myTermConceptDaoSvc.saveConcept(theConcept);
282        }
283
284        @Override
285        @Transactional(propagation = Propagation.MANDATORY)
286        public void storeNewCodeSystemVersionIfNeeded(
287                        CodeSystem theCodeSystem, ResourceTable theResourceEntity, RequestDetails theRequestDetails) {
288                if (theCodeSystem != null && isNotBlank(theCodeSystem.getUrl())) {
289                        String codeSystemUrl = theCodeSystem.getUrl();
290                        if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.COMPLETE
291                                        || theCodeSystem.getContent() == null
292                                        || theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
293                                ourLog.info(
294                                                "CodeSystem {} has a status of {}, going to store concepts in terminology tables",
295                                                theResourceEntity.getIdDt().getValue(),
296                                                theCodeSystem.getContentElement().getValueAsString());
297
298                                Long pid = (Long) theCodeSystem.getUserData(RESOURCE_PID_KEY);
299                                assert pid != null;
300                                JpaPid codeSystemResourcePid = JpaPid.fromId(pid);
301
302                                /*
303                                 * If this is a not-present codesystem and codesystem version already exists, we don't want to
304                                 * overwrite the existing version since that will wipe out the existing concepts. We do create
305                                 * or update the TermCodeSystem table though, since that allows the DB to reject changes that would
306                                 * result in duplicate CodeSystem.url values.
307                                 */
308                                if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
309                                        TermCodeSystem termCodeSystem = myCodeSystemDao.findByCodeSystemUri(theCodeSystem.getUrl());
310                                        if (termCodeSystem != null) {
311                                                TermCodeSystemVersion codeSystemVersion =
312                                                                getExistingTermCodeSystemVersion(termCodeSystem.getPid(), theCodeSystem.getVersion());
313                                                if (codeSystemVersion != null) {
314                                                        TermCodeSystem myCodeSystemEntity = getOrCreateDistinctTermCodeSystem(
315                                                                        codeSystemResourcePid,
316                                                                        theCodeSystem.getUrl(),
317                                                                        theCodeSystem.getUrl(),
318                                                                        theCodeSystem.getVersion(),
319                                                                        theResourceEntity);
320                                                        return;
321                                                }
322                                        }
323                                }
324
325                                TermCodeSystemVersion persCs = new TermCodeSystemVersion();
326
327                                populateCodeSystemVersionProperties(persCs, theCodeSystem, theResourceEntity);
328
329                                persCs.getConcepts().addAll(TermReadSvcImpl.toPersistedConcepts(theCodeSystem.getConcept(), persCs));
330                                ourLog.debug("Code system has {} concepts", persCs.getConcepts().size());
331                                storeNewCodeSystemVersion(
332                                                codeSystemResourcePid,
333                                                codeSystemUrl,
334                                                theCodeSystem.getName(),
335                                                theCodeSystem.getVersion(),
336                                                persCs,
337                                                theResourceEntity,
338                                                theRequestDetails);
339                        }
340                }
341        }
342
343        @Override
344        @Transactional
345        public IIdType storeNewCodeSystemVersion(
346                        CodeSystem theCodeSystemResource,
347                        TermCodeSystemVersion theCodeSystemVersion,
348                        RequestDetails theRequest,
349                        List<ValueSet> theValueSets,
350                        List<ConceptMap> theConceptMaps) {
351                assert TransactionSynchronizationManager.isActualTransactionActive();
352
353                Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
354
355                // Note that this creates the TermCodeSystem and TermCodeSystemVersion entities if needed
356                IIdType csId = myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(theCodeSystemResource, theRequest);
357
358                JpaPid codeSystemResourcePid = myIdHelperService.resolveResourcePersistentIds(
359                                RequestPartitionId.allPartitions(), csId.getResourceType(), csId.getIdPart());
360                ResourceTable resource = myResourceTableDao.getOne(codeSystemResourcePid.getId());
361
362                ourLog.info("CodeSystem resource has ID: {}", csId.getValue());
363
364                populateCodeSystemVersionProperties(theCodeSystemVersion, theCodeSystemResource, resource);
365
366                storeNewCodeSystemVersion(
367                                codeSystemResourcePid,
368                                theCodeSystemResource.getUrl(),
369                                theCodeSystemResource.getName(),
370                                theCodeSystemResource.getVersion(),
371                                theCodeSystemVersion,
372                                resource,
373                                theRequest);
374
375                myDeferredStorageSvc.addConceptMapsToStorageQueue(theConceptMaps);
376                myDeferredStorageSvc.addValueSetsToStorageQueue(theValueSets);
377
378                return csId;
379        }
380
381        @Override
382        @Transactional
383        public void storeNewCodeSystemVersion(
384                        IResourcePersistentId theCodeSystemResourcePid,
385                        String theSystemUri,
386                        String theSystemName,
387                        String theCodeSystemVersionId,
388                        TermCodeSystemVersion theCodeSystemVersion,
389                        ResourceTable theCodeSystemResourceTable,
390                        RequestDetails theRequestDetails) {
391                assert TransactionSynchronizationManager.isActualTransactionActive();
392
393                ourLog.debug("Storing code system");
394
395                TermCodeSystemVersion codeSystemToStore = theCodeSystemVersion;
396                ValidateUtil.isTrueOrThrowInvalidRequest(codeSystemToStore.getResource() != null, "No resource supplied");
397                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied");
398
399                TermCodeSystem codeSystem = getOrCreateDistinctTermCodeSystem(
400                                theCodeSystemResourcePid,
401                                theSystemUri,
402                                theSystemName,
403                                theCodeSystemVersionId,
404                                theCodeSystemResourceTable);
405
406                List<TermCodeSystemVersion> existing =
407                                myCodeSystemVersionDao.findByCodeSystemResourcePid(((JpaPid) theCodeSystemResourcePid).getId());
408                for (TermCodeSystemVersion next : existing) {
409                        if (Objects.equals(next.getCodeSystemVersionId(), theCodeSystemVersionId)
410                                        && myConceptDao.countByCodeSystemVersion(next.getPid()) == 0) {
411
412                                /*
413                                 * If we already have a CodeSystemVersion that matches the version we're storing, we
414                                 * can reuse it.
415                                 */
416                                next.setCodeSystemDisplayName(theSystemName);
417                                codeSystemToStore = next;
418
419                        } else {
420
421                                /*
422                                 * If we already have a TermCodeSystemVersion that corresponds to the FHIR Resource ID we're
423                                 * adding a version to, we will mark it for deletion. For any one resource there can only
424                                 * be one TermCodeSystemVersion entity in the DB. Multiple versions of a codesystem uses
425                                 * multiple CodeSystem resources with CodeSystem.version set differently (as opposed to
426                                 * multiple versions of the same CodeSystem, where CodeSystem.meta.versionId is different)
427                                 */
428                                next.setCodeSystemVersionId("DELETED_" + UUID.randomUUID().toString());
429                                myCodeSystemVersionDao.saveAndFlush(next);
430                                myDeferredStorageSvc.deleteCodeSystemVersion(next);
431                        }
432                }
433
434                /*
435                 * Do the upload
436                 */
437
438                codeSystemToStore.setCodeSystem(codeSystem);
439                codeSystemToStore.setCodeSystemDisplayName(theSystemName);
440                codeSystemToStore.setCodeSystemVersionId(theCodeSystemVersionId);
441
442                ourLog.debug("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
443
444                // Validate the code system
445                ArrayList<String> conceptsStack = new ArrayList<>();
446                IdentityHashMap<TermConcept, Object> allConcepts = new IdentityHashMap<>();
447                int totalCodeCount = 0;
448                Collection<TermConcept> conceptsToSave = theCodeSystemVersion.getConcepts();
449                for (TermConcept next : conceptsToSave) {
450                        totalCodeCount += validateConceptForStorage(next, codeSystemToStore, conceptsStack, allConcepts);
451                }
452
453                ourLog.debug("Saving version containing {} concepts", totalCodeCount);
454                if (codeSystemToStore.getPid() == null) {
455                        codeSystemToStore = myCodeSystemVersionDao.saveAndFlush(codeSystemToStore);
456                }
457
458                boolean isMakeVersionCurrent = ITermCodeSystemStorageSvc.isMakeVersionCurrent(theRequestDetails);
459                if (isMakeVersionCurrent) {
460                        codeSystem.setCurrentVersion(codeSystemToStore);
461                        if (codeSystem.getPid() == null) {
462                                codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
463                        }
464                }
465
466                ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
467                for (TermConcept next : conceptsToSave) {
468                        populateVersion(next, codeSystemToStore);
469                }
470
471                ourLog.debug("Saving {} concepts...", totalCodeCount);
472                IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
473                for (TermConcept next : conceptsToSave) {
474                        persistChildren(next, codeSystemToStore, conceptsStack2, totalCodeCount);
475                }
476
477                ourLog.debug("Done saving concepts, flushing to database");
478                if (!myDeferredStorageSvc.isStorageQueueEmpty(true)) {
479                        ourLog.info("Note that some concept saving has been deferred");
480                }
481        }
482
483        private TermCodeSystemVersion getExistingTermCodeSystemVersion(
484                        Long theCodeSystemVersionPid, String theCodeSystemVersion) {
485                TermCodeSystemVersion existing;
486                if (theCodeSystemVersion == null) {
487                        existing = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystemVersionPid);
488                } else {
489                        existing =
490                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystemVersionPid, theCodeSystemVersion);
491                }
492
493                return existing;
494        }
495
496        private void validateDstu3OrNewer() {
497                Validate.isTrue(
498                                myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3),
499                                "Terminology operations only supported in DSTU3+ mode");
500        }
501
502        private void addConceptInHierarchy(
503                        TermCodeSystemVersion theCsv,
504                        Collection<String> theParentCodes,
505                        TermConcept theConceptToAdd,
506                        UploadStatistics theStatisticsTracker,
507                        Map<String, TermConcept> theCodeToConcept,
508                        int theSequence) {
509                TermConcept conceptToAdd = theConceptToAdd;
510                List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren();
511
512                String nextCodeToAdd = conceptToAdd.getCode();
513                String parentDescription = "(root concept)";
514
515                ourLog.info(
516                                "Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription);
517
518                Optional<TermConcept> existingCodeOpt = myConceptDao.findByCodeSystemAndCode(theCsv.getPid(), nextCodeToAdd);
519                List<TermConceptParentChildLink> existingParentLinks;
520                if (existingCodeOpt.isPresent()) {
521                        TermConcept existingCode = existingCodeOpt.get();
522                        existingCode.setIndexStatus(null);
523                        existingCode.setDisplay(conceptToAdd.getDisplay());
524                        conceptToAdd = existingCode;
525                        existingParentLinks = conceptToAdd.getParents();
526                } else {
527                        existingParentLinks = Collections.emptyList();
528                }
529
530                Set<TermConcept> parentConceptsWeShouldLinkTo = new HashSet<>();
531                for (String nextParentCode : theParentCodes) {
532
533                        // Don't add parent links that already exist for the code
534                        if (existingParentLinks.stream()
535                                        .anyMatch(t -> t.getParent().getCode().equals(nextParentCode))) {
536                                continue;
537                        }
538
539                        TermConcept nextParentOpt = theCodeToConcept.get(nextParentCode);
540                        if (nextParentOpt == null) {
541                                nextParentOpt = myConceptDao
542                                                .findByCodeSystemAndCode(theCsv.getPid(), nextParentCode)
543                                                .orElse(null);
544                        }
545                        if (nextParentOpt == null) {
546                                throw new InvalidRequestException(Msg.code(846) + "Unable to add code \"" + nextCodeToAdd
547                                                + "\" to unknown parent: " + nextParentCode);
548                        }
549                        parentConceptsWeShouldLinkTo.add(nextParentOpt);
550                }
551
552                if (conceptToAdd.getSequence() == null) {
553                        conceptToAdd.setSequence(theSequence);
554                }
555
556                // Null out the hierarchy PIDs for this concept always. We do this because we're going to
557                // force a reindex, and it'll be regenerated then
558                conceptToAdd.setParentPids(null);
559                conceptToAdd.setCodeSystemVersion(theCsv);
560
561                if (conceptToAdd.getProperties() != null)
562                        conceptToAdd.getProperties().forEach(termConceptProperty -> {
563                                termConceptProperty.setConcept(theConceptToAdd);
564                                termConceptProperty.setCodeSystemVersion(theCsv);
565                        });
566                if (theStatisticsTracker.getUpdatedConceptCount() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
567                        saveConcept(conceptToAdd);
568                        Long nextConceptPid = conceptToAdd.getId();
569                        Validate.notNull(nextConceptPid);
570                } else {
571                        myDeferredStorageSvc.addConceptToStorageQueue(conceptToAdd);
572                }
573
574                theCodeToConcept.put(conceptToAdd.getCode(), conceptToAdd);
575
576                theStatisticsTracker.incrementUpdatedConceptCount();
577
578                // Add link to new child to the parent
579                for (TermConcept nextParentConcept : parentConceptsWeShouldLinkTo) {
580                        TermConceptParentChildLink parentLink = new TermConceptParentChildLink();
581                        parentLink.setParent(nextParentConcept);
582                        parentLink.setChild(conceptToAdd);
583                        parentLink.setCodeSystem(theCsv);
584                        parentLink.setRelationshipType(TermConceptParentChildLink.RelationshipTypeEnum.ISA);
585                        nextParentConcept.getChildren().add(parentLink);
586                        conceptToAdd.getParents().add(parentLink);
587                        ourLog.info(
588                                        "Saving parent/child link - Parent[{}] Child[{}]",
589                                        parentLink.getParent().getCode(),
590                                        parentLink.getChild().getCode());
591
592                        if (theStatisticsTracker.getUpdatedConceptCount()
593                                        <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
594                                myConceptParentChildLinkDao.save(parentLink);
595                        } else {
596                                myDeferredStorageSvc.addConceptLinkToStorageQueue(parentLink);
597                        }
598                }
599
600                ourLog.trace("About to save parent-child links");
601
602                // Save children recursively
603                int childIndex = 0;
604                for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) {
605
606                        TermConcept nextChild = nextChildConceptLink.getChild();
607
608                        for (int i = 0; i < nextChild.getParents().size(); i++) {
609                                if (nextChild.getParents().get(i).getId() == null) {
610                                        String parentCode =
611                                                        nextChild.getParents().get(i).getParent().getCode();
612                                        TermConcept parentConcept = theCodeToConcept.get(parentCode);
613                                        if (parentConcept == null) {
614                                                parentConcept = myConceptDao
615                                                                .findByCodeSystemAndCode(theCsv.getPid(), parentCode)
616                                                                .orElse(null);
617                                        }
618                                        if (parentConcept == null) {
619                                                throw new IllegalArgumentException(Msg.code(847) + "Unknown parent code: " + parentCode);
620                                        }
621
622                                        nextChild.getParents().get(i).setParent(parentConcept);
623                                }
624                        }
625
626                        Collection<String> parentCodes = nextChild.getParents().stream()
627                                        .map(t -> t.getParent().getCode())
628                                        .collect(Collectors.toList());
629                        addConceptInHierarchy(theCsv, parentCodes, nextChild, theStatisticsTracker, theCodeToConcept, childIndex);
630
631                        childIndex++;
632                }
633        }
634
635        private void persistChildren(
636                        TermConcept theConcept,
637                        TermCodeSystemVersion theCodeSystem,
638                        IdentityHashMap<TermConcept, Object> theConceptsStack,
639                        int theTotalConcepts) {
640                if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) {
641                        return;
642                }
643
644                if ((theConceptsStack.size() + 1) % 10000 == 0) {
645                        float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
646                        ourLog.info(
647                                        "Have processed {}/{} concepts ({}%)",
648                                        theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
649                }
650
651                theConcept.setCodeSystemVersion(theCodeSystem);
652                theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);
653
654                if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
655                        saveConcept(theConcept);
656                } else {
657                        myDeferredStorageSvc.addConceptToStorageQueue(theConcept);
658                }
659
660                for (TermConceptParentChildLink next : theConcept.getChildren()) {
661                        persistChildren(next.getChild(), theCodeSystem, theConceptsStack, theTotalConcepts);
662                }
663
664                for (TermConceptParentChildLink next : theConcept.getChildren()) {
665                        if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
666                                saveConceptLink(next);
667                        } else {
668                                myDeferredStorageSvc.addConceptLinkToStorageQueue(next);
669                        }
670                }
671        }
672
673        private void populateVersion(TermConcept theNext, TermCodeSystemVersion theCodeSystemVersion) {
674                theNext.setCodeSystemVersion(theCodeSystemVersion);
675                for (TermConceptParentChildLink next : theNext.getChildren()) {
676                        populateVersion(next.getChild(), theCodeSystemVersion);
677                }
678                theNext.getProperties().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
679                theNext.getDesignations().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
680        }
681
682        private void saveConceptLink(TermConceptParentChildLink next) {
683                if (next.getId() == null) {
684                        myConceptParentChildLinkDao.save(next);
685                }
686        }
687
688        private int ensureParentsSaved(Collection<TermConceptParentChildLink> theParents) {
689                ourLog.trace("Checking {} parents", theParents.size());
690                int retVal = 0;
691
692                for (TermConceptParentChildLink nextLink : theParents) {
693                        if (nextLink.getRelationshipType() == TermConceptParentChildLink.RelationshipTypeEnum.ISA) {
694                                TermConcept nextParent = nextLink.getParent();
695                                retVal += ensureParentsSaved(nextParent.getParents());
696                                if (nextParent.getId() == null) {
697                                        nextParent.setUpdated(new Date());
698                                        myConceptDao.saveAndFlush(nextParent);
699                                        retVal++;
700                                        ourLog.debug("Saved parent code {} and got id {}", nextParent.getCode(), nextParent.getId());
701                                }
702                        }
703                }
704
705                return retVal;
706        }
707
708        @Nonnull
709        private TermCodeSystem getOrCreateDistinctTermCodeSystem(
710                        IResourcePersistentId theCodeSystemResourcePid,
711                        String theSystemUri,
712                        String theSystemName,
713                        String theSystemVersionId,
714                        ResourceTable theCodeSystemResourceTable) {
715                TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(theSystemUri);
716                if (codeSystem == null) {
717                        codeSystem = myCodeSystemDao.findByResourcePid(((JpaPid) theCodeSystemResourcePid).getId());
718                        if (codeSystem == null) {
719                                codeSystem = new TermCodeSystem();
720                        }
721                } else {
722                        checkForCodeSystemVersionDuplicate(
723                                        codeSystem, theSystemUri, theSystemVersionId, theCodeSystemResourceTable);
724                }
725
726                codeSystem.setResource(theCodeSystemResourceTable);
727                codeSystem.setCodeSystemUri(theSystemUri);
728                codeSystem.setName(theSystemName);
729                codeSystem = myCodeSystemDao.save(codeSystem);
730                return codeSystem;
731        }
732
733        private void checkForCodeSystemVersionDuplicate(
734                        TermCodeSystem theCodeSystem,
735                        String theSystemUri,
736                        String theSystemVersionId,
737                        ResourceTable theCodeSystemResourceTable) {
738                TermCodeSystemVersion codeSystemVersionEntity;
739                String msg = null;
740                if (theSystemVersionId == null) {
741                        // Check if a non-versioned TermCodeSystemVersion entity already exists for this TermCodeSystem.
742                        codeSystemVersionEntity = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystem.getPid());
743                        if (codeSystemVersionEntity != null) {
744                                msg = myContext
745                                                .getLocalizer()
746                                                .getMessage(
747                                                                TermReadSvcImpl.class,
748                                                                "cannotCreateDuplicateCodeSystemUrl",
749                                                                theSystemUri,
750                                                                codeSystemVersionEntity
751                                                                                .getResource()
752                                                                                .getIdDt()
753                                                                                .toUnqualifiedVersionless()
754                                                                                .getValue());
755                        }
756                } else {
757                        // Check if a TermCodeSystemVersion entity already exists for this TermCodeSystem and version.
758                        codeSystemVersionEntity =
759                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystem.getPid(), theSystemVersionId);
760                        if (codeSystemVersionEntity != null) {
761                                msg = myContext
762                                                .getLocalizer()
763                                                .getMessage(
764                                                                TermReadSvcImpl.class,
765                                                                "cannotCreateDuplicateCodeSystemUrlAndVersion",
766                                                                theSystemUri,
767                                                                theSystemVersionId,
768                                                                codeSystemVersionEntity
769                                                                                .getResource()
770                                                                                .getIdDt()
771                                                                                .toUnqualifiedVersionless()
772                                                                                .getValue());
773                        }
774                }
775                // Throw exception if the TermCodeSystemVersion is being duplicated.
776                if (codeSystemVersionEntity != null) {
777                        if (!ObjectUtil.equals(codeSystemVersionEntity.getResource().getId(), theCodeSystemResourceTable.getId())) {
778                                throw new UnprocessableEntityException(Msg.code(848) + msg);
779                        }
780                }
781        }
782
783        private void populateCodeSystemVersionProperties(
784                        TermCodeSystemVersion theCodeSystemVersion,
785                        CodeSystem theCodeSystemResource,
786                        ResourceTable theResourceTable) {
787                theCodeSystemVersion.setResource(theResourceTable);
788                theCodeSystemVersion.setCodeSystemDisplayName(theCodeSystemResource.getName());
789                theCodeSystemVersion.setCodeSystemVersionId(theCodeSystemResource.getVersion());
790        }
791
792        private int validateConceptForStorage(
793                        TermConcept theConcept,
794                        TermCodeSystemVersion theCodeSystemVersion,
795                        ArrayList<String> theConceptsStack,
796                        IdentityHashMap<TermConcept, Object> theAllConcepts) {
797                ValidateUtil.isTrueOrThrowInvalidRequest(
798                                theConcept.getCodeSystemVersion() != null, "CodeSystemVersion is null");
799                ValidateUtil.isNotBlankOrThrowInvalidRequest(
800                                theConcept.getCode(), "CodeSystem contains a code with no code value");
801
802                theConcept.setCodeSystemVersion(theCodeSystemVersion);
803                if (theConceptsStack.contains(theConcept.getCode())) {
804                        throw new InvalidRequestException(
805                                        Msg.code(849) + "CodeSystem contains circular reference around code " + theConcept.getCode());
806                }
807                theConceptsStack.add(theConcept.getCode());
808
809                int retVal = 0;
810                if (theAllConcepts.put(theConcept, theAllConcepts) == null) {
811                        if (theAllConcepts.size() % 1000 == 0) {
812                                ourLog.info("Have validated {} concepts", theAllConcepts.size());
813                        }
814                        retVal = 1;
815                }
816
817                for (TermConceptParentChildLink next : theConcept.getChildren()) {
818                        next.setCodeSystem(theCodeSystemVersion);
819                        retVal +=
820                                        validateConceptForStorage(next.getChild(), theCodeSystemVersion, theConceptsStack, theAllConcepts);
821                }
822
823                theConceptsStack.remove(theConceptsStack.size() - 1);
824
825                return retVal;
826        }
827}