Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.term;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.i18n.Msg;
025import ca.uhn.fhir.interceptor.model.RequestPartitionId;
026import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
027import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
028import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode;
029import ca.uhn.fhir.jpa.dao.data.IResourceTableDao;
030import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
031import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
032import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
033import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
034import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao;
035import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
036import ca.uhn.fhir.jpa.entity.TermCodeSystem;
037import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
038import ca.uhn.fhir.jpa.entity.TermConcept;
039import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
040import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
041import ca.uhn.fhir.jpa.entity.TermConceptProperty;
042import ca.uhn.fhir.jpa.model.dao.JpaPid;
043import ca.uhn.fhir.jpa.model.entity.EntityIndexStatusEnum;
044import ca.uhn.fhir.jpa.model.entity.ResourceTable;
045import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
046import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
047import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
048import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc;
049import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
050import ca.uhn.fhir.rest.api.Constants;
051import ca.uhn.fhir.rest.api.server.RequestDetails;
052import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
053import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException;
054import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
055import ca.uhn.fhir.util.ObjectUtil;
056import ca.uhn.fhir.util.UrlUtil;
057import ca.uhn.fhir.util.ValidateUtil;
058import jakarta.annotation.Nonnull;
059import jakarta.persistence.EntityManager;
060import jakarta.persistence.PersistenceContext;
061import jakarta.persistence.PersistenceContextType;
062import org.apache.commons.lang3.Validate;
063import org.hl7.fhir.instance.model.api.IIdType;
064import org.hl7.fhir.r4.model.CodeSystem;
065import org.hl7.fhir.r4.model.ConceptMap;
066import org.hl7.fhir.r4.model.ValueSet;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069import org.springframework.beans.factory.annotation.Autowired;
070import org.springframework.transaction.annotation.Propagation;
071import org.springframework.transaction.annotation.Transactional;
072import org.springframework.transaction.support.TransactionSynchronizationManager;
073
074import java.util.ArrayList;
075import java.util.Arrays;
076import java.util.Collection;
077import java.util.Collections;
078import java.util.Date;
079import java.util.HashMap;
080import java.util.HashSet;
081import java.util.IdentityHashMap;
082import java.util.List;
083import java.util.Map;
084import java.util.Objects;
085import java.util.Optional;
086import java.util.Set;
087import java.util.UUID;
088import java.util.concurrent.atomic.AtomicInteger;
089import java.util.stream.Collectors;
090
091import static org.apache.commons.lang3.StringUtils.isBlank;
092import static org.apache.commons.lang3.StringUtils.isNotBlank;
093import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
094
095public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc {
096        private static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemStorageSvcImpl.class);
097        private static final Object PLACEHOLDER_OBJECT = new Object();
098
099        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
100        protected EntityManager myEntityManager;
101
102        @Autowired
103        protected ITermCodeSystemDao myCodeSystemDao;
104
105        @Autowired
106        protected ITermCodeSystemVersionDao myCodeSystemVersionDao;
107
108        @Autowired
109        protected ITermConceptDao myConceptDao;
110
111        @Autowired
112        protected ITermConceptPropertyDao myConceptPropertyDao;
113
114        @Autowired
115        protected ITermConceptDesignationDao myConceptDesignationDao;
116
117        @Autowired
118        protected IIdHelperService<JpaPid> myIdHelperService;
119
120        @Autowired
121        private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
122
123        @Autowired
124        private ITermVersionAdapterSvc myTerminologyVersionAdapterSvc;
125
126        @Autowired
127        private ITermDeferredStorageSvc myDeferredStorageSvc;
128
129        @Autowired
130        private FhirContext myContext;
131
132        @Autowired
133        private ITermReadSvc myTerminologySvc;
134
135        @Autowired
136        private JpaStorageSettings myStorageSettings;
137
138        @Autowired
139        private IResourceTableDao myResourceTableDao;
140
141        @Autowired
142        private TermConceptDaoSvc myTermConceptDaoSvc;
143
144        @Transactional
145        @Override
146        public UploadStatistics applyDeltaCodeSystemsAdd(String theSystem, CustomTerminologySet theAdditions) {
147                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
148                validateDstu3OrNewer();
149                theAdditions.validateNoCycleOrThrowInvalidRequest();
150
151                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
152                if (cs == null) {
153                        CodeSystem codeSystemResource = new CodeSystem();
154                        codeSystemResource.setUrl(theSystem);
155                        codeSystemResource.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
156                        if (isBlank(codeSystemResource.getIdElement().getIdPart()) && theSystem.contains(LOINC_LOW)) {
157                                codeSystemResource.setId(LOINC_LOW);
158                        }
159                        myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(codeSystemResource);
160
161                        cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
162                }
163
164                TermCodeSystemVersion csv = cs.getCurrentVersion();
165                Validate.notNull(csv);
166
167                CodeSystem codeSystem = myTerminologySvc.fetchCanonicalCodeSystemFromCompleteContext(theSystem);
168                if (codeSystem.getContent() != CodeSystem.CodeSystemContentMode.NOTPRESENT) {
169                        throw new InvalidRequestException(
170                                        Msg.code(844) + "CodeSystem with url[" + Constants.codeSystemWithDefaultDescription(theSystem)
171                                                        + "] can not apply a delta - wrong content mode: " + codeSystem.getContent());
172                }
173
174                Validate.notNull(cs);
175                Validate.notNull(cs.getPid());
176
177                IIdType codeSystemId = cs.getResource().getIdDt();
178
179                UploadStatistics retVal = new UploadStatistics(codeSystemId);
180                HashMap<String, TermConcept> codeToConcept = new HashMap<>();
181
182                // Add root concepts
183                for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) {
184                        List<String> parentCodes = Collections.emptyList();
185                        addConceptInHierarchy(csv, parentCodes, nextRootConcept, retVal, codeToConcept, 0);
186                }
187
188                return retVal;
189        }
190
191        @Transactional
192        @Override
193        public UploadStatistics applyDeltaCodeSystemsRemove(String theSystem, CustomTerminologySet theValue) {
194                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided");
195                validateDstu3OrNewer();
196
197                TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem);
198                if (cs == null) {
199                        throw new InvalidRequestException(Msg.code(845) + "Unknown code system: " + theSystem);
200                }
201                IIdType target = cs.getResource().getIdDt();
202
203                AtomicInteger removeCounter = new AtomicInteger(0);
204
205                // We need to delete all termconcepts, and their children. This stream flattens the TermConcepts and their
206                // children into a single set of TermConcept objects retrieved from the DB. Note that we have to do this because
207                // deleteById() in JPA doesnt appear to actually commit or flush a transaction until way later, and we end up
208                // iterating multiple times over the same elements, which screws up our counter.
209
210                // Grab the actual entities
211                List<TermConcept> collect = theValue.getRootConcepts().stream()
212                                .map(val -> myTerminologySvc.findCode(theSystem, val.getCode()))
213                                .filter(Optional::isPresent)
214                                .map(Optional::get)
215                                .collect(Collectors.toList());
216
217                // Iterate over the actual entities and fill out their children
218                Set<TermConcept> allFoundTermConcepts = collect.stream()
219                                .flatMap(concept -> flattenChildren(concept).stream())
220                                .map(suppliedTermConcept -> myTerminologySvc.findCode(theSystem, suppliedTermConcept.getCode()))
221                                .filter(Optional::isPresent)
222                                .map(Optional::get)
223                                .collect(Collectors.toSet());
224
225                // Delete everything about these codes.
226                for (TermConcept code : allFoundTermConcepts) {
227                        deleteEverythingRelatedToConcept(code, removeCounter);
228                }
229
230                return new UploadStatistics(removeCounter.get(), target);
231        }
232
233        private void deleteEverythingRelatedToConcept(TermConcept theConcept, AtomicInteger theRemoveCounter) {
234
235                for (TermConceptParentChildLink nextParent : theConcept.getParents()) {
236                        nextParent.getParent().getChildren().remove(nextParent);
237                        myConceptParentChildLinkDao.deleteById(nextParent.getPid());
238                }
239                for (TermConceptParentChildLink nextChild : theConcept.getChildren()) {
240                        nextChild.getChild().getParents().remove(nextChild);
241                        myConceptParentChildLinkDao.deleteById(nextChild.getPid());
242                }
243
244                for (TermConceptDesignation next : theConcept.getDesignations()) {
245                        myConceptDesignationDao.deleteById(next.getPartitionedId());
246                }
247                theConcept.getDesignations().clear();
248                for (TermConceptProperty next : theConcept.getProperties()) {
249                        myConceptPropertyDao.deleteById(next.getPartitionedId());
250                }
251                theConcept.getProperties().clear();
252
253                ourLog.info("Deleting concept {} - Code {}", theConcept.getId(), theConcept.getCode());
254
255                myConceptDao.deleteById(theConcept.getPid());
256                //              myEntityManager.remove(theConcept);
257
258                theRemoveCounter.incrementAndGet();
259        }
260
261        private List<TermConcept> flattenChildren(TermConcept theTermConcept) {
262                if (theTermConcept.getChildren().isEmpty()) {
263                        return Arrays.asList(theTermConcept);
264                }
265
266                // Recursively flatten children
267                List<TermConcept> childTermConcepts = theTermConcept.getChildren().stream()
268                                .map(TermConceptParentChildLink::getChild)
269                                .flatMap(childConcept -> flattenChildren(childConcept).stream())
270                                .collect(Collectors.toList());
271
272                // Add itself before its list of children
273                childTermConcepts.add(0, theTermConcept);
274                return childTermConcepts;
275        }
276
277        /**
278         * Returns the number of saved concepts
279         */
280        @Override
281        public int saveConcept(TermConcept theConcept) {
282                return myTermConceptDaoSvc.saveConcept(theConcept);
283        }
284
285        @Override
286        @Transactional(propagation = Propagation.MANDATORY)
287        public void storeNewCodeSystemVersionIfNeeded(
288                        CodeSystem theCodeSystem, ResourceTable theResourceEntity, RequestDetails theRequestDetails) {
289                if (theCodeSystem != null && isNotBlank(theCodeSystem.getUrl())) {
290                        String codeSystemUrl = theCodeSystem.getUrl();
291                        if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.COMPLETE
292                                        || theCodeSystem.getContent() == null
293                                        || theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
294                                ourLog.info(
295                                                "CodeSystem {} has a status of {}, going to store concepts in terminology tables",
296                                                theResourceEntity.getIdDt().getValue(),
297                                                theCodeSystem.getContentElement().getValueAsString());
298
299                                detectDuplicatesInCodeSystem(theCodeSystem);
300
301                                /*
302                                 * If this is a not-present codesystem and codesystem version already exists, we don't want to
303                                 * overwrite the existing version since that will wipe out the existing concepts. We do create
304                                 * or update the TermCodeSystem table though, since that allows the DB to reject changes that would
305                                 * result in duplicate CodeSystem.url values.
306                                 */
307                                if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) {
308                                        TermCodeSystem termCodeSystem = myCodeSystemDao.findByCodeSystemUri(theCodeSystem.getUrl());
309                                        if (termCodeSystem != null) {
310                                                TermCodeSystemVersion codeSystemVersion =
311                                                                getExistingTermCodeSystemVersion(termCodeSystem.getPid(), theCodeSystem.getVersion());
312                                                if (codeSystemVersion != null) {
313                                                        getOrCreateDistinctTermCodeSystem(
314                                                                        theCodeSystem.getUrl(),
315                                                                        theCodeSystem.getUrl(),
316                                                                        theCodeSystem.getVersion(),
317                                                                        theResourceEntity);
318                                                        return;
319                                                }
320                                        }
321                                }
322
323                                TermCodeSystemVersion persCs = new TermCodeSystemVersion();
324                                populateCodeSystemVersionProperties(persCs, theCodeSystem, theResourceEntity);
325                                myEntityManager.persist(persCs);
326
327                                persCs.getConcepts().addAll(TermReadSvcImpl.toPersistedConcepts(theCodeSystem.getConcept(), persCs));
328                                ourLog.debug("Code system has {} concepts", persCs.getConcepts().size());
329                                storeNewCodeSystemVersion(
330                                                codeSystemUrl,
331                                                theCodeSystem.getName(),
332                                                theCodeSystem.getVersion(),
333                                                persCs,
334                                                theResourceEntity,
335                                                theRequestDetails);
336                        }
337                }
338        }
339
340        private static void detectDuplicatesInCodeSystem(CodeSystem theCodeSystem) {
341                detectDuplicatesInCodeSystem(theCodeSystem.getConcept(), new HashSet<>());
342        }
343
344        private static void detectDuplicatesInCodeSystem(
345                        List<CodeSystem.ConceptDefinitionComponent> theCodeList, Set<String> theFoundCodesBuffer) {
346                for (var next : theCodeList) {
347                        if (isNotBlank(next.getCode())) {
348                                if (!theFoundCodesBuffer.add(next.getCode())) {
349                                        /*
350                                         * Note: We could possibly modify this behaviour to be forgiving, and just
351                                         * ignore duplicates. The only issue is that concepts can have properties,
352                                         * designations, etc. and it could be dangerous to just pick one and ignore the
353                                         * other. So the safer thing seems to be to just throw an error.
354                                         */
355                                        throw new PreconditionFailedException(Msg.code(2528) + "Duplicate concept detected in CodeSystem: "
356                                                        + UrlUtil.sanitizeUrlPart(next.getCode()));
357                                }
358                        }
359                        // Test child concepts within the parent concept
360                        detectDuplicatesInCodeSystem(next.getConcept(), theFoundCodesBuffer);
361                }
362        }
363
364        @Override
365        @Transactional
366        public IIdType storeNewCodeSystemVersion(
367                        CodeSystem theCodeSystemResource,
368                        TermCodeSystemVersion theCodeSystemVersion,
369                        RequestDetails theRequest,
370                        List<ValueSet> theValueSets,
371                        List<ConceptMap> theConceptMaps) {
372                assert TransactionSynchronizationManager.isActualTransactionActive();
373
374                Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
375
376                // Note that this creates the TermCodeSystem and TermCodeSystemVersion entities if needed
377                IIdType csId = myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(theCodeSystemResource, theRequest);
378
379                JpaPid codeSystemResourcePid = myIdHelperService.resolveResourceIdentityPid(
380                                RequestPartitionId.allPartitions(),
381                                csId.getResourceType(),
382                                csId.getIdPart(),
383                                ResolveIdentityMode.includeDeleted().cacheOk());
384                ResourceTable resource = myResourceTableDao.getOne(codeSystemResourcePid);
385
386                ourLog.info("CodeSystem resource has ID: {}", csId.getValue());
387
388                populateCodeSystemVersionProperties(theCodeSystemVersion, theCodeSystemResource, resource);
389
390                storeNewCodeSystemVersion(
391                                theCodeSystemResource.getUrl(),
392                                theCodeSystemResource.getName(),
393                                theCodeSystemResource.getVersion(),
394                                theCodeSystemVersion,
395                                resource,
396                                theRequest);
397
398                myDeferredStorageSvc.addConceptMapsToStorageQueue(theConceptMaps);
399                myDeferredStorageSvc.addValueSetsToStorageQueue(theValueSets);
400
401                return csId;
402        }
403
404        @Override
405        @Transactional
406        public void storeNewCodeSystemVersion(
407                        String theSystemUri,
408                        String theSystemName,
409                        String theCodeSystemVersionId,
410                        TermCodeSystemVersion theCodeSystemVersion,
411                        ResourceTable theCodeSystemResourceTable,
412                        RequestDetails theRequestDetails) {
413                assert TransactionSynchronizationManager.isActualTransactionActive();
414
415                ourLog.debug("Storing code system");
416                Date updated = new Date();
417
418                TermCodeSystemVersion codeSystemToStore = theCodeSystemVersion;
419                ValidateUtil.isTrueOrThrowInvalidRequest(codeSystemToStore.getResource() != null, "No resource supplied");
420                ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied");
421
422                TermCodeSystem codeSystem = getOrCreateDistinctTermCodeSystem(
423                                theSystemUri, theSystemName, theCodeSystemVersionId, theCodeSystemResourceTable);
424
425                List<TermCodeSystemVersion> existing =
426                                myCodeSystemVersionDao.findByCodeSystemResourcePid(theCodeSystemResourceTable.getResourceId());
427                for (TermCodeSystemVersion next : existing) {
428                        if (Objects.equals(next.getCodeSystemVersionId(), theCodeSystemVersionId)
429                                        && myConceptDao.countByCodeSystemVersion(next.getPid()) == 0) {
430
431                                /*
432                                 * If we already have a CodeSystemVersion that matches the version we're storing, we
433                                 * can reuse it. Note that we only reuse if there are no concepts attached to the
434                                 * existing codesystem because we always write a completely fresh set of concepts
435                                 * and mark the old one for deletion. Theoretically we could optimize this by
436                                 * figuring out a delta and only writing that, but that is fairly involved
437                                 * since concepts have parents and children and properties and designations and
438                                 * all that - so it's safer to just always assume changes and write everything
439                                 * fresh. Also, this isn't the kind of thing that's expected to happen often
440                                 * so we aren't particularly performance sensitive here.
441                                 */
442                                next.setCodeSystemDisplayName(theSystemName);
443                                codeSystemToStore = next;
444
445                        } else {
446
447                                /*
448                                 * If we already have a TermCodeSystemVersion that corresponds to the FHIR Resource ID we're
449                                 * adding a version to, we will mark it for deletion. For any one resource there can only
450                                 * be one TermCodeSystemVersion entity in the DB. Multiple versions of a codesystem uses
451                                 * multiple CodeSystem resources with CodeSystem.version set differently (as opposed to
452                                 * multiple versions of the same CodeSystem, where CodeSystem.meta.versionId is different)
453                                 */
454                                next.setCodeSystemVersionId("DELETED_" + UUID.randomUUID());
455                                myCodeSystemVersionDao.saveAndFlush(next);
456                                myDeferredStorageSvc.deleteCodeSystemVersion(next);
457                        }
458                }
459
460                /*
461                 * Do the upload
462                 */
463
464                codeSystemToStore.setCodeSystem(codeSystem);
465                codeSystemToStore.setCodeSystemDisplayName(theSystemName);
466                codeSystemToStore.setCodeSystemVersionId(theCodeSystemVersionId);
467
468                if (codeSystemToStore.getPid() == null) {
469                        myEntityManager.persist(codeSystemToStore);
470                }
471
472                ourLog.debug("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
473
474                // Validate the code system
475                ArrayList<String> conceptsStack = new ArrayList<>();
476                IdentityHashMap<TermConcept, Object> allConcepts = new IdentityHashMap<>();
477                int totalCodeCount = 0;
478                Collection<TermConcept> conceptsToSave = theCodeSystemVersion.getConcepts();
479                for (TermConcept next : conceptsToSave) {
480                        totalCodeCount += validateConceptForStorage(next, codeSystemToStore, conceptsStack, allConcepts);
481                        Validate.isTrue(next.getPid().getId() == null);
482                        Validate.isTrue(codeSystemToStore.getPid() != null);
483
484                        // Make sure to initialize the PK object so that hibernate doesn't choke on creation
485                        next.setId(null);
486
487                        next.setCodeSystemVersion(codeSystemToStore);
488                        next.setUpdated(updated);
489
490                        myEntityManager.persist(next);
491                        for (var property : next.getProperties()) {
492                                assert property.getId() == null;
493                                property.setCodeSystemVersion(codeSystemToStore);
494                                myEntityManager.persist(property);
495                        }
496                        for (var designation : next.getDesignations()) {
497                                assert designation.getId() == null;
498                                designation.setCodeSystemVersion(codeSystemToStore);
499                                myEntityManager.persist(designation);
500                        }
501                }
502
503                ourLog.debug("Saving version containing {} concepts", totalCodeCount);
504                Validate.notNull(codeSystemToStore.getPid(), "Code system not saved");
505                codeSystemToStore = myEntityManager.merge(codeSystemToStore);
506
507                boolean isMakeVersionCurrent = ITermCodeSystemStorageSvc.isMakeVersionCurrent(theRequestDetails);
508                if (isMakeVersionCurrent) {
509                        codeSystem.setCurrentVersion(codeSystemToStore);
510                        if (codeSystem.getPid() == null) {
511                                codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
512                        }
513                }
514
515                ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount);
516                for (TermConcept next : conceptsToSave) {
517                        populateVersion(next, codeSystemToStore);
518                }
519
520                ourLog.debug("Saving {} concepts...", totalCodeCount);
521                IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>();
522                for (TermConcept next : conceptsToSave) {
523                        persistChildren(next, codeSystemToStore, conceptsStack2, totalCodeCount);
524                }
525
526                ourLog.debug("Done saving concepts, flushing to database");
527                if (!myDeferredStorageSvc.isStorageQueueEmpty(true)) {
528                        ourLog.info("Note that some concept saving has been deferred");
529                }
530        }
531
532        private TermCodeSystemVersion getExistingTermCodeSystemVersion(
533                        Long theCodeSystemVersionPid, String theCodeSystemVersion) {
534                TermCodeSystemVersion existing;
535                if (theCodeSystemVersion == null) {
536                        existing = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystemVersionPid);
537                } else {
538                        existing =
539                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystemVersionPid, theCodeSystemVersion);
540                }
541
542                return existing;
543        }
544
545        private void validateDstu3OrNewer() {
546                Validate.isTrue(
547                                myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3),
548                                "Terminology operations only supported in DSTU3+ mode");
549        }
550
551        private void addConceptInHierarchy(
552                        TermCodeSystemVersion theCsv,
553                        Collection<String> theParentCodes,
554                        TermConcept theConceptToAdd,
555                        UploadStatistics theStatisticsTracker,
556                        Map<String, TermConcept> theCodeToConcept,
557                        int theSequence) {
558                TermConcept conceptToAdd = theConceptToAdd;
559                List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren();
560
561                String nextCodeToAdd = conceptToAdd.getCode();
562                String parentDescription = "(root concept)";
563
564                ourLog.info(
565                                "Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription);
566
567                Optional<TermConcept> existingCodeOpt = myConceptDao.findByCodeSystemAndCode(theCsv.getPid(), nextCodeToAdd);
568                List<TermConceptParentChildLink> existingParentLinks;
569                if (existingCodeOpt.isPresent()) {
570                        TermConcept existingCode = existingCodeOpt.get();
571                        existingCode.setIndexStatus(null);
572                        existingCode.setDisplay(conceptToAdd.getDisplay());
573                        conceptToAdd = existingCode;
574                        existingParentLinks = conceptToAdd.getParents();
575                } else {
576                        existingParentLinks = Collections.emptyList();
577                }
578
579                Set<TermConcept> parentConceptsWeShouldLinkTo = new HashSet<>();
580                for (String nextParentCode : theParentCodes) {
581
582                        // Don't add parent links that already exist for the code
583                        if (existingParentLinks.stream()
584                                        .anyMatch(t -> t.getParent().getCode().equals(nextParentCode))) {
585                                continue;
586                        }
587
588                        TermConcept nextParentOpt = theCodeToConcept.get(nextParentCode);
589                        if (nextParentOpt == null) {
590                                nextParentOpt = myConceptDao
591                                                .findByCodeSystemAndCode(theCsv.getPid(), nextParentCode)
592                                                .orElse(null);
593                        }
594                        if (nextParentOpt == null) {
595                                throw new InvalidRequestException(Msg.code(846) + "Unable to add code \"" + nextCodeToAdd
596                                                + "\" to unknown parent: " + nextParentCode);
597                        }
598                        parentConceptsWeShouldLinkTo.add(nextParentOpt);
599                }
600
601                if (conceptToAdd.getSequence() == null) {
602                        conceptToAdd.setSequence(theSequence);
603                }
604
605                // Null out the hierarchy PIDs for this concept always. We do this because we're going to
606                // force a reindex, and it'll be regenerated then
607                conceptToAdd.setParentPids(null);
608                conceptToAdd.setCodeSystemVersion(theCsv);
609
610                if (conceptToAdd.getProperties() != null)
611                        conceptToAdd.getProperties().forEach(termConceptProperty -> {
612                                termConceptProperty.setConcept(theConceptToAdd);
613                                termConceptProperty.setCodeSystemVersion(theCsv);
614                        });
615                if (theStatisticsTracker.getUpdatedConceptCount() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
616                        saveConcept(conceptToAdd);
617                        Long nextConceptPid = conceptToAdd.getId();
618                        Objects.requireNonNull(nextConceptPid);
619                } else {
620                        myDeferredStorageSvc.addConceptToStorageQueue(conceptToAdd);
621                }
622
623                theCodeToConcept.put(conceptToAdd.getCode(), conceptToAdd);
624
625                theStatisticsTracker.incrementUpdatedConceptCount();
626
627                // Add link to new child to the parent
628                for (TermConcept nextParentConcept : parentConceptsWeShouldLinkTo) {
629                        TermConceptParentChildLink parentLink = new TermConceptParentChildLink();
630                        parentLink.setParent(nextParentConcept);
631                        parentLink.setChild(conceptToAdd);
632                        parentLink.setCodeSystem(theCsv);
633                        parentLink.setRelationshipType(TermConceptParentChildLink.RelationshipTypeEnum.ISA);
634                        nextParentConcept.getChildren().add(parentLink);
635                        conceptToAdd.getParents().add(parentLink);
636                        ourLog.info(
637                                        "Saving parent/child link - Parent[{}] Child[{}]",
638                                        parentLink.getParent().getCode(),
639                                        parentLink.getChild().getCode());
640
641                        if (theStatisticsTracker.getUpdatedConceptCount()
642                                        <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
643                                myConceptParentChildLinkDao.save(parentLink);
644                        } else {
645                                myDeferredStorageSvc.addConceptLinkToStorageQueue(parentLink);
646                        }
647                }
648
649                ourLog.trace("About to save parent-child links");
650
651                // Save children recursively
652                int childIndex = 0;
653                for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) {
654
655                        TermConcept nextChild = nextChildConceptLink.getChild();
656
657                        for (int i = 0; i < nextChild.getParents().size(); i++) {
658                                if (nextChild.getParents().get(i).getId() == null) {
659                                        String parentCode =
660                                                        nextChild.getParents().get(i).getParent().getCode();
661                                        TermConcept parentConcept = theCodeToConcept.get(parentCode);
662                                        if (parentConcept == null) {
663                                                parentConcept = myConceptDao
664                                                                .findByCodeSystemAndCode(theCsv.getPid(), parentCode)
665                                                                .orElse(null);
666                                        }
667                                        if (parentConcept == null) {
668                                                throw new IllegalArgumentException(Msg.code(847) + "Unknown parent code: " + parentCode);
669                                        }
670
671                                        nextChild.getParents().get(i).setParent(parentConcept);
672                                }
673                        }
674
675                        Collection<String> parentCodes = nextChild.getParents().stream()
676                                        .map(t -> t.getParent().getCode())
677                                        .collect(Collectors.toList());
678                        addConceptInHierarchy(theCsv, parentCodes, nextChild, theStatisticsTracker, theCodeToConcept, childIndex);
679
680                        childIndex++;
681                }
682        }
683
684        private void persistChildren(
685                        TermConcept theConcept,
686                        TermCodeSystemVersion theCodeSystem,
687                        IdentityHashMap<TermConcept, Object> theConceptsStack,
688                        int theTotalConcepts) {
689                if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) {
690                        return;
691                }
692
693                if ((theConceptsStack.size() + 1) % 10000 == 0) {
694                        float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
695                        ourLog.info(
696                                        "Have processed {}/{} concepts ({}%)",
697                                        theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
698                }
699
700                theConcept.setCodeSystemVersion(theCodeSystem);
701                theConcept.setIndexStatus(EntityIndexStatusEnum.INDEXED_ALL);
702
703                if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
704                        saveConcept(theConcept);
705                } else {
706                        myDeferredStorageSvc.addConceptToStorageQueue(theConcept);
707                }
708
709                for (TermConceptParentChildLink next : theConcept.getChildren()) {
710                        persistChildren(next.getChild(), theCodeSystem, theConceptsStack, theTotalConcepts);
711                }
712
713                for (TermConceptParentChildLink next : theConcept.getChildren()) {
714                        if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) {
715                                saveConceptLink(next);
716                        } else {
717                                myDeferredStorageSvc.addConceptLinkToStorageQueue(next);
718                        }
719                }
720        }
721
722        private void populateVersion(TermConcept theNext, TermCodeSystemVersion theCodeSystemVersion) {
723                theNext.setCodeSystemVersion(theCodeSystemVersion);
724                for (TermConceptParentChildLink next : theNext.getChildren()) {
725                        populateVersion(next.getChild(), theCodeSystemVersion);
726                }
727                theNext.getProperties().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
728                theNext.getDesignations().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion));
729        }
730
731        private void saveConceptLink(TermConceptParentChildLink next) {
732                if (next.getId() == null) {
733                        myConceptParentChildLinkDao.save(next);
734                }
735        }
736
737        @Nonnull
738        private TermCodeSystem getOrCreateDistinctTermCodeSystem(
739                        String theSystemUri,
740                        String theSystemName,
741                        String theSystemVersionId,
742                        ResourceTable theCodeSystemResourceTable) {
743                TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(theSystemUri);
744                if (codeSystem == null) {
745                        codeSystem = myCodeSystemDao.findByResourcePid((theCodeSystemResourceTable.getId()));
746                        if (codeSystem == null) {
747                                codeSystem = new TermCodeSystem();
748                        }
749                } else {
750                        checkForCodeSystemVersionDuplicate(
751                                        codeSystem, theSystemUri, theSystemVersionId, theCodeSystemResourceTable);
752                }
753
754                codeSystem.setResource(theCodeSystemResourceTable);
755                codeSystem.setCodeSystemUri(theSystemUri);
756                codeSystem.setName(theSystemName);
757                codeSystem = myCodeSystemDao.save(codeSystem);
758                return codeSystem;
759        }
760
761        private void checkForCodeSystemVersionDuplicate(
762                        TermCodeSystem theCodeSystem,
763                        String theSystemUri,
764                        String theSystemVersionId,
765                        ResourceTable theCodeSystemResourceTable) {
766                TermCodeSystemVersion codeSystemVersionEntity;
767                String msg = null;
768                if (theSystemVersionId == null) {
769                        // Check if a non-versioned TermCodeSystemVersion entity already exists for this TermCodeSystem.
770                        codeSystemVersionEntity = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystem.getPid());
771                        if (codeSystemVersionEntity != null) {
772                                msg = myContext
773                                                .getLocalizer()
774                                                .getMessage(
775                                                                TermReadSvcImpl.class,
776                                                                "cannotCreateDuplicateCodeSystemUrl",
777                                                                theSystemUri,
778                                                                codeSystemVersionEntity
779                                                                                .getResource()
780                                                                                .getIdDt()
781                                                                                .toUnqualifiedVersionless()
782                                                                                .getValue());
783                        }
784                } else {
785                        // Check if a TermCodeSystemVersion entity already exists for this TermCodeSystem and version.
786                        codeSystemVersionEntity =
787                                        myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystem.getPid(), theSystemVersionId);
788                        if (codeSystemVersionEntity != null) {
789                                msg = myContext
790                                                .getLocalizer()
791                                                .getMessage(
792                                                                TermReadSvcImpl.class,
793                                                                "cannotCreateDuplicateCodeSystemUrlAndVersion",
794                                                                theSystemUri,
795                                                                theSystemVersionId,
796                                                                codeSystemVersionEntity
797                                                                                .getResource()
798                                                                                .getIdDt()
799                                                                                .toUnqualifiedVersionless()
800                                                                                .getValue());
801                        }
802                }
803                // Throw exception if the TermCodeSystemVersion is being duplicated.
804                if (codeSystemVersionEntity != null) {
805                        if (!ObjectUtil.equals(codeSystemVersionEntity.getResource().getId(), theCodeSystemResourceTable.getId())) {
806                                throw new UnprocessableEntityException(Msg.code(848) + msg);
807                        }
808                }
809        }
810
811        private void populateCodeSystemVersionProperties(
812                        TermCodeSystemVersion theCodeSystemVersion,
813                        CodeSystem theCodeSystemResource,
814                        ResourceTable theResourceTable) {
815                theCodeSystemVersion.setResource(theResourceTable);
816                theCodeSystemVersion.setCodeSystemDisplayName(theCodeSystemResource.getName());
817                theCodeSystemVersion.setCodeSystemVersionId(theCodeSystemResource.getVersion());
818        }
819
820        private int validateConceptForStorage(
821                        TermConcept theConcept,
822                        TermCodeSystemVersion theCodeSystemVersion,
823                        ArrayList<String> theConceptsStack,
824                        IdentityHashMap<TermConcept, Object> theAllConcepts) {
825                ValidateUtil.isTrueOrThrowInvalidRequest(
826                                theConcept.getCodeSystemVersion() != null, "CodeSystemVersion is null");
827                ValidateUtil.isNotBlankOrThrowInvalidRequest(
828                                theConcept.getCode(), "CodeSystem contains a code with no code value");
829
830                theConcept.setCodeSystemVersion(theCodeSystemVersion);
831                if (theConceptsStack.contains(theConcept.getCode())) {
832                        throw new InvalidRequestException(
833                                        Msg.code(849) + "CodeSystem contains circular reference around code " + theConcept.getCode());
834                }
835                theConceptsStack.add(theConcept.getCode());
836
837                int retVal = 0;
838                if (theAllConcepts.put(theConcept, theAllConcepts) == null) {
839                        if (theAllConcepts.size() % 1000 == 0) {
840                                ourLog.info("Have validated {} concepts", theAllConcepts.size());
841                        }
842                        retVal = 1;
843                }
844
845                for (TermConceptParentChildLink next : theConcept.getChildren()) {
846                        next.setCodeSystem(theCodeSystemVersion);
847                        retVal +=
848                                        validateConceptForStorage(next.getChild(), theCodeSystemVersion, theConceptsStack, theAllConcepts);
849                }
850
851                theConceptsStack.remove(theConceptsStack.size() - 1);
852
853                return retVal;
854        }
855}