001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.term; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.context.FhirVersionEnum; 024import ca.uhn.fhir.i18n.Msg; 025import ca.uhn.fhir.interceptor.model.RequestPartitionId; 026import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 027import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 028import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; 029import ca.uhn.fhir.jpa.dao.data.IResourceTableDao; 030import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; 031import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; 032import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; 033import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao; 034import ca.uhn.fhir.jpa.dao.data.ITermConceptParentChildLinkDao; 035import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao; 036import ca.uhn.fhir.jpa.entity.TermCodeSystem; 037import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; 038import ca.uhn.fhir.jpa.entity.TermConcept; 039import ca.uhn.fhir.jpa.entity.TermConceptDesignation; 040import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; 041import ca.uhn.fhir.jpa.entity.TermConceptProperty; 042import ca.uhn.fhir.jpa.model.dao.JpaPid; 043import ca.uhn.fhir.jpa.model.entity.ResourceTable; 044import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc; 045import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; 046import ca.uhn.fhir.jpa.term.api.ITermReadSvc; 047import ca.uhn.fhir.jpa.term.api.ITermVersionAdapterSvc; 048import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet; 049import ca.uhn.fhir.rest.api.Constants; 050import ca.uhn.fhir.rest.api.server.RequestDetails; 051import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; 052import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 053import ca.uhn.fhir.rest.server.exceptions.PreconditionFailedException; 054import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; 055import ca.uhn.fhir.util.ObjectUtil; 056import ca.uhn.fhir.util.UrlUtil; 057import ca.uhn.fhir.util.ValidateUtil; 058import jakarta.annotation.Nonnull; 059import jakarta.persistence.EntityManager; 060import jakarta.persistence.PersistenceContext; 061import jakarta.persistence.PersistenceContextType; 062import org.apache.commons.lang3.Validate; 063import org.hl7.fhir.instance.model.api.IIdType; 064import org.hl7.fhir.r4.model.CodeSystem; 065import org.hl7.fhir.r4.model.ConceptMap; 066import org.hl7.fhir.r4.model.ValueSet; 067import org.slf4j.Logger; 068import org.slf4j.LoggerFactory; 069import org.springframework.beans.factory.annotation.Autowired; 070import org.springframework.transaction.annotation.Propagation; 071import org.springframework.transaction.annotation.Transactional; 072import org.springframework.transaction.support.TransactionSynchronizationManager; 073 074import java.util.ArrayList; 075import java.util.Arrays; 076import java.util.Collection; 077import java.util.Collections; 078import java.util.HashMap; 079import java.util.HashSet; 080import java.util.IdentityHashMap; 081import java.util.List; 082import java.util.Map; 083import java.util.Objects; 084import java.util.Optional; 085import java.util.Set; 086import java.util.UUID; 087import java.util.concurrent.atomic.AtomicInteger; 088import java.util.stream.Collectors; 089 090import static ca.uhn.fhir.jpa.api.dao.IDao.RESOURCE_PID_KEY; 091import static org.apache.commons.lang3.StringUtils.isBlank; 092import static org.apache.commons.lang3.StringUtils.isNotBlank; 093import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW; 094 095public class TermCodeSystemStorageSvcImpl implements ITermCodeSystemStorageSvc { 096 private static final Logger ourLog = LoggerFactory.getLogger(TermCodeSystemStorageSvcImpl.class); 097 private static final Object PLACEHOLDER_OBJECT = new Object(); 098 099 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 100 protected EntityManager myEntityManager; 101 102 @Autowired 103 protected ITermCodeSystemDao myCodeSystemDao; 104 105 @Autowired 106 protected ITermCodeSystemVersionDao myCodeSystemVersionDao; 107 108 @Autowired 109 protected ITermConceptDao myConceptDao; 110 111 @Autowired 112 protected ITermConceptPropertyDao myConceptPropertyDao; 113 114 @Autowired 115 protected ITermConceptDesignationDao myConceptDesignationDao; 116 117 @Autowired 118 protected IIdHelperService<JpaPid> myIdHelperService; 119 120 @Autowired 121 private ITermConceptParentChildLinkDao myConceptParentChildLinkDao; 122 123 @Autowired 124 private ITermVersionAdapterSvc myTerminologyVersionAdapterSvc; 125 126 @Autowired 127 private ITermDeferredStorageSvc myDeferredStorageSvc; 128 129 @Autowired 130 private FhirContext myContext; 131 132 @Autowired 133 private ITermReadSvc myTerminologySvc; 134 135 @Autowired 136 private JpaStorageSettings myStorageSettings; 137 138 @Autowired 139 private IResourceTableDao myResourceTableDao; 140 141 @Autowired 142 private TermConceptDaoSvc myTermConceptDaoSvc; 143 144 @Transactional 145 @Override 146 public UploadStatistics applyDeltaCodeSystemsAdd(String theSystem, CustomTerminologySet theAdditions) { 147 ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided"); 148 validateDstu3OrNewer(); 149 theAdditions.validateNoCycleOrThrowInvalidRequest(); 150 151 TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem); 152 if (cs == null) { 153 CodeSystem codeSystemResource = new CodeSystem(); 154 codeSystemResource.setUrl(theSystem); 155 codeSystemResource.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT); 156 if (isBlank(codeSystemResource.getIdElement().getIdPart()) && theSystem.contains(LOINC_LOW)) { 157 codeSystemResource.setId(LOINC_LOW); 158 } 159 myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(codeSystemResource); 160 161 cs = myCodeSystemDao.findByCodeSystemUri(theSystem); 162 } 163 164 TermCodeSystemVersion csv = cs.getCurrentVersion(); 165 Validate.notNull(csv); 166 167 CodeSystem codeSystem = myTerminologySvc.fetchCanonicalCodeSystemFromCompleteContext(theSystem); 168 if (codeSystem.getContent() != CodeSystem.CodeSystemContentMode.NOTPRESENT) { 169 throw new InvalidRequestException( 170 Msg.code(844) + "CodeSystem with url[" + Constants.codeSystemWithDefaultDescription(theSystem) 171 + "] can not apply a delta - wrong content mode: " + codeSystem.getContent()); 172 } 173 174 Validate.notNull(cs); 175 Validate.notNull(cs.getPid()); 176 177 IIdType codeSystemId = cs.getResource().getIdDt(); 178 179 UploadStatistics retVal = new UploadStatistics(codeSystemId); 180 HashMap<String, TermConcept> codeToConcept = new HashMap<>(); 181 182 // Add root concepts 183 for (TermConcept nextRootConcept : theAdditions.getRootConcepts()) { 184 List<String> parentCodes = Collections.emptyList(); 185 addConceptInHierarchy(csv, parentCodes, nextRootConcept, retVal, codeToConcept, 0); 186 } 187 188 return retVal; 189 } 190 191 @Transactional 192 @Override 193 public UploadStatistics applyDeltaCodeSystemsRemove(String theSystem, CustomTerminologySet theValue) { 194 ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystem, "No system provided"); 195 validateDstu3OrNewer(); 196 197 TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(theSystem); 198 if (cs == null) { 199 throw new InvalidRequestException(Msg.code(845) + "Unknown code system: " + theSystem); 200 } 201 IIdType target = cs.getResource().getIdDt(); 202 203 AtomicInteger removeCounter = new AtomicInteger(0); 204 205 // We need to delete all termconcepts, and their children. This stream flattens the TermConcepts and their 206 // children into a single set of TermConcept objects retrieved from the DB. Note that we have to do this because 207 // deleteById() in JPA doesnt appear to actually commit or flush a transaction until way later, and we end up 208 // iterating multiple times over the same elements, which screws up our counter. 209 210 // Grab the actual entities 211 List<TermConcept> collect = theValue.getRootConcepts().stream() 212 .map(val -> myTerminologySvc.findCode(theSystem, val.getCode())) 213 .filter(Optional::isPresent) 214 .map(Optional::get) 215 .collect(Collectors.toList()); 216 217 // Iterate over the actual entities and fill out their children 218 Set<TermConcept> allFoundTermConcepts = collect.stream() 219 .flatMap(concept -> flattenChildren(concept).stream()) 220 .map(suppliedTermConcept -> myTerminologySvc.findCode(theSystem, suppliedTermConcept.getCode())) 221 .filter(Optional::isPresent) 222 .map(Optional::get) 223 .collect(Collectors.toSet()); 224 225 // Delete everything about these codes. 226 for (TermConcept code : allFoundTermConcepts) { 227 deleteEverythingRelatedToConcept(code, removeCounter); 228 } 229 230 return new UploadStatistics(removeCounter.get(), target); 231 } 232 233 private void deleteEverythingRelatedToConcept(TermConcept theConcept, AtomicInteger theRemoveCounter) { 234 235 for (TermConceptParentChildLink nextParent : theConcept.getParents()) { 236 nextParent.getParent().getChildren().remove(nextParent); 237 myConceptParentChildLinkDao.deleteById(nextParent.getId()); 238 } 239 for (TermConceptParentChildLink nextChild : theConcept.getChildren()) { 240 nextChild.getChild().getParents().remove(nextChild); 241 myConceptParentChildLinkDao.deleteById(nextChild.getId()); 242 } 243 244 for (TermConceptDesignation next : theConcept.getDesignations()) { 245 myConceptDesignationDao.deleteById(next.getPid()); 246 } 247 theConcept.getDesignations().clear(); 248 for (TermConceptProperty next : theConcept.getProperties()) { 249 myConceptPropertyDao.deleteById(next.getPid()); 250 } 251 theConcept.getProperties().clear(); 252 253 ourLog.info("Deleting concept {} - Code {}", theConcept.getId(), theConcept.getCode()); 254 255 myConceptDao.deleteById(theConcept.getId()); 256 // myEntityManager.remove(theConcept); 257 258 theRemoveCounter.incrementAndGet(); 259 } 260 261 private List<TermConcept> flattenChildren(TermConcept theTermConcept) { 262 if (theTermConcept.getChildren().isEmpty()) { 263 return Arrays.asList(theTermConcept); 264 } 265 266 // Recursively flatten children 267 List<TermConcept> childTermConcepts = theTermConcept.getChildren().stream() 268 .map(TermConceptParentChildLink::getChild) 269 .flatMap(childConcept -> flattenChildren(childConcept).stream()) 270 .collect(Collectors.toList()); 271 272 // Add itself before its list of children 273 childTermConcepts.add(0, theTermConcept); 274 return childTermConcepts; 275 } 276 277 /** 278 * Returns the number of saved concepts 279 */ 280 @Override 281 public int saveConcept(TermConcept theConcept) { 282 return myTermConceptDaoSvc.saveConcept(theConcept); 283 } 284 285 @Override 286 @Transactional(propagation = Propagation.MANDATORY) 287 public void storeNewCodeSystemVersionIfNeeded( 288 CodeSystem theCodeSystem, ResourceTable theResourceEntity, RequestDetails theRequestDetails) { 289 if (theCodeSystem != null && isNotBlank(theCodeSystem.getUrl())) { 290 String codeSystemUrl = theCodeSystem.getUrl(); 291 if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.COMPLETE 292 || theCodeSystem.getContent() == null 293 || theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) { 294 ourLog.info( 295 "CodeSystem {} has a status of {}, going to store concepts in terminology tables", 296 theResourceEntity.getIdDt().getValue(), 297 theCodeSystem.getContentElement().getValueAsString()); 298 299 detectDuplicatesInCodeSystem(theCodeSystem); 300 301 Long pid = (Long) theCodeSystem.getUserData(RESOURCE_PID_KEY); 302 assert pid != null; 303 JpaPid codeSystemResourcePid = JpaPid.fromId(pid); 304 305 /* 306 * If this is a not-present codesystem and codesystem version already exists, we don't want to 307 * overwrite the existing version since that will wipe out the existing concepts. We do create 308 * or update the TermCodeSystem table though, since that allows the DB to reject changes that would 309 * result in duplicate CodeSystem.url values. 310 */ 311 if (theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT) { 312 TermCodeSystem termCodeSystem = myCodeSystemDao.findByCodeSystemUri(theCodeSystem.getUrl()); 313 if (termCodeSystem != null) { 314 TermCodeSystemVersion codeSystemVersion = 315 getExistingTermCodeSystemVersion(termCodeSystem.getPid(), theCodeSystem.getVersion()); 316 if (codeSystemVersion != null) { 317 TermCodeSystem myCodeSystemEntity = getOrCreateDistinctTermCodeSystem( 318 codeSystemResourcePid, 319 theCodeSystem.getUrl(), 320 theCodeSystem.getUrl(), 321 theCodeSystem.getVersion(), 322 theResourceEntity); 323 return; 324 } 325 } 326 } 327 328 TermCodeSystemVersion persCs = new TermCodeSystemVersion(); 329 330 populateCodeSystemVersionProperties(persCs, theCodeSystem, theResourceEntity); 331 332 persCs.getConcepts().addAll(TermReadSvcImpl.toPersistedConcepts(theCodeSystem.getConcept(), persCs)); 333 ourLog.debug("Code system has {} concepts", persCs.getConcepts().size()); 334 storeNewCodeSystemVersion( 335 codeSystemResourcePid, 336 codeSystemUrl, 337 theCodeSystem.getName(), 338 theCodeSystem.getVersion(), 339 persCs, 340 theResourceEntity, 341 theRequestDetails); 342 } 343 } 344 } 345 346 private static void detectDuplicatesInCodeSystem(CodeSystem theCodeSystem) { 347 detectDuplicatesInCodeSystem(theCodeSystem.getConcept(), new HashSet<>()); 348 } 349 350 private static void detectDuplicatesInCodeSystem( 351 List<CodeSystem.ConceptDefinitionComponent> theCodeList, Set<String> theFoundCodesBuffer) { 352 for (var next : theCodeList) { 353 if (isNotBlank(next.getCode())) { 354 if (!theFoundCodesBuffer.add(next.getCode())) { 355 /* 356 * Note: We could possibly modify this behaviour to be forgiving, and just 357 * ignore duplicates. The only issue is that concepts can have properties, 358 * designations, etc. and it could be dangerous to just pick one and ignore the 359 * other. So the safer thing seems to be to just throw an error. 360 */ 361 throw new PreconditionFailedException(Msg.code(2528) + "Duplicate concept detected in CodeSystem: " 362 + UrlUtil.sanitizeUrlPart(next.getCode())); 363 } 364 } 365 // Test child concepts within the parent concept 366 detectDuplicatesInCodeSystem(next.getConcept(), theFoundCodesBuffer); 367 } 368 } 369 370 @Override 371 @Transactional 372 public IIdType storeNewCodeSystemVersion( 373 CodeSystem theCodeSystemResource, 374 TermCodeSystemVersion theCodeSystemVersion, 375 RequestDetails theRequest, 376 List<ValueSet> theValueSets, 377 List<ConceptMap> theConceptMaps) { 378 assert TransactionSynchronizationManager.isActualTransactionActive(); 379 380 Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL"); 381 382 // Note that this creates the TermCodeSystem and TermCodeSystemVersion entities if needed 383 IIdType csId = myTerminologyVersionAdapterSvc.createOrUpdateCodeSystem(theCodeSystemResource, theRequest); 384 385 JpaPid codeSystemResourcePid = myIdHelperService.resolveResourcePersistentIds( 386 RequestPartitionId.allPartitions(), csId.getResourceType(), csId.getIdPart()); 387 ResourceTable resource = myResourceTableDao.getOne(codeSystemResourcePid.getId()); 388 389 ourLog.info("CodeSystem resource has ID: {}", csId.getValue()); 390 391 populateCodeSystemVersionProperties(theCodeSystemVersion, theCodeSystemResource, resource); 392 393 storeNewCodeSystemVersion( 394 codeSystemResourcePid, 395 theCodeSystemResource.getUrl(), 396 theCodeSystemResource.getName(), 397 theCodeSystemResource.getVersion(), 398 theCodeSystemVersion, 399 resource, 400 theRequest); 401 402 myDeferredStorageSvc.addConceptMapsToStorageQueue(theConceptMaps); 403 myDeferredStorageSvc.addValueSetsToStorageQueue(theValueSets); 404 405 return csId; 406 } 407 408 @Override 409 @Transactional 410 public void storeNewCodeSystemVersion( 411 IResourcePersistentId theCodeSystemResourcePid, 412 String theSystemUri, 413 String theSystemName, 414 String theCodeSystemVersionId, 415 TermCodeSystemVersion theCodeSystemVersion, 416 ResourceTable theCodeSystemResourceTable, 417 RequestDetails theRequestDetails) { 418 assert TransactionSynchronizationManager.isActualTransactionActive(); 419 420 ourLog.debug("Storing code system"); 421 422 TermCodeSystemVersion codeSystemToStore = theCodeSystemVersion; 423 ValidateUtil.isTrueOrThrowInvalidRequest(codeSystemToStore.getResource() != null, "No resource supplied"); 424 ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied"); 425 426 TermCodeSystem codeSystem = getOrCreateDistinctTermCodeSystem( 427 theCodeSystemResourcePid, 428 theSystemUri, 429 theSystemName, 430 theCodeSystemVersionId, 431 theCodeSystemResourceTable); 432 433 List<TermCodeSystemVersion> existing = 434 myCodeSystemVersionDao.findByCodeSystemResourcePid(((JpaPid) theCodeSystemResourcePid).getId()); 435 for (TermCodeSystemVersion next : existing) { 436 if (Objects.equals(next.getCodeSystemVersionId(), theCodeSystemVersionId) 437 && myConceptDao.countByCodeSystemVersion(next.getPid()) == 0) { 438 439 /* 440 * If we already have a CodeSystemVersion that matches the version we're storing, we 441 * can reuse it. 442 */ 443 next.setCodeSystemDisplayName(theSystemName); 444 codeSystemToStore = next; 445 446 } else { 447 448 /* 449 * If we already have a TermCodeSystemVersion that corresponds to the FHIR Resource ID we're 450 * adding a version to, we will mark it for deletion. For any one resource there can only 451 * be one TermCodeSystemVersion entity in the DB. Multiple versions of a codesystem uses 452 * multiple CodeSystem resources with CodeSystem.version set differently (as opposed to 453 * multiple versions of the same CodeSystem, where CodeSystem.meta.versionId is different) 454 */ 455 next.setCodeSystemVersionId("DELETED_" + UUID.randomUUID().toString()); 456 myCodeSystemVersionDao.saveAndFlush(next); 457 myDeferredStorageSvc.deleteCodeSystemVersion(next); 458 } 459 } 460 461 /* 462 * Do the upload 463 */ 464 465 codeSystemToStore.setCodeSystem(codeSystem); 466 codeSystemToStore.setCodeSystemDisplayName(theSystemName); 467 codeSystemToStore.setCodeSystemVersionId(theCodeSystemVersionId); 468 469 ourLog.debug("Validating all codes in CodeSystem for storage (this can take some time for large sets)"); 470 471 // Validate the code system 472 ArrayList<String> conceptsStack = new ArrayList<>(); 473 IdentityHashMap<TermConcept, Object> allConcepts = new IdentityHashMap<>(); 474 int totalCodeCount = 0; 475 Collection<TermConcept> conceptsToSave = theCodeSystemVersion.getConcepts(); 476 for (TermConcept next : conceptsToSave) { 477 totalCodeCount += validateConceptForStorage(next, codeSystemToStore, conceptsStack, allConcepts); 478 } 479 480 ourLog.debug("Saving version containing {} concepts", totalCodeCount); 481 if (codeSystemToStore.getPid() == null) { 482 codeSystemToStore = myCodeSystemVersionDao.saveAndFlush(codeSystemToStore); 483 } 484 485 boolean isMakeVersionCurrent = ITermCodeSystemStorageSvc.isMakeVersionCurrent(theRequestDetails); 486 if (isMakeVersionCurrent) { 487 codeSystem.setCurrentVersion(codeSystemToStore); 488 if (codeSystem.getPid() == null) { 489 codeSystem = myCodeSystemDao.saveAndFlush(codeSystem); 490 } 491 } 492 493 ourLog.debug("Setting CodeSystemVersion[{}] on {} concepts...", codeSystem.getPid(), totalCodeCount); 494 for (TermConcept next : conceptsToSave) { 495 populateVersion(next, codeSystemToStore); 496 } 497 498 ourLog.debug("Saving {} concepts...", totalCodeCount); 499 IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<>(); 500 for (TermConcept next : conceptsToSave) { 501 persistChildren(next, codeSystemToStore, conceptsStack2, totalCodeCount); 502 } 503 504 ourLog.debug("Done saving concepts, flushing to database"); 505 if (!myDeferredStorageSvc.isStorageQueueEmpty(true)) { 506 ourLog.info("Note that some concept saving has been deferred"); 507 } 508 } 509 510 private TermCodeSystemVersion getExistingTermCodeSystemVersion( 511 Long theCodeSystemVersionPid, String theCodeSystemVersion) { 512 TermCodeSystemVersion existing; 513 if (theCodeSystemVersion == null) { 514 existing = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystemVersionPid); 515 } else { 516 existing = 517 myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystemVersionPid, theCodeSystemVersion); 518 } 519 520 return existing; 521 } 522 523 private void validateDstu3OrNewer() { 524 Validate.isTrue( 525 myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3), 526 "Terminology operations only supported in DSTU3+ mode"); 527 } 528 529 private void addConceptInHierarchy( 530 TermCodeSystemVersion theCsv, 531 Collection<String> theParentCodes, 532 TermConcept theConceptToAdd, 533 UploadStatistics theStatisticsTracker, 534 Map<String, TermConcept> theCodeToConcept, 535 int theSequence) { 536 TermConcept conceptToAdd = theConceptToAdd; 537 List<TermConceptParentChildLink> childrenToAdd = theConceptToAdd.getChildren(); 538 539 String nextCodeToAdd = conceptToAdd.getCode(); 540 String parentDescription = "(root concept)"; 541 542 ourLog.info( 543 "Saving concept {} with parent {}", theStatisticsTracker.getUpdatedConceptCount(), parentDescription); 544 545 Optional<TermConcept> existingCodeOpt = myConceptDao.findByCodeSystemAndCode(theCsv.getPid(), nextCodeToAdd); 546 List<TermConceptParentChildLink> existingParentLinks; 547 if (existingCodeOpt.isPresent()) { 548 TermConcept existingCode = existingCodeOpt.get(); 549 existingCode.setIndexStatus(null); 550 existingCode.setDisplay(conceptToAdd.getDisplay()); 551 conceptToAdd = existingCode; 552 existingParentLinks = conceptToAdd.getParents(); 553 } else { 554 existingParentLinks = Collections.emptyList(); 555 } 556 557 Set<TermConcept> parentConceptsWeShouldLinkTo = new HashSet<>(); 558 for (String nextParentCode : theParentCodes) { 559 560 // Don't add parent links that already exist for the code 561 if (existingParentLinks.stream() 562 .anyMatch(t -> t.getParent().getCode().equals(nextParentCode))) { 563 continue; 564 } 565 566 TermConcept nextParentOpt = theCodeToConcept.get(nextParentCode); 567 if (nextParentOpt == null) { 568 nextParentOpt = myConceptDao 569 .findByCodeSystemAndCode(theCsv.getPid(), nextParentCode) 570 .orElse(null); 571 } 572 if (nextParentOpt == null) { 573 throw new InvalidRequestException(Msg.code(846) + "Unable to add code \"" + nextCodeToAdd 574 + "\" to unknown parent: " + nextParentCode); 575 } 576 parentConceptsWeShouldLinkTo.add(nextParentOpt); 577 } 578 579 if (conceptToAdd.getSequence() == null) { 580 conceptToAdd.setSequence(theSequence); 581 } 582 583 // Null out the hierarchy PIDs for this concept always. We do this because we're going to 584 // force a reindex, and it'll be regenerated then 585 conceptToAdd.setParentPids(null); 586 conceptToAdd.setCodeSystemVersion(theCsv); 587 588 if (conceptToAdd.getProperties() != null) 589 conceptToAdd.getProperties().forEach(termConceptProperty -> { 590 termConceptProperty.setConcept(theConceptToAdd); 591 termConceptProperty.setCodeSystemVersion(theCsv); 592 }); 593 if (theStatisticsTracker.getUpdatedConceptCount() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) { 594 saveConcept(conceptToAdd); 595 Long nextConceptPid = conceptToAdd.getId(); 596 Validate.notNull(nextConceptPid); 597 } else { 598 myDeferredStorageSvc.addConceptToStorageQueue(conceptToAdd); 599 } 600 601 theCodeToConcept.put(conceptToAdd.getCode(), conceptToAdd); 602 603 theStatisticsTracker.incrementUpdatedConceptCount(); 604 605 // Add link to new child to the parent 606 for (TermConcept nextParentConcept : parentConceptsWeShouldLinkTo) { 607 TermConceptParentChildLink parentLink = new TermConceptParentChildLink(); 608 parentLink.setParent(nextParentConcept); 609 parentLink.setChild(conceptToAdd); 610 parentLink.setCodeSystem(theCsv); 611 parentLink.setRelationshipType(TermConceptParentChildLink.RelationshipTypeEnum.ISA); 612 nextParentConcept.getChildren().add(parentLink); 613 conceptToAdd.getParents().add(parentLink); 614 ourLog.info( 615 "Saving parent/child link - Parent[{}] Child[{}]", 616 parentLink.getParent().getCode(), 617 parentLink.getChild().getCode()); 618 619 if (theStatisticsTracker.getUpdatedConceptCount() 620 <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) { 621 myConceptParentChildLinkDao.save(parentLink); 622 } else { 623 myDeferredStorageSvc.addConceptLinkToStorageQueue(parentLink); 624 } 625 } 626 627 ourLog.trace("About to save parent-child links"); 628 629 // Save children recursively 630 int childIndex = 0; 631 for (TermConceptParentChildLink nextChildConceptLink : new ArrayList<>(childrenToAdd)) { 632 633 TermConcept nextChild = nextChildConceptLink.getChild(); 634 635 for (int i = 0; i < nextChild.getParents().size(); i++) { 636 if (nextChild.getParents().get(i).getId() == null) { 637 String parentCode = 638 nextChild.getParents().get(i).getParent().getCode(); 639 TermConcept parentConcept = theCodeToConcept.get(parentCode); 640 if (parentConcept == null) { 641 parentConcept = myConceptDao 642 .findByCodeSystemAndCode(theCsv.getPid(), parentCode) 643 .orElse(null); 644 } 645 if (parentConcept == null) { 646 throw new IllegalArgumentException(Msg.code(847) + "Unknown parent code: " + parentCode); 647 } 648 649 nextChild.getParents().get(i).setParent(parentConcept); 650 } 651 } 652 653 Collection<String> parentCodes = nextChild.getParents().stream() 654 .map(t -> t.getParent().getCode()) 655 .collect(Collectors.toList()); 656 addConceptInHierarchy(theCsv, parentCodes, nextChild, theStatisticsTracker, theCodeToConcept, childIndex); 657 658 childIndex++; 659 } 660 } 661 662 private void persistChildren( 663 TermConcept theConcept, 664 TermCodeSystemVersion theCodeSystem, 665 IdentityHashMap<TermConcept, Object> theConceptsStack, 666 int theTotalConcepts) { 667 if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) { 668 return; 669 } 670 671 if ((theConceptsStack.size() + 1) % 10000 == 0) { 672 float pct = (float) theConceptsStack.size() / (float) theTotalConcepts; 673 ourLog.info( 674 "Have processed {}/{} concepts ({}%)", 675 theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f)); 676 } 677 678 theConcept.setCodeSystemVersion(theCodeSystem); 679 theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED); 680 681 if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) { 682 saveConcept(theConcept); 683 } else { 684 myDeferredStorageSvc.addConceptToStorageQueue(theConcept); 685 } 686 687 for (TermConceptParentChildLink next : theConcept.getChildren()) { 688 persistChildren(next.getChild(), theCodeSystem, theConceptsStack, theTotalConcepts); 689 } 690 691 for (TermConceptParentChildLink next : theConcept.getChildren()) { 692 if (theConceptsStack.size() <= myStorageSettings.getDeferIndexingForCodesystemsOfSize()) { 693 saveConceptLink(next); 694 } else { 695 myDeferredStorageSvc.addConceptLinkToStorageQueue(next); 696 } 697 } 698 } 699 700 private void populateVersion(TermConcept theNext, TermCodeSystemVersion theCodeSystemVersion) { 701 theNext.setCodeSystemVersion(theCodeSystemVersion); 702 for (TermConceptParentChildLink next : theNext.getChildren()) { 703 populateVersion(next.getChild(), theCodeSystemVersion); 704 } 705 theNext.getProperties().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion)); 706 theNext.getDesignations().forEach(t -> t.setCodeSystemVersion(theCodeSystemVersion)); 707 } 708 709 private void saveConceptLink(TermConceptParentChildLink next) { 710 if (next.getId() == null) { 711 myConceptParentChildLinkDao.save(next); 712 } 713 } 714 715 @Nonnull 716 private TermCodeSystem getOrCreateDistinctTermCodeSystem( 717 IResourcePersistentId theCodeSystemResourcePid, 718 String theSystemUri, 719 String theSystemName, 720 String theSystemVersionId, 721 ResourceTable theCodeSystemResourceTable) { 722 TermCodeSystem codeSystem = myCodeSystemDao.findByCodeSystemUri(theSystemUri); 723 if (codeSystem == null) { 724 codeSystem = myCodeSystemDao.findByResourcePid(((JpaPid) theCodeSystemResourcePid).getId()); 725 if (codeSystem == null) { 726 codeSystem = new TermCodeSystem(); 727 } 728 } else { 729 checkForCodeSystemVersionDuplicate( 730 codeSystem, theSystemUri, theSystemVersionId, theCodeSystemResourceTable); 731 } 732 733 codeSystem.setResource(theCodeSystemResourceTable); 734 codeSystem.setCodeSystemUri(theSystemUri); 735 codeSystem.setName(theSystemName); 736 codeSystem = myCodeSystemDao.save(codeSystem); 737 return codeSystem; 738 } 739 740 private void checkForCodeSystemVersionDuplicate( 741 TermCodeSystem theCodeSystem, 742 String theSystemUri, 743 String theSystemVersionId, 744 ResourceTable theCodeSystemResourceTable) { 745 TermCodeSystemVersion codeSystemVersionEntity; 746 String msg = null; 747 if (theSystemVersionId == null) { 748 // Check if a non-versioned TermCodeSystemVersion entity already exists for this TermCodeSystem. 749 codeSystemVersionEntity = myCodeSystemVersionDao.findByCodeSystemPidVersionIsNull(theCodeSystem.getPid()); 750 if (codeSystemVersionEntity != null) { 751 msg = myContext 752 .getLocalizer() 753 .getMessage( 754 TermReadSvcImpl.class, 755 "cannotCreateDuplicateCodeSystemUrl", 756 theSystemUri, 757 codeSystemVersionEntity 758 .getResource() 759 .getIdDt() 760 .toUnqualifiedVersionless() 761 .getValue()); 762 } 763 } else { 764 // Check if a TermCodeSystemVersion entity already exists for this TermCodeSystem and version. 765 codeSystemVersionEntity = 766 myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCodeSystem.getPid(), theSystemVersionId); 767 if (codeSystemVersionEntity != null) { 768 msg = myContext 769 .getLocalizer() 770 .getMessage( 771 TermReadSvcImpl.class, 772 "cannotCreateDuplicateCodeSystemUrlAndVersion", 773 theSystemUri, 774 theSystemVersionId, 775 codeSystemVersionEntity 776 .getResource() 777 .getIdDt() 778 .toUnqualifiedVersionless() 779 .getValue()); 780 } 781 } 782 // Throw exception if the TermCodeSystemVersion is being duplicated. 783 if (codeSystemVersionEntity != null) { 784 if (!ObjectUtil.equals(codeSystemVersionEntity.getResource().getId(), theCodeSystemResourceTable.getId())) { 785 throw new UnprocessableEntityException(Msg.code(848) + msg); 786 } 787 } 788 } 789 790 private void populateCodeSystemVersionProperties( 791 TermCodeSystemVersion theCodeSystemVersion, 792 CodeSystem theCodeSystemResource, 793 ResourceTable theResourceTable) { 794 theCodeSystemVersion.setResource(theResourceTable); 795 theCodeSystemVersion.setCodeSystemDisplayName(theCodeSystemResource.getName()); 796 theCodeSystemVersion.setCodeSystemVersionId(theCodeSystemResource.getVersion()); 797 } 798 799 private int validateConceptForStorage( 800 TermConcept theConcept, 801 TermCodeSystemVersion theCodeSystemVersion, 802 ArrayList<String> theConceptsStack, 803 IdentityHashMap<TermConcept, Object> theAllConcepts) { 804 ValidateUtil.isTrueOrThrowInvalidRequest( 805 theConcept.getCodeSystemVersion() != null, "CodeSystemVersion is null"); 806 ValidateUtil.isNotBlankOrThrowInvalidRequest( 807 theConcept.getCode(), "CodeSystem contains a code with no code value"); 808 809 theConcept.setCodeSystemVersion(theCodeSystemVersion); 810 if (theConceptsStack.contains(theConcept.getCode())) { 811 throw new InvalidRequestException( 812 Msg.code(849) + "CodeSystem contains circular reference around code " + theConcept.getCode()); 813 } 814 theConceptsStack.add(theConcept.getCode()); 815 816 int retVal = 0; 817 if (theAllConcepts.put(theConcept, theAllConcepts) == null) { 818 if (theAllConcepts.size() % 1000 == 0) { 819 ourLog.info("Have validated {} concepts", theAllConcepts.size()); 820 } 821 retVal = 1; 822 } 823 824 for (TermConceptParentChildLink next : theConcept.getChildren()) { 825 next.setCodeSystem(theCodeSystemVersion); 826 retVal += 827 validateConceptForStorage(next.getChild(), theCodeSystemVersion, theConceptsStack, theAllConcepts); 828 } 829 830 theConceptsStack.remove(theConceptsStack.size() - 1); 831 832 return retVal; 833 } 834}