001/* 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.term; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.context.FhirVersionEnum; 024import ca.uhn.fhir.context.support.ConceptValidationOptions; 025import ca.uhn.fhir.context.support.IValidationSupport; 026import ca.uhn.fhir.context.support.LookupCodeRequest; 027import ca.uhn.fhir.context.support.ValidationSupportContext; 028import ca.uhn.fhir.context.support.ValueSetExpansionOptions; 029import ca.uhn.fhir.i18n.Msg; 030import ca.uhn.fhir.interceptor.model.RequestPartitionId; 031import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 032import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 033import ca.uhn.fhir.jpa.api.dao.IDao; 034import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem; 036import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 037import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider; 038import ca.uhn.fhir.jpa.config.util.ConnectionPoolInfoProvider; 039import ca.uhn.fhir.jpa.config.util.IConnectionPoolInfoProvider; 040import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; 041import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser; 042import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; 043import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; 044import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; 045import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao; 046import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao; 047import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao; 048import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao; 049import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao; 050import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao; 051import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao; 052import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView; 053import ca.uhn.fhir.jpa.entity.TermCodeSystem; 054import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; 055import ca.uhn.fhir.jpa.entity.TermConcept; 056import ca.uhn.fhir.jpa.entity.TermConceptDesignation; 057import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; 058import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; 059import ca.uhn.fhir.jpa.entity.TermConceptProperty; 060import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum; 061import ca.uhn.fhir.jpa.entity.TermValueSet; 062import ca.uhn.fhir.jpa.entity.TermValueSetConcept; 063import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum; 064import ca.uhn.fhir.jpa.model.dao.JpaPid; 065import ca.uhn.fhir.jpa.model.entity.ResourceTable; 066import ca.uhn.fhir.jpa.model.sched.HapiJob; 067import ca.uhn.fhir.jpa.model.sched.IHasScheduledJobs; 068import ca.uhn.fhir.jpa.model.sched.ISchedulerService; 069import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition; 070import ca.uhn.fhir.jpa.model.util.JpaConstants; 071import ca.uhn.fhir.jpa.search.builder.SearchBuilder; 072import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; 073import ca.uhn.fhir.jpa.term.api.ITermReadSvc; 074import ca.uhn.fhir.jpa.term.api.ReindexTerminologyResult; 075import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException; 076import ca.uhn.fhir.rest.api.Constants; 077import ca.uhn.fhir.rest.api.server.RequestDetails; 078import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 079import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 080import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; 081import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; 082import ca.uhn.fhir.sl.cache.Cache; 083import ca.uhn.fhir.sl.cache.CacheFactory; 084import ca.uhn.fhir.util.CoverageIgnore; 085import ca.uhn.fhir.util.FhirVersionIndependentConcept; 086import ca.uhn.fhir.util.HapiExtensions; 087import ca.uhn.fhir.util.StopWatch; 088import ca.uhn.fhir.util.UrlUtil; 089import ca.uhn.fhir.util.ValidateUtil; 090import ca.uhn.hapi.converters.canonical.VersionCanonicalizer; 091import com.google.common.annotations.VisibleForTesting; 092import com.google.common.base.Stopwatch; 093import com.google.common.collect.ArrayListMultimap; 094import jakarta.annotation.Nonnull; 095import jakarta.annotation.Nullable; 096import jakarta.annotation.PostConstruct; 097import jakarta.persistence.EntityManager; 098import jakarta.persistence.NonUniqueResultException; 099import jakarta.persistence.PersistenceContext; 100import jakarta.persistence.PersistenceContextType; 101import org.apache.commons.collections4.ListUtils; 102import org.apache.commons.lang3.ObjectUtils; 103import org.apache.commons.lang3.StringUtils; 104import org.apache.commons.lang3.Validate; 105import org.apache.commons.lang3.time.DateUtils; 106import org.apache.lucene.index.Term; 107import org.apache.lucene.search.IndexSearcher; 108import org.hibernate.CacheMode; 109import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep; 110import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep; 111import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; 112import org.hibernate.search.engine.search.query.SearchQuery; 113import org.hibernate.search.engine.search.query.SearchScroll; 114import org.hibernate.search.engine.search.query.SearchScrollResult; 115import org.hibernate.search.mapper.orm.Search; 116import org.hibernate.search.mapper.orm.common.EntityReference; 117import org.hibernate.search.mapper.orm.session.SearchSession; 118import org.hibernate.search.mapper.pojo.massindexing.impl.PojoMassIndexingLoggingMonitor; 119import org.hl7.fhir.common.hapi.validation.support.CachingValidationSupport; 120import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; 121import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50; 122import org.hl7.fhir.convertors.context.ConversionContext40_50; 123import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50; 124import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50; 125import org.hl7.fhir.instance.model.api.IAnyResource; 126import org.hl7.fhir.instance.model.api.IBaseCoding; 127import org.hl7.fhir.instance.model.api.IBaseDatatype; 128import org.hl7.fhir.instance.model.api.IBaseResource; 129import org.hl7.fhir.instance.model.api.IIdType; 130import org.hl7.fhir.instance.model.api.IPrimitiveType; 131import org.hl7.fhir.r4.model.BooleanType; 132import org.hl7.fhir.r4.model.CanonicalType; 133import org.hl7.fhir.r4.model.CodeSystem; 134import org.hl7.fhir.r4.model.CodeableConcept; 135import org.hl7.fhir.r4.model.Coding; 136import org.hl7.fhir.r4.model.DomainResource; 137import org.hl7.fhir.r4.model.Enumerations; 138import org.hl7.fhir.r4.model.Extension; 139import org.hl7.fhir.r4.model.InstantType; 140import org.hl7.fhir.r4.model.IntegerType; 141import org.hl7.fhir.r4.model.StringType; 142import org.hl7.fhir.r4.model.ValueSet; 143import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome; 144import org.quartz.JobExecutionContext; 145import org.springframework.beans.factory.annotation.Autowired; 146import org.springframework.context.ApplicationContext; 147import org.springframework.data.domain.PageRequest; 148import org.springframework.data.domain.Pageable; 149import org.springframework.data.domain.Slice; 150import org.springframework.transaction.PlatformTransactionManager; 151import org.springframework.transaction.TransactionDefinition; 152import org.springframework.transaction.annotation.Propagation; 153import org.springframework.transaction.annotation.Transactional; 154import org.springframework.transaction.interceptor.NoRollbackRuleAttribute; 155import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute; 156import org.springframework.transaction.support.TransactionSynchronizationManager; 157import org.springframework.transaction.support.TransactionTemplate; 158import org.springframework.util.comparator.Comparators; 159 160import java.util.ArrayList; 161import java.util.Arrays; 162import java.util.Collection; 163import java.util.Collections; 164import java.util.Date; 165import java.util.HashMap; 166import java.util.HashSet; 167import java.util.LinkedHashMap; 168import java.util.List; 169import java.util.Map; 170import java.util.Objects; 171import java.util.Optional; 172import java.util.Set; 173import java.util.StringTokenizer; 174import java.util.UUID; 175import java.util.concurrent.TimeUnit; 176import java.util.function.Consumer; 177import java.util.function.Supplier; 178import java.util.stream.Collectors; 179 180import static ca.uhn.fhir.jpa.entity.TermConceptPropertyBinder.CONCEPT_PROPERTY_PREFIX_NAME; 181import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI; 182import static java.lang.String.join; 183import static java.util.stream.Collectors.joining; 184import static java.util.stream.Collectors.toList; 185import static java.util.stream.Collectors.toSet; 186import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; 187import static org.apache.commons.lang3.StringUtils.defaultString; 188import static org.apache.commons.lang3.StringUtils.isBlank; 189import static org.apache.commons.lang3.StringUtils.isEmpty; 190import static org.apache.commons.lang3.StringUtils.isNoneBlank; 191import static org.apache.commons.lang3.StringUtils.isNotBlank; 192import static org.apache.commons.lang3.StringUtils.lowerCase; 193import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase; 194 195public class TermReadSvcImpl implements ITermReadSvc, IHasScheduledJobs { 196 public static final int DEFAULT_FETCH_SIZE = 250; 197 public static final int DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS = 2; 198 // doesn't seem to be much gain by using more threads than this value 199 public static final int MAX_MASS_INDEXER_OBJECT_LOADING_THREADS = 6; 200 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TermReadSvcImpl.class); 201 private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions(); 202 private static final TermCodeSystemVersionDetails NO_CURRENT_VERSION = new TermCodeSystemVersionDetails(-1L, null); 203 private static final String OUR_PIPE_CHARACTER = "|"; 204 private static final int SECONDS_IN_MINUTE = 60; 205 private static final int INDEXED_ROOTS_LOGGING_COUNT = 50_000; 206 private static Runnable myInvokeOnNextCallForUnitTest; 207 private static boolean ourForceDisableHibernateSearchForUnitTest; 208 private final Cache<String, TermCodeSystemVersionDetails> myCodeSystemCurrentVersionCache = 209 CacheFactory.build(TimeUnit.MINUTES.toMillis(1)); 210 211 @Autowired 212 protected DaoRegistry myDaoRegistry; 213 214 @Autowired 215 protected ITermCodeSystemDao myCodeSystemDao; 216 217 @Autowired 218 protected ITermConceptDao myConceptDao; 219 220 @Autowired 221 protected ITermConceptPropertyDao myConceptPropertyDao; 222 223 @Autowired 224 protected ITermConceptDesignationDao myConceptDesignationDao; 225 226 @Autowired 227 protected ITermValueSetDao myTermValueSetDao; 228 229 @Autowired 230 protected ITermValueSetConceptDao myValueSetConceptDao; 231 232 @Autowired 233 protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao; 234 235 @Autowired 236 protected FhirContext myContext; 237 238 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 239 protected EntityManager myEntityManager; 240 241 private boolean myPreExpandingValueSets = false; 242 243 @Autowired 244 private ITermCodeSystemVersionDao myCodeSystemVersionDao; 245 246 @Autowired 247 private JpaStorageSettings myStorageSettings; 248 249 private TransactionTemplate myTxTemplate; 250 251 @Autowired 252 private PlatformTransactionManager myTransactionManager; 253 254 @Autowired(required = false) 255 private IFulltextSearchSvc myFulltextSearchSvc; 256 257 @Autowired 258 private PlatformTransactionManager myTxManager; 259 260 @Autowired 261 private ITermConceptDao myTermConceptDao; 262 263 @Autowired 264 private ITermValueSetConceptViewDao myTermValueSetConceptViewDao; 265 266 @Autowired 267 private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao; 268 269 @Autowired(required = false) 270 private ITermDeferredStorageSvc myDeferredStorageSvc; 271 272 @Autowired 273 private IIdHelperService<JpaPid> myIdHelperService; 274 275 @Autowired 276 private ApplicationContext myApplicationContext; 277 278 private volatile IValidationSupport myJpaValidationSupport; 279 private volatile IValidationSupport myValidationSupport; 280 // We need this bean so we can tell which mode hibernate search is running in. 281 @Autowired 282 private HibernatePropertiesProvider myHibernatePropertiesProvider; 283 284 @Autowired 285 private CachingValidationSupport myCachingValidationSupport; 286 287 @Autowired 288 private VersionCanonicalizer myVersionCanonicalizer; 289 290 @Autowired 291 private IJpaStorageResourceParser myJpaStorageResourceParser; 292 293 @Autowired 294 private InMemoryTerminologyServerValidationSupport myInMemoryTerminologyServerValidationSupport; 295 296 @Override 297 public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) { 298 TermCodeSystemVersionDetails cs = getCurrentCodeSystemVersion(theSystem); 299 return cs != null; 300 } 301 302 @Override 303 public boolean isValueSetSupported(ValidationSupportContext theValidationSupportContext, String theValueSetUrl) { 304 return fetchValueSet(theValueSetUrl) != null; 305 } 306 307 private boolean addCodeIfNotAlreadyAdded( 308 @Nullable ValueSetExpansionOptions theExpansionOptions, 309 IValueSetConceptAccumulator theValueSetCodeAccumulator, 310 Set<String> theAddedCodes, 311 TermConcept theConcept, 312 boolean theAdd, 313 String theValueSetIncludeVersion) { 314 String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); 315 String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId(); 316 String code = theConcept.getCode(); 317 String display = theConcept.getDisplay(); 318 Long sourceConceptPid = theConcept.getId(); 319 String directParentPids = ""; 320 321 if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) { 322 directParentPids = theConcept.getParents().stream() 323 .map(t -> t.getParent().getId().toString()) 324 .collect(joining(" ")); 325 } 326 327 Collection<TermConceptDesignation> designations = theConcept.getDesignations(); 328 if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) { 329 return addCodeIfNotAlreadyAdded( 330 theValueSetCodeAccumulator, 331 theAddedCodes, 332 designations, 333 theAdd, 334 codeSystem + OUR_PIPE_CHARACTER + theValueSetIncludeVersion, 335 code, 336 display, 337 sourceConceptPid, 338 directParentPids, 339 codeSystemVersion); 340 } else { 341 return addCodeIfNotAlreadyAdded( 342 theValueSetCodeAccumulator, 343 theAddedCodes, 344 designations, 345 theAdd, 346 codeSystem, 347 code, 348 display, 349 sourceConceptPid, 350 directParentPids, 351 codeSystemVersion); 352 } 353 } 354 355 private boolean addCodeIfNotAlreadyAdded( 356 IValueSetConceptAccumulator theValueSetCodeAccumulator, 357 Set<String> theAddedCodes, 358 boolean theAdd, 359 String theCodeSystem, 360 String theCodeSystemVersion, 361 String theCode, 362 String theDisplay, 363 Long theSourceConceptPid, 364 String theSourceConceptDirectParentPids, 365 Collection<TermConceptDesignation> theDesignations) { 366 if (StringUtils.isNotEmpty(theCodeSystemVersion)) { 367 if (isNoneBlank(theCodeSystem, theCode)) { 368 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 369 theValueSetCodeAccumulator.includeConceptWithDesignations( 370 theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion, 371 theCode, 372 theDisplay, 373 theDesignations, 374 theSourceConceptPid, 375 theSourceConceptDirectParentPids, 376 theCodeSystemVersion); 377 return true; 378 } 379 380 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 381 theValueSetCodeAccumulator.excludeConcept( 382 theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion, theCode); 383 return true; 384 } 385 } 386 } else { 387 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 388 theValueSetCodeAccumulator.includeConceptWithDesignations( 389 theCodeSystem, 390 theCode, 391 theDisplay, 392 theDesignations, 393 theSourceConceptPid, 394 theSourceConceptDirectParentPids, 395 theCodeSystemVersion); 396 return true; 397 } 398 399 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 400 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 401 return true; 402 } 403 } 404 405 return false; 406 } 407 408 private boolean addCodeIfNotAlreadyAdded( 409 IValueSetConceptAccumulator theValueSetCodeAccumulator, 410 Set<String> theAddedCodes, 411 Collection<TermConceptDesignation> theDesignations, 412 boolean theAdd, 413 String theCodeSystem, 414 String theCode, 415 String theDisplay, 416 Long theSourceConceptPid, 417 String theSourceConceptDirectParentPids, 418 String theSystemVersion) { 419 if (isNoneBlank(theCodeSystem, theCode)) { 420 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 421 theValueSetCodeAccumulator.includeConceptWithDesignations( 422 theCodeSystem, 423 theCode, 424 theDisplay, 425 theDesignations, 426 theSourceConceptPid, 427 theSourceConceptDirectParentPids, 428 theSystemVersion); 429 return true; 430 } 431 432 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 433 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 434 return true; 435 } 436 } 437 438 return false; 439 } 440 441 private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) { 442 boolean retVal = theSetToPopulate.add(theConcept); 443 if (retVal) { 444 if (theSetToPopulate.size() >= myStorageSettings.getMaximumExpansionSize()) { 445 String msg = myContext 446 .getLocalizer() 447 .getMessage( 448 TermReadSvcImpl.class, 449 "expansionTooLarge", 450 myStorageSettings.getMaximumExpansionSize()); 451 throw new ExpansionTooCostlyException(Msg.code(885) + msg); 452 } 453 } 454 return retVal; 455 } 456 457 /** 458 * This method is present only for unit tests, do not call from client code 459 */ 460 @VisibleForTesting 461 public void clearCaches() { 462 myCodeSystemCurrentVersionCache.invalidateAll(); 463 } 464 465 public void deleteValueSetForResource(ResourceTable theResourceTable) { 466 // Get existing entity so it can be deleted. 467 Optional<TermValueSet> optionalExistingTermValueSetById = 468 myTermValueSetDao.findByResourcePid(theResourceTable.getId()); 469 470 if (optionalExistingTermValueSetById.isPresent()) { 471 TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get(); 472 473 ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId()); 474 deletePreCalculatedValueSetContents(existingTermValueSet); 475 myTermValueSetDao.deleteById(existingTermValueSet.getId()); 476 ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId()); 477 } 478 } 479 480 private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) { 481 myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId()); 482 myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId()); 483 } 484 485 @Override 486 @Transactional 487 public void deleteValueSetAndChildren(ResourceTable theResourceTable) { 488 deleteValueSetForResource(theResourceTable); 489 } 490 491 @Override 492 @Transactional 493 public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList( 494 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 495 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found 496 // in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may 497 // time-out. 498 499 ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl); 500 501 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 502 for (ValueSet.ValueSetExpansionContainsComponent nextContains : 503 expanded.getExpansion().getContains()) { 504 retVal.add(new FhirVersionIndependentConcept( 505 nextContains.getSystem(), 506 nextContains.getCode(), 507 nextContains.getDisplay(), 508 nextContains.getVersion())); 509 } 510 return retVal; 511 } 512 513 @Override 514 public ValueSet expandValueSet( 515 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 516 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl); 517 if (valueSet == null) { 518 throw new ResourceNotFoundException( 519 Msg.code(886) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl)); 520 } 521 522 return expandValueSet(theExpansionOptions, valueSet); 523 } 524 525 @Override 526 public ValueSet expandValueSet( 527 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) { 528 String filter = null; 529 if (theExpansionOptions != null) { 530 filter = theExpansionOptions.getFilter(); 531 } 532 return doExpandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter)); 533 } 534 535 private ValueSet doExpandValueSet( 536 @Nullable ValueSetExpansionOptions theExpansionOptions, 537 ValueSet theValueSetToExpand, 538 ExpansionFilter theFilter) { 539 Set<String> addedCodes = new HashSet<>(); 540 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null"); 541 542 ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions); 543 int offset = expansionOptions.getOffset(); 544 int count = expansionOptions.getCount(); 545 546 ValueSetExpansionComponentWithConceptAccumulator accumulator = 547 new ValueSetExpansionComponentWithConceptAccumulator( 548 myContext, count, expansionOptions.isIncludeHierarchy()); 549 accumulator.setHardExpansionMaximumSize(myStorageSettings.getMaximumExpansionSize()); 550 accumulator.setSkipCountRemaining(offset); 551 accumulator.setIdentifier(UUID.randomUUID().toString()); 552 accumulator.setTimestamp(new Date()); 553 accumulator.setOffset(offset); 554 555 if (theExpansionOptions != null && isHibernateSearchEnabled()) { 556 accumulator.addParameter().setName("offset").setValue(new IntegerType(offset)); 557 accumulator.addParameter().setName("count").setValue(new IntegerType(count)); 558 } 559 560 myTxTemplate.executeWithoutResult(tx -> expandValueSetIntoAccumulator( 561 theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true, addedCodes)); 562 563 if (accumulator.getTotalConcepts() != null) { 564 accumulator.setTotal(accumulator.getTotalConcepts()); 565 } 566 567 ValueSet valueSet = new ValueSet(); 568 valueSet.setUrl(theValueSetToExpand.getUrl()); 569 valueSet.setId(theValueSetToExpand.getId()); 570 valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE); 571 valueSet.setCompose(theValueSetToExpand.getCompose()); 572 valueSet.setExpansion(accumulator); 573 574 for (String next : accumulator.getMessages()) { 575 valueSet.getMeta() 576 .addExtension() 577 .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE) 578 .setValue(new StringType(next)); 579 } 580 581 if (expansionOptions.isIncludeHierarchy()) { 582 accumulator.applyHierarchy(); 583 } 584 585 return valueSet; 586 } 587 588 private void expandValueSetIntoAccumulator( 589 ValueSet theValueSetToExpand, 590 ValueSetExpansionOptions theExpansionOptions, 591 IValueSetConceptAccumulator theAccumulator, 592 ExpansionFilter theFilter, 593 boolean theAdd, 594 Set<String> theAddedCodes) { 595 Optional<TermValueSet> optionalTermValueSet; 596 if (theValueSetToExpand.hasUrl()) { 597 if (theValueSetToExpand.hasVersion()) { 598 optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion( 599 theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion()); 600 } else { 601 optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl()); 602 } 603 } else { 604 optionalTermValueSet = Optional.empty(); 605 } 606 607 /* 608 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion 609 */ 610 if (optionalTermValueSet.isEmpty()) { 611 ourLog.debug( 612 "ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", 613 getValueSetInfo(theValueSetToExpand)); 614 String msg = myContext 615 .getLocalizer() 616 .getMessage( 617 TermReadSvcImpl.class, 618 "valueSetExpandedUsingInMemoryExpansion", 619 getValueSetInfo(theValueSetToExpand)); 620 theAccumulator.addMessage(msg); 621 doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes); 622 return; 623 } 624 625 /* 626 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion 627 */ 628 TermValueSet termValueSet = optionalTermValueSet.get(); 629 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 630 String msg = myContext 631 .getLocalizer() 632 .getMessage( 633 TermReadSvcImpl.class, 634 "valueSetNotYetExpanded", 635 getValueSetInfo(theValueSetToExpand), 636 termValueSet.getExpansionStatus().name(), 637 termValueSet.getExpansionStatus().getDescription()); 638 theAccumulator.addMessage(msg); 639 doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes); 640 return; 641 } 642 643 /* 644 * ValueSet is pre-expanded in database so let's use that 645 */ 646 String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet); 647 String msg = myContext 648 .getLocalizer() 649 .getMessage(TermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp); 650 theAccumulator.addMessage(msg); 651 expandConcepts( 652 theExpansionOptions, 653 theAccumulator, 654 termValueSet, 655 theFilter, 656 theAdd, 657 theAddedCodes, 658 myHibernatePropertiesProvider.isOracleDialect()); 659 } 660 661 @Nonnull 662 private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) { 663 String expansionTimestamp = "(unknown)"; 664 if (termValueSet.getExpansionTimestamp() != null) { 665 String timeElapsed = StopWatch.formatMillis(System.currentTimeMillis() 666 - termValueSet.getExpansionTimestamp().getTime()); 667 expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString() + " (" 668 + timeElapsed + " ago)"; 669 } 670 return expansionTimestamp; 671 } 672 673 private void expandConcepts( 674 ValueSetExpansionOptions theExpansionOptions, 675 IValueSetConceptAccumulator theAccumulator, 676 TermValueSet theTermValueSet, 677 ExpansionFilter theFilter, 678 boolean theAdd, 679 Set<String> theAddedCodes, 680 boolean theOracle) { 681 // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its 682 // copy pasted sibling 683 Integer offset = theAccumulator.getSkipCountRemaining(); 684 offset = ObjectUtils.defaultIfNull(offset, 0); 685 offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue()); 686 687 Integer count = theAccumulator.getCapacityRemaining(); 688 count = defaultIfNull(count, myStorageSettings.getMaximumExpansionSize()); 689 690 int conceptsExpanded = 0; 691 int designationsExpanded = 0; 692 int toIndex = offset + count; 693 694 Collection<? extends ITermValueSetConceptView> conceptViews; 695 boolean wasFilteredResult = false; 696 String filterDisplayValue = null; 697 if (!theFilter.getFilters().isEmpty() 698 && JpaConstants.VALUESET_FILTER_DISPLAY.equals( 699 theFilter.getFilters().get(0).getProperty()) 700 && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) { 701 filterDisplayValue = 702 lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]")); 703 String displayValue = "%" + lowerCase(filterDisplayValue) + "%"; 704 if (theOracle) { 705 conceptViews = 706 myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 707 } else { 708 conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 709 } 710 wasFilteredResult = true; 711 } else { 712 if (theOracle) { 713 conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId( 714 offset, toIndex, theTermValueSet.getId()); 715 } else { 716 conceptViews = 717 myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId()); 718 } 719 theAccumulator.consumeSkipCount(offset); 720 if (theAdd) { 721 theAccumulator.incrementOrDecrementTotalConcepts( 722 true, theTermValueSet.getTotalConcepts().intValue()); 723 } 724 } 725 726 if (conceptViews.isEmpty()) { 727 logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded); 728 return; 729 } 730 731 Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>(); 732 ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create(); 733 Map<Long, Long> pidToSourcePid = new HashMap<>(); 734 Map<Long, String> pidToSourceDirectParentPids = new HashMap<>(); 735 736 for (ITermValueSetConceptView conceptView : conceptViews) { 737 738 String system = conceptView.getConceptSystemUrl(); 739 String code = conceptView.getConceptCode(); 740 String display = conceptView.getConceptDisplay(); 741 String systemVersion = conceptView.getConceptSystemVersion(); 742 743 // -- this is quick solution, may need to revisit 744 if (!applyFilter(display, filterDisplayValue)) { 745 continue; 746 } 747 748 Long conceptPid = conceptView.getConceptPid(); 749 if (!pidToConcept.containsKey(conceptPid)) { 750 FhirVersionIndependentConcept concept = 751 new FhirVersionIndependentConcept(system, code, display, systemVersion); 752 pidToConcept.put(conceptPid, concept); 753 } 754 755 // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations 756 // optional. 757 if (conceptView.getDesignationPid() != null) { 758 TermConceptDesignation designation = new TermConceptDesignation(); 759 760 if (isValueSetDisplayLanguageMatch(theExpansionOptions, conceptView.getDesignationLang())) { 761 designation.setUseSystem(conceptView.getDesignationUseSystem()); 762 designation.setUseCode(conceptView.getDesignationUseCode()); 763 designation.setUseDisplay(conceptView.getDesignationUseDisplay()); 764 designation.setValue(conceptView.getDesignationVal()); 765 designation.setLanguage(conceptView.getDesignationLang()); 766 pidToDesignations.put(conceptPid, designation); 767 } 768 769 if (++designationsExpanded % 250 == 0) { 770 logDesignationsExpanded( 771 "Expansion of designations in progress. ", theTermValueSet, designationsExpanded); 772 } 773 } 774 775 if (theAccumulator.isTrackingHierarchy()) { 776 pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid()); 777 pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids()); 778 } 779 780 if (++conceptsExpanded % 250 == 0) { 781 logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded); 782 } 783 } 784 785 for (Long nextPid : pidToConcept.keySet()) { 786 FhirVersionIndependentConcept concept = pidToConcept.get(nextPid); 787 List<TermConceptDesignation> designations = pidToDesignations.get(nextPid); 788 String system = concept.getSystem(); 789 String code = concept.getCode(); 790 String display = concept.getDisplay(); 791 String systemVersion = concept.getSystemVersion(); 792 793 if (theAdd) { 794 if (theAccumulator.getCapacityRemaining() != null) { 795 if (theAccumulator.getCapacityRemaining() == 0) { 796 break; 797 } 798 } 799 800 Long sourceConceptPid = pidToSourcePid.get(nextPid); 801 String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid); 802 if (theAddedCodes.add(system + OUR_PIPE_CHARACTER + code)) { 803 theAccumulator.includeConceptWithDesignations( 804 system, 805 code, 806 display, 807 designations, 808 sourceConceptPid, 809 sourceConceptDirectParentPids, 810 systemVersion); 811 if (wasFilteredResult) { 812 theAccumulator.incrementOrDecrementTotalConcepts(true, 1); 813 } 814 } 815 } else { 816 if (theAddedCodes.remove(system + OUR_PIPE_CHARACTER + code)) { 817 theAccumulator.excludeConcept(system, code); 818 theAccumulator.incrementOrDecrementTotalConcepts(false, 1); 819 } 820 } 821 } 822 823 logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded); 824 logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded); 825 } 826 827 private void logConceptsExpanded( 828 String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) { 829 if (theConceptsExpanded > 0) { 830 ourLog.debug( 831 "{}Have expanded {} concepts in ValueSet[{}]", 832 theLogDescriptionPrefix, 833 theConceptsExpanded, 834 theTermValueSet.getUrl()); 835 } 836 } 837 838 private void logDesignationsExpanded( 839 String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) { 840 if (theDesignationsExpanded > 0) { 841 ourLog.debug( 842 "{}Have expanded {} designations in ValueSet[{}]", 843 theLogDescriptionPrefix, 844 theDesignationsExpanded, 845 theTermValueSet.getUrl()); 846 } 847 } 848 849 public boolean applyFilter(final String theDisplay, final String theFilterDisplay) { 850 851 // -- safety check only, no need to apply filter 852 if (theDisplay == null || theFilterDisplay == null) return true; 853 854 // -- sentence case 855 if (startsWithIgnoreCase(theDisplay, theFilterDisplay)) return true; 856 857 // -- token case 858 return startsWithByWordBoundaries(theDisplay, theFilterDisplay); 859 } 860 861 private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) { 862 // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo' 863 StringTokenizer tok = new StringTokenizer(theDisplay); 864 List<String> tokens = new ArrayList<>(); 865 while (tok.hasMoreTokens()) { 866 String token = tok.nextToken(); 867 if (startsWithIgnoreCase(token, theFilterDisplay)) return true; 868 tokens.add(token); 869 } 870 871 // Allow to search by the end of the phrase. E.g. "working proficiency" will match "Limited working 872 // proficiency" 873 for (int start = 0; start <= tokens.size() - 1; ++start) { 874 for (int end = start + 1; end <= tokens.size(); ++end) { 875 String sublist = String.join(" ", tokens.subList(start, end)); 876 if (startsWithIgnoreCase(sublist, theFilterDisplay)) return true; 877 } 878 } 879 return false; 880 } 881 882 @Override 883 public void expandValueSet( 884 ValueSetExpansionOptions theExpansionOptions, 885 ValueSet theValueSetToExpand, 886 IValueSetConceptAccumulator theValueSetCodeAccumulator) { 887 Set<String> addedCodes = new HashSet<>(); 888 doExpandValueSet( 889 theExpansionOptions, 890 theValueSetToExpand, 891 theValueSetCodeAccumulator, 892 ExpansionFilter.NO_FILTER, 893 addedCodes); 894 } 895 896 /** 897 * Note: Not transactional because specific calls within this method 898 * get executed in a transaction 899 */ 900 private void doExpandValueSet( 901 ValueSetExpansionOptions theExpansionOptions, 902 ValueSet theValueSetToExpand, 903 IValueSetConceptAccumulator theValueSetCodeAccumulator, 904 @Nonnull ExpansionFilter theExpansionFilter, 905 Set<String> theAddedCodes) { 906 907 StopWatch sw = new StopWatch(); 908 String valueSetInfo = getValueSetInfo(theValueSetToExpand); 909 ourLog.debug("Working with {}", valueSetInfo); 910 911 // Offset can't be combined with excludes 912 Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining(); 913 if (skipCountRemaining != null && skipCountRemaining > 0) { 914 if (!theValueSetToExpand.getCompose().getExclude().isEmpty()) { 915 String msg = myContext 916 .getLocalizer() 917 .getMessage(TermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo); 918 throw new InvalidRequestException(Msg.code(887) + msg); 919 } 920 } 921 922 // Handle includes 923 ourLog.debug("Handling includes"); 924 for (ValueSet.ConceptSetComponent include : 925 theValueSetToExpand.getCompose().getInclude()) { 926 myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude( 927 theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, include, true, theExpansionFilter)); 928 } 929 930 // Handle excludes 931 ourLog.debug("Handling excludes"); 932 for (ValueSet.ConceptSetComponent exclude : 933 theValueSetToExpand.getCompose().getExclude()) { 934 myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude( 935 theExpansionOptions, 936 theValueSetCodeAccumulator, 937 theAddedCodes, 938 exclude, 939 false, 940 ExpansionFilter.NO_FILTER)); 941 } 942 943 if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) { 944 myTxTemplate.execute( 945 t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder()); 946 } 947 948 ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis()); 949 } 950 951 private String getValueSetInfo(ValueSet theValueSet) { 952 StringBuilder sb = new StringBuilder(); 953 boolean isIdentified = false; 954 if (theValueSet.hasUrl()) { 955 isIdentified = true; 956 sb.append("ValueSet.url[").append(theValueSet.getUrl()).append("]"); 957 } else if (theValueSet.hasId()) { 958 isIdentified = true; 959 sb.append("ValueSet.id[").append(theValueSet.getId()).append("]"); 960 } 961 962 if (!isIdentified) { 963 sb.append("Unidentified ValueSet"); 964 } 965 966 return sb.toString(); 967 } 968 969 /** 970 * Returns true if there are potentially more results to process. 971 */ 972 private void expandValueSetHandleIncludeOrExclude( 973 @Nullable ValueSetExpansionOptions theExpansionOptions, 974 IValueSetConceptAccumulator theValueSetCodeAccumulator, 975 Set<String> theAddedCodes, 976 ValueSet.ConceptSetComponent theIncludeOrExclude, 977 boolean theAdd, 978 @Nonnull ExpansionFilter theExpansionFilter) { 979 980 String system = theIncludeOrExclude.getSystem(); 981 boolean hasSystem = isNotBlank(system); 982 boolean hasValueSet = !theIncludeOrExclude.getValueSet().isEmpty(); 983 984 if (hasSystem) { 985 986 if (theExpansionFilter.hasCode() 987 && theExpansionFilter.getSystem() != null 988 && !system.equals(theExpansionFilter.getSystem())) { 989 return; 990 } 991 992 ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system); 993 994 Optional<TermCodeSystemVersion> termCodeSystemVersion = 995 optionalFindTermCodeSystemVersion(theIncludeOrExclude); 996 if (termCodeSystemVersion.isPresent()) { 997 998 expandValueSetHandleIncludeOrExcludeUsingDatabase( 999 theExpansionOptions, 1000 theValueSetCodeAccumulator, 1001 theAddedCodes, 1002 theIncludeOrExclude, 1003 theAdd, 1004 theExpansionFilter, 1005 system, 1006 termCodeSystemVersion.get()); 1007 1008 } else { 1009 1010 if (!theIncludeOrExclude.getConcept().isEmpty() && theExpansionFilter.hasCode()) { 1011 if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) { 1012 if (theIncludeOrExclude.getConcept().stream() 1013 .noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) { 1014 return; 1015 } 1016 } 1017 } 1018 1019 Consumer<FhirVersionIndependentConcept> consumer = c -> addOrRemoveCode( 1020 theValueSetCodeAccumulator, 1021 theAddedCodes, 1022 theAdd, 1023 system, 1024 c.getCode(), 1025 c.getDisplay(), 1026 c.getSystemVersion()); 1027 1028 try { 1029 ConversionContext40_50.INSTANCE.init( 1030 new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet"); 1031 org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude = 1032 ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude); 1033 myInMemoryTerminologyServerValidationSupport.expandValueSetIncludeOrExclude( 1034 new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude); 1035 } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) { 1036 if (theExpansionOptions != null 1037 && !theExpansionOptions.isFailOnMissingCodeSystem() 1038 && e.getFailureType() 1039 == InMemoryTerminologyServerValidationSupport.FailureType.UNKNOWN_CODE_SYSTEM) { 1040 return; 1041 } 1042 throw new InternalErrorException(Msg.code(888) + e); 1043 } finally { 1044 ConversionContext40_50.INSTANCE.close("ValueSet"); 1045 } 1046 } 1047 1048 } else if (hasValueSet) { 1049 1050 for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) { 1051 String valueSetUrl = nextValueSet.getValueAsString(); 1052 ourLog.debug( 1053 "Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl); 1054 1055 ExpansionFilter subExpansionFilter = new ExpansionFilter( 1056 theExpansionFilter, 1057 theIncludeOrExclude.getFilter(), 1058 theValueSetCodeAccumulator.getCapacityRemaining()); 1059 1060 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not 1061 // be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the 1062 // expansion may time-out. 1063 1064 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl); 1065 if (valueSet == null) { 1066 throw new ResourceNotFoundException( 1067 Msg.code(889) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl)); 1068 } 1069 1070 expandValueSetIntoAccumulator( 1071 valueSet, 1072 theExpansionOptions, 1073 theValueSetCodeAccumulator, 1074 subExpansionFilter, 1075 theAdd, 1076 theAddedCodes); 1077 } 1078 1079 } else { 1080 throw new InvalidRequestException(Msg.code(890) + "ValueSet contains " + (theAdd ? "include" : "exclude") 1081 + " criteria with no system defined"); 1082 } 1083 } 1084 1085 private Optional<TermCodeSystemVersion> optionalFindTermCodeSystemVersion( 1086 ValueSet.ConceptSetComponent theIncludeOrExclude) { 1087 if (isEmpty(theIncludeOrExclude.getVersion())) { 1088 return Optional.ofNullable(myCodeSystemDao.findByCodeSystemUri(theIncludeOrExclude.getSystem())) 1089 .map(TermCodeSystem::getCurrentVersion); 1090 } else { 1091 return Optional.ofNullable(myCodeSystemVersionDao.findByCodeSystemUriAndVersion( 1092 theIncludeOrExclude.getSystem(), theIncludeOrExclude.getVersion())); 1093 } 1094 } 1095 1096 private boolean isHibernateSearchEnabled() { 1097 return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest; 1098 } 1099 1100 private void expandValueSetHandleIncludeOrExcludeUsingDatabase( 1101 ValueSetExpansionOptions theExpansionOptions, 1102 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1103 Set<String> theAddedCodes, 1104 ValueSet.ConceptSetComponent theIncludeOrExclude, 1105 boolean theAdd, 1106 @Nonnull ExpansionFilter theExpansionFilter, 1107 String theSystem, 1108 TermCodeSystemVersion theTermCodeSystemVersion) { 1109 1110 StopWatch fullOperationSw = new StopWatch(); 1111 String includeOrExcludeVersion = theIncludeOrExclude.getVersion(); 1112 1113 /* 1114 * If FullText searching is not enabled, we can handle only basic expansions 1115 * since we're going to do it without the database. 1116 */ 1117 if (!isHibernateSearchEnabled()) { 1118 expandWithoutHibernateSearch( 1119 theValueSetCodeAccumulator, 1120 theTermCodeSystemVersion, 1121 theAddedCodes, 1122 theIncludeOrExclude, 1123 theSystem, 1124 theAdd); 1125 return; 1126 } 1127 1128 /* 1129 * Ok, let's use hibernate search to build the expansion 1130 */ 1131 1132 int count = 0; 1133 1134 Optional<Integer> chunkSizeOpt = getScrollChunkSize(theAdd, theValueSetCodeAccumulator); 1135 if (chunkSizeOpt.isEmpty()) { 1136 return; 1137 } 1138 int chunkSize = chunkSizeOpt.get(); 1139 1140 /* 1141 * Turn the filter into one or more Hibernate Search queries. Ideally we want it 1142 * to be handled by a single query, but Lucene/ES don't like it when we exceed 1143 * 1024 different terms in a single query. So if we have that many terms (which 1144 * can happen if a ValueSet has a lot of explicitly enumerated codes that it's 1145 * including) we split this into multiple searches. The method below builds these 1146 * searches lazily, returning a Supplier that creates and executes the search 1147 * when it's actually time to. 1148 */ 1149 SearchProperties searchProps = buildSearchScrolls( 1150 theTermCodeSystemVersion, 1151 theExpansionFilter, 1152 theSystem, 1153 theIncludeOrExclude, 1154 chunkSize, 1155 includeOrExcludeVersion); 1156 int accumulatedBatchesSoFar = 0; 1157 for (var next : searchProps.getSearchScroll()) { 1158 try (SearchScroll<EntityReference> scroll = next.get()) { 1159 1160 ourLog.debug( 1161 "Beginning batch expansion for {} with max results per batch: {}", 1162 (theAdd ? "inclusion" : "exclusion"), 1163 chunkSize); 1164 for (SearchScrollResult<EntityReference> chunk = scroll.next(); 1165 chunk.hasHits(); 1166 chunk = scroll.next()) { 1167 int countForBatch = 0; 1168 1169 List<Long> pids = 1170 chunk.hits().stream().map(t -> (Long) t.id()).collect(Collectors.toList()); 1171 1172 List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids); 1173 1174 // If the include section had multiple codes, return the codes in the same order 1175 termConcepts = sortTermConcepts(searchProps, termConcepts); 1176 1177 // int firstResult = theQueryIndex * maxResultsPerBatch; 1178 // TODO GGG HS we lose the ability to check the 1179 // index of the first result, so just best-guessing it here. 1180 int delta = 0; 1181 for (TermConcept concept : termConcepts) { 1182 count++; 1183 countForBatch++; 1184 if (theAdd && searchProps.hasIncludeOrExcludeCodes()) { 1185 ValueSet.ConceptReferenceComponent theIncludeConcept = 1186 getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept); 1187 if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) { 1188 concept.setDisplay(theIncludeConcept.getDisplay()); 1189 } 1190 } 1191 boolean added = addCodeIfNotAlreadyAdded( 1192 theExpansionOptions, 1193 theValueSetCodeAccumulator, 1194 theAddedCodes, 1195 concept, 1196 theAdd, 1197 includeOrExcludeVersion); 1198 if (added) { 1199 delta++; 1200 } 1201 } 1202 1203 ourLog.debug( 1204 "Batch expansion scroll for {} with offset {} produced {} results in {}ms", 1205 (theAdd ? "inclusion" : "exclusion"), 1206 accumulatedBatchesSoFar, 1207 chunk.hits().size(), 1208 chunk.took().toMillis()); 1209 1210 theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta); 1211 accumulatedBatchesSoFar += countForBatch; 1212 1213 // keep session bounded 1214 myEntityManager.flush(); 1215 myEntityManager.clear(); 1216 } 1217 1218 ourLog.debug( 1219 "Expansion for {} produced {} results in {}ms", 1220 (theAdd ? "inclusion" : "exclusion"), 1221 count, 1222 fullOperationSw.getMillis()); 1223 } 1224 } 1225 } 1226 1227 private List<TermConcept> sortTermConcepts(SearchProperties searchProps, List<TermConcept> termConcepts) { 1228 List<String> codes = searchProps.getIncludeOrExcludeCodes(); 1229 if (codes.size() > 1) { 1230 termConcepts = new ArrayList<>(termConcepts); 1231 Map<String, Integer> codeToIndex = new HashMap<>(codes.size()); 1232 for (int i = 0; i < codes.size(); i++) { 1233 codeToIndex.put(codes.get(i), i); 1234 } 1235 termConcepts.sort(((o1, o2) -> { 1236 Integer idx1 = codeToIndex.get(o1.getCode()); 1237 Integer idx2 = codeToIndex.get(o2.getCode()); 1238 return Comparators.nullsHigh().compare(idx1, idx2); 1239 })); 1240 } 1241 return termConcepts; 1242 } 1243 1244 private Optional<Integer> getScrollChunkSize( 1245 boolean theAdd, IValueSetConceptAccumulator theValueSetCodeAccumulator) { 1246 int maxResultsPerBatch = SearchBuilder.getMaximumPageSize(); 1247 1248 /* 1249 * If the accumulator is bounded, we may reduce the size of the query to 1250 * Lucene in order to be more efficient. 1251 */ 1252 if (theAdd) { 1253 Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining(); 1254 if (accumulatorCapacityRemaining != null) { 1255 maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1); 1256 } 1257 } 1258 return maxResultsPerBatch > 0 ? Optional.of(maxResultsPerBatch) : Optional.empty(); 1259 } 1260 1261 private SearchProperties buildSearchScrolls( 1262 TermCodeSystemVersion theTermCodeSystemVersion, 1263 ExpansionFilter theExpansionFilter, 1264 String theSystem, 1265 ValueSet.ConceptSetComponent theIncludeOrExclude, 1266 Integer theScrollChunkSize, 1267 String theIncludeOrExcludeVersion) { 1268 SearchSession searchSession = Search.session(myEntityManager); 1269 // Manually building a predicate since we need to throw it around. 1270 SearchPredicateFactory predicate = 1271 searchSession.scope(TermConcept.class).predicate(); 1272 1273 List<String> allCodes = theIncludeOrExclude.getConcept().stream() 1274 .filter(Objects::nonNull) 1275 .map(ValueSet.ConceptReferenceComponent::getCode) 1276 .filter(StringUtils::isNotBlank) 1277 .collect(Collectors.toList()); 1278 SearchProperties returnProps = new SearchProperties(); 1279 returnProps.setIncludeOrExcludeCodes(allCodes); 1280 1281 /* 1282 * Lucene/ES can't typically handle more than 1024 clauses per search, so if 1283 * we have more than that number (e.g. because of a ValueSet that explicitly 1284 * includes thousands of codes), we break this up into multiple searches. 1285 */ 1286 List<List<String>> partitionedCodes = ListUtils.partition(allCodes, IndexSearcher.getMaxClauseCount() - 10); 1287 if (partitionedCodes.isEmpty()) { 1288 partitionedCodes = List.of(List.of()); 1289 } 1290 1291 for (List<String> nextCodePartition : partitionedCodes) { 1292 Supplier<SearchScroll<EntityReference>> nextScroll = () -> { 1293 // Build the top-level expansion on filters. 1294 PredicateFinalStep step = predicate.bool(b -> { 1295 b.must(predicate 1296 .match() 1297 .field("myCodeSystemVersionPid") 1298 .matching(theTermCodeSystemVersion.getPid())); 1299 1300 if (theExpansionFilter.hasCode()) { 1301 b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode())); 1302 } 1303 1304 String codeSystemUrlAndVersion = 1305 buildCodeSystemUrlAndVersion(theSystem, theIncludeOrExcludeVersion); 1306 for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) { 1307 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 1308 } 1309 for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) { 1310 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 1311 } 1312 }); 1313 1314 // Add a selector on any explicitly enumerated codes in the VS component 1315 final PredicateFinalStep finishedQuery; 1316 if (nextCodePartition.isEmpty()) { 1317 finishedQuery = step; 1318 } else { 1319 PredicateFinalStep expansionStep = buildExpansionPredicate(nextCodePartition, predicate); 1320 finishedQuery = predicate.bool().must(step).must(expansionStep); 1321 } 1322 1323 SearchQuery<EntityReference> termConceptsQuery = searchSession 1324 .search(TermConcept.class) 1325 .selectEntityReference() 1326 .where(f -> finishedQuery) 1327 .toQuery(); 1328 1329 return termConceptsQuery.scroll(theScrollChunkSize); 1330 }; 1331 1332 returnProps.addSearchScroll(nextScroll); 1333 } 1334 1335 return returnProps; 1336 } 1337 1338 private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet( 1339 ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) { 1340 return theIncludeOrExclude.getConcept().stream() 1341 .filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode())) 1342 .findFirst() 1343 .orElse(null); 1344 } 1345 1346 /** 1347 * Helper method which builds a predicate for the expansion 1348 */ 1349 private PredicateFinalStep buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) { 1350 assert !theCodes.isEmpty(); 1351 return thePredicate.simpleQueryString().field("myCode").matching(String.join(" | ", theCodes)); 1352 } 1353 1354 private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) { 1355 String codeSystemUrlAndVersion; 1356 if (theIncludeOrExcludeVersion != null) { 1357 codeSystemUrlAndVersion = theSystem + OUR_PIPE_CHARACTER + theIncludeOrExcludeVersion; 1358 } else { 1359 codeSystemUrlAndVersion = theSystem; 1360 } 1361 return codeSystemUrlAndVersion; 1362 } 1363 1364 private @Nonnull ValueSetExpansionOptions provideExpansionOptions( 1365 @Nullable ValueSetExpansionOptions theExpansionOptions) { 1366 return Objects.requireNonNullElse(theExpansionOptions, DEFAULT_EXPANSION_OPTIONS); 1367 } 1368 1369 private void addOrRemoveCode( 1370 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1371 Set<String> theAddedCodes, 1372 boolean theAdd, 1373 String theSystem, 1374 String theCode, 1375 String theDisplay, 1376 String theSystemVersion) { 1377 if (theAdd && theAddedCodes.add(theSystem + OUR_PIPE_CHARACTER + theCode)) { 1378 theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion); 1379 } 1380 if (!theAdd && theAddedCodes.remove(theSystem + OUR_PIPE_CHARACTER + theCode)) { 1381 theValueSetCodeAccumulator.excludeConcept(theSystem, theCode); 1382 } 1383 } 1384 1385 private void handleFilter( 1386 String theCodeSystemIdentifier, 1387 SearchPredicateFactory theF, 1388 BooleanPredicateClausesStep<?> theB, 1389 ValueSet.ConceptSetFilterComponent theFilter) { 1390 if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) { 1391 return; 1392 } 1393 1394 if (isBlank(theFilter.getValue()) || theFilter.getOp() == null || isBlank(theFilter.getProperty())) { 1395 throw new InvalidRequestException( 1396 Msg.code(891) + "Invalid filter, must have fields populated: property op value"); 1397 } 1398 1399 switch (theFilter.getProperty()) { 1400 case "display:exact": 1401 case "display": 1402 handleFilterDisplay(theF, theB, theFilter); 1403 break; 1404 case "concept": 1405 case "code": 1406 handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter); 1407 break; 1408 case "parent": 1409 case "child": 1410 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1411 handleFilterLoincParentChild(theF, theB, theFilter); 1412 break; 1413 case "ancestor": 1414 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1415 handleFilterLoincAncestor(theCodeSystemIdentifier, theF, theB, theFilter); 1416 break; 1417 case "descendant": 1418 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1419 handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter); 1420 break; 1421 case "copyright": 1422 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1423 handleFilterLoincCopyright(theF, theB, theFilter); 1424 break; 1425 default: 1426 if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) { 1427 handleFilterRegex(theF, theB, theFilter); 1428 } else { 1429 handleFilterPropertyDefault(theF, theB, theFilter); 1430 } 1431 break; 1432 } 1433 } 1434 1435 private void handleFilterPropertyDefault( 1436 SearchPredicateFactory theF, 1437 BooleanPredicateClausesStep<?> theB, 1438 ValueSet.ConceptSetFilterComponent theFilter) { 1439 1440 String value = theFilter.getValue(); 1441 Term term = new Term(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty(), value); 1442 theB.must(theF.match().field(term.field()).matching(term.text())); 1443 } 1444 1445 private void handleFilterRegex( 1446 SearchPredicateFactory theF, 1447 BooleanPredicateClausesStep<?> theB, 1448 ValueSet.ConceptSetFilterComponent theFilter) { 1449 /* 1450 * We treat the regex filter as a match on the regex 1451 * anywhere in the property string. The spec does not 1452 * say whether this is the right behaviour or not, but 1453 * there are examples that seem to suggest that it is. 1454 */ 1455 String value = theFilter.getValue(); 1456 if (value.endsWith("$")) { 1457 value = value.substring(0, value.length() - 1); 1458 } else if (!value.endsWith(".*")) { 1459 value = value + ".*"; 1460 } 1461 if (!value.startsWith("^") && !value.startsWith(".*")) { 1462 value = ".*" + value; 1463 } else if (value.startsWith("^")) { 1464 value = value.substring(1); 1465 } 1466 1467 theB.must(theF.regexp() 1468 .field(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty()) 1469 .matching(value)); 1470 } 1471 1472 private void handleFilterLoincCopyright( 1473 SearchPredicateFactory theF, 1474 BooleanPredicateClausesStep<?> theB, 1475 ValueSet.ConceptSetFilterComponent theFilter) { 1476 1477 if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1478 1479 String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase(); 1480 switch (copyrightFilterValue) { 1481 case "3rdparty": 1482 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1483 addFilterLoincCopyright3rdParty(theF, theB); 1484 break; 1485 case "loinc": 1486 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1487 addFilterLoincCopyrightLoinc(theF, theB); 1488 break; 1489 default: 1490 throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1491 } 1492 1493 } else { 1494 throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty()); 1495 } 1496 } 1497 1498 private void addFilterLoincCopyrightLoinc(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) { 1499 theB.mustNot(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE")); 1500 } 1501 1502 private void addFilterLoincCopyright3rdParty(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) { 1503 theB.must(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE")); 1504 } 1505 1506 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1507 private void handleFilterLoincAncestor( 1508 String theSystem, 1509 SearchPredicateFactory f, 1510 BooleanPredicateClausesStep<?> b, 1511 ValueSet.ConceptSetFilterComponent theFilter) { 1512 switch (theFilter.getOp()) { 1513 case EQUAL: 1514 addLoincFilterAncestorEqual(theSystem, f, b, theFilter); 1515 break; 1516 case IN: 1517 addLoincFilterAncestorIn(theSystem, f, b, theFilter); 1518 break; 1519 default: 1520 throw new InvalidRequestException(Msg.code(892) + "Don't know how to handle op=" + theFilter.getOp() 1521 + " on property " + theFilter.getProperty()); 1522 } 1523 } 1524 1525 private void addLoincFilterAncestorEqual( 1526 String theSystem, 1527 SearchPredicateFactory f, 1528 BooleanPredicateClausesStep<?> b, 1529 ValueSet.ConceptSetFilterComponent theFilter) { 1530 addLoincFilterAncestorEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue()); 1531 } 1532 1533 private void addLoincFilterAncestorEqual( 1534 String theSystem, 1535 SearchPredicateFactory f, 1536 BooleanPredicateClausesStep<?> b, 1537 String theProperty, 1538 String theValue) { 1539 List<Term> terms = getAncestorTerms(theSystem, theProperty, theValue); 1540 b.must(f.bool(innerB -> terms.forEach( 1541 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1542 } 1543 1544 private void addLoincFilterAncestorIn( 1545 String theSystem, 1546 SearchPredicateFactory f, 1547 BooleanPredicateClausesStep<?> b, 1548 ValueSet.ConceptSetFilterComponent theFilter) { 1549 String[] values = theFilter.getValue().split(","); 1550 List<Term> terms = new ArrayList<>(); 1551 for (String value : values) { 1552 terms.addAll(getAncestorTerms(theSystem, theFilter.getProperty(), value)); 1553 } 1554 b.must(f.bool(innerB -> terms.forEach( 1555 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1556 } 1557 1558 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1559 private void handleFilterLoincParentChild( 1560 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1561 switch (theFilter.getOp()) { 1562 case EQUAL: 1563 addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue()); 1564 break; 1565 case IN: 1566 addLoincFilterParentChildIn(f, b, theFilter); 1567 break; 1568 default: 1569 throw new InvalidRequestException(Msg.code(893) + "Don't know how to handle op=" + theFilter.getOp() 1570 + " on property " + theFilter.getProperty()); 1571 } 1572 } 1573 1574 private void addLoincFilterParentChildIn( 1575 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1576 String[] values = theFilter.getValue().split(","); 1577 List<Term> terms = new ArrayList<>(); 1578 for (String value : values) { 1579 logFilteringValueOnProperty(value, theFilter.getProperty()); 1580 terms.add(getPropertyTerm(theFilter.getProperty(), value)); 1581 } 1582 1583 b.must(f.bool(innerB -> terms.forEach( 1584 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1585 } 1586 1587 private void addLoincFilterParentChildEqual( 1588 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) { 1589 logFilteringValueOnProperty(theValue, theProperty); 1590 b.must(f.match().field(CONCEPT_PROPERTY_PREFIX_NAME + theProperty).matching(theValue)); 1591 } 1592 1593 private void handleFilterConceptAndCode( 1594 String theSystem, 1595 SearchPredicateFactory f, 1596 BooleanPredicateClausesStep<?> b, 1597 ValueSet.ConceptSetFilterComponent theFilter) { 1598 TermConcept code = findCodeForFilterCriteria(theSystem, theFilter); 1599 1600 if (theFilter.getOp() == ValueSet.FilterOperator.ISA) { 1601 ourLog.debug( 1602 " * Filtering on specific code and codes with a parent of {}/{}/{}", 1603 code.getId(), 1604 code.getCode(), 1605 code.getDisplay()); 1606 1607 b.must(f.bool() 1608 .should(f.match().field("myParentPids").matching("" + code.getId())) 1609 .should(f.match().field("myId").matching(code.getId()))); 1610 } else if (theFilter.getOp() == ValueSet.FilterOperator.DESCENDENTOF) { 1611 ourLog.debug( 1612 " * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay()); 1613 1614 b.must(f.match().field("myParentPids").matching("" + code.getId())); 1615 } else { 1616 throwInvalidFilter(theFilter, ""); 1617 } 1618 } 1619 1620 @Nonnull 1621 private TermConcept findCodeForFilterCriteria(String theSystem, ValueSet.ConceptSetFilterComponent theFilter) { 1622 return findCode(theSystem, theFilter.getValue()) 1623 .orElseThrow(() -> 1624 new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" 1625 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue())); 1626 } 1627 1628 private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) { 1629 throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() 1630 + " on property " + theFilter.getProperty() + theErrorSuffix); 1631 } 1632 1633 private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) { 1634 String systemUrl = getUrlFromIdentifier(theSystemIdentifier); 1635 if (!isCodeSystemLoinc(systemUrl)) { 1636 throw new InvalidRequestException(Msg.code(895) + "Invalid filter, property " + theProperty 1637 + " is LOINC-specific and cannot be used with system: " + systemUrl); 1638 } 1639 } 1640 1641 private boolean isCodeSystemLoinc(String theSystem) { 1642 return LOINC_URI.equals(theSystem); 1643 } 1644 1645 private void handleFilterDisplay( 1646 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1647 if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1648 addDisplayFilterExact(f, b, theFilter); 1649 } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1650 if (theFilter.getValue().trim().contains(" ")) { 1651 addDisplayFilterExact(f, b, theFilter); 1652 } else { 1653 addDisplayFilterInexact(f, b, theFilter); 1654 } 1655 } 1656 } 1657 1658 private void addDisplayFilterExact( 1659 SearchPredicateFactory f, 1660 BooleanPredicateClausesStep<?> bool, 1661 ValueSet.ConceptSetFilterComponent nextFilter) { 1662 bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue())); 1663 } 1664 1665 private void addDisplayFilterInexact( 1666 SearchPredicateFactory f, 1667 BooleanPredicateClausesStep<?> bool, 1668 ValueSet.ConceptSetFilterComponent nextFilter) { 1669 bool.must(f.phrase() 1670 .field("myDisplay") 1671 .boost(4.0f) 1672 .field("myDisplayWordEdgeNGram") 1673 .boost(1.0f) 1674 .field("myDisplayEdgeNGram") 1675 .boost(1.0f) 1676 .matching(nextFilter.getValue().toLowerCase()) 1677 .slop(2)); 1678 } 1679 1680 private Term getPropertyTerm(String theProperty, String theValue) { 1681 return new Term(CONCEPT_PROPERTY_PREFIX_NAME + theProperty, theValue); 1682 } 1683 1684 private List<Term> getAncestorTerms(String theSystem, String theProperty, String theValue) { 1685 List<Term> retVal = new ArrayList<>(); 1686 1687 TermConcept code = findCode(theSystem, theValue) 1688 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" 1689 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1690 1691 retVal.add(new Term("myParentPids", "" + code.getId())); 1692 logFilteringValueOnProperty(theValue, theProperty); 1693 1694 return retVal; 1695 } 1696 1697 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1698 private void handleFilterLoincDescendant( 1699 String theSystem, 1700 SearchPredicateFactory f, 1701 BooleanPredicateClausesStep<?> b, 1702 ValueSet.ConceptSetFilterComponent theFilter) { 1703 switch (theFilter.getOp()) { 1704 case EQUAL: 1705 addLoincFilterDescendantEqual(theSystem, f, b, theFilter); 1706 break; 1707 case IN: 1708 addLoincFilterDescendantIn(theSystem, f, b, theFilter); 1709 break; 1710 default: 1711 throw new InvalidRequestException(Msg.code(896) + "Don't know how to handle op=" + theFilter.getOp() 1712 + " on property " + theFilter.getProperty()); 1713 } 1714 } 1715 1716 private void addLoincFilterDescendantEqual( 1717 String theSystem, 1718 SearchPredicateFactory f, 1719 BooleanPredicateClausesStep<?> b, 1720 ValueSet.ConceptSetFilterComponent theFilter) { 1721 1722 List<Long> parentPids = getCodeParentPids(theSystem, theFilter.getProperty(), theFilter.getValue()); 1723 if (parentPids.isEmpty()) { 1724 // Can't return empty must, because it wil match according to other predicates. 1725 // Some day there will be a 'matchNone' predicate 1726 // (https://discourse.hibernate.org/t/fail-fast-predicate/6062) 1727 b.mustNot(f.matchAll()); 1728 return; 1729 } 1730 1731 b.must(f.bool(innerB -> { 1732 innerB.minimumShouldMatchNumber(1); 1733 parentPids.forEach(pid -> innerB.should(f.match().field("myId").matching(pid))); 1734 })); 1735 } 1736 1737 /** 1738 * We are looking for codes which have codes indicated in theFilter.getValue() as descendants. 1739 * Strategy is to find codes which have their pId(s) in the list of the parentId(s) of all the TermConcept(s) 1740 * representing the codes in theFilter.getValue() 1741 */ 1742 private void addLoincFilterDescendantIn( 1743 String theSystem, 1744 SearchPredicateFactory f, 1745 BooleanPredicateClausesStep<?> b, 1746 ValueSet.ConceptSetFilterComponent theFilter) { 1747 1748 String[] values = theFilter.getValue().split(","); 1749 if (values.length == 0) { 1750 throw new InvalidRequestException(Msg.code(2062) + "Invalid filter criteria - no codes specified"); 1751 } 1752 1753 List<Long> descendantCodePidList = getMultipleCodeParentPids(theSystem, theFilter.getProperty(), values); 1754 1755 b.must(f.bool(innerB -> descendantCodePidList.forEach( 1756 pId -> innerB.should(f.match().field("myId").matching(pId))))); 1757 } 1758 1759 /** 1760 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1761 */ 1762 private List<Long> getCodeParentPids(String theSystem, String theProperty, String theValue) { 1763 TermConcept code = findCode(theSystem, theValue) 1764 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" 1765 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1766 1767 String[] parentPids = code.getParentPidsAsString().split(" "); 1768 List<Long> retVal = Arrays.stream(parentPids) 1769 .filter(pid -> !StringUtils.equals(pid, "NONE")) 1770 .map(Long::parseLong) 1771 .collect(Collectors.toList()); 1772 logFilteringValueOnProperty(theValue, theProperty); 1773 return retVal; 1774 } 1775 1776 /** 1777 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1778 */ 1779 private List<Long> getMultipleCodeParentPids(String theSystem, String theProperty, String[] theValues) { 1780 List<String> valuesList = Arrays.asList(theValues); 1781 List<TermConcept> termConcepts = findCodes(theSystem, valuesList); 1782 if (valuesList.size() != termConcepts.size()) { 1783 String exMsg = getTermConceptsFetchExceptionMsg(termConcepts, valuesList); 1784 throw new InvalidRequestException(Msg.code(2064) + "Invalid filter criteria - {" 1785 + Constants.codeSystemWithDefaultDescription(theSystem) + "}: " + exMsg); 1786 } 1787 1788 List<Long> retVal = termConcepts.stream() 1789 .flatMap(tc -> Arrays.stream(tc.getParentPidsAsString().split(" "))) 1790 .filter(pid -> !StringUtils.equals(pid, "NONE")) 1791 .map(Long::parseLong) 1792 .collect(Collectors.toList()); 1793 1794 logFilteringValueOnProperties(valuesList, theProperty); 1795 1796 return retVal; 1797 } 1798 1799 /** 1800 * Generate message indicating for which of theValues a TermConcept was not found 1801 */ 1802 private String getTermConceptsFetchExceptionMsg(List<TermConcept> theTermConcepts, List<String> theValues) { 1803 // case: more TermConcept(s) retrieved than codes queried 1804 if (theTermConcepts.size() > theValues.size()) { 1805 return "Invalid filter criteria - More TermConcepts were found than indicated codes. Queried codes: [" 1806 + join( 1807 ",", 1808 theValues + "]; Obtained TermConcept IDs, codes: [" 1809 + theTermConcepts.stream() 1810 .map(tc -> tc.getId() + ", " + tc.getCode()) 1811 .collect(joining("; ")) 1812 + "]"); 1813 } 1814 1815 // case: less TermConcept(s) retrieved than codes queried 1816 Set<String> matchedCodes = 1817 theTermConcepts.stream().map(TermConcept::getCode).collect(toSet()); 1818 List<String> notMatchedValues = 1819 theValues.stream().filter(v -> !matchedCodes.contains(v)).collect(toList()); 1820 1821 return "Invalid filter criteria - No TermConcept(s) were found for the requested codes: [" 1822 + join(",", notMatchedValues + "]"); 1823 } 1824 1825 private void logFilteringValueOnProperty(String theValue, String theProperty) { 1826 ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty); 1827 } 1828 1829 private void logFilteringValueOnProperties(List<String> theValues, String theProperty) { 1830 ourLog.debug(" * Filtering with values={} on property {}", String.join(", ", theValues), theProperty); 1831 } 1832 1833 private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) { 1834 throw new InvalidRequestException( 1835 Msg.code(897) + "Don't know how to handle op=" + theOp + " on property " + theProperty); 1836 } 1837 1838 private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) { 1839 throw new InvalidRequestException( 1840 Msg.code(898) + "Don't know how to handle value=" + theValue + " on property " + theProperty); 1841 } 1842 1843 private void expandWithoutHibernateSearch( 1844 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1845 TermCodeSystemVersion theVersion, 1846 Set<String> theAddedCodes, 1847 ValueSet.ConceptSetComponent theInclude, 1848 String theSystem, 1849 boolean theAdd) { 1850 ourLog.trace("Hibernate search is not enabled"); 1851 1852 if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) { 1853 Validate.isTrue( 1854 ((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator) 1855 .getParameter() 1856 .isEmpty(), 1857 "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server."); 1858 } 1859 1860 Validate.isTrue( 1861 isNotBlank(theSystem), 1862 "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server."); 1863 1864 for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { 1865 boolean handled = false; 1866 switch (nextFilter.getProperty()) { 1867 case "concept": 1868 case "code": 1869 if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) { 1870 theValueSetCodeAccumulator.addMessage( 1871 "Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient."); 1872 TermConcept code = findCodeForFilterCriteria(theSystem, nextFilter); 1873 addConceptAndChildren( 1874 theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code); 1875 handled = true; 1876 } 1877 break; 1878 } 1879 1880 if (!handled) { 1881 throwInvalidFilter( 1882 nextFilter, 1883 " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion functionality is available."); 1884 } 1885 } 1886 1887 if (theInclude.getFilter().isEmpty() && theInclude.getConcept().isEmpty()) { 1888 Collection<TermConcept> concepts = 1889 myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid()); 1890 for (TermConcept next : concepts) { 1891 addCodeIfNotAlreadyAdded( 1892 theValueSetCodeAccumulator, 1893 theAddedCodes, 1894 theAdd, 1895 theSystem, 1896 theInclude.getVersion(), 1897 next.getCode(), 1898 next.getDisplay(), 1899 next.getId(), 1900 next.getParentPidsAsString(), 1901 next.getDesignations()); 1902 } 1903 } 1904 1905 for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) { 1906 if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) { 1907 continue; 1908 } 1909 Collection<TermConceptDesignation> designations = next.getDesignation().stream() 1910 .map(t -> new TermConceptDesignation() 1911 .setValue(t.getValue()) 1912 .setLanguage(t.getLanguage()) 1913 .setUseCode(t.getUse().getCode()) 1914 .setUseSystem(t.getUse().getSystem()) 1915 .setUseDisplay(t.getUse().getDisplay())) 1916 .collect(Collectors.toList()); 1917 addCodeIfNotAlreadyAdded( 1918 theValueSetCodeAccumulator, 1919 theAddedCodes, 1920 theAdd, 1921 theSystem, 1922 theInclude.getVersion(), 1923 next.getCode(), 1924 next.getDisplay(), 1925 null, 1926 null, 1927 designations); 1928 } 1929 } 1930 1931 private void addConceptAndChildren( 1932 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1933 Set<String> theAddedCodes, 1934 ValueSet.ConceptSetComponent theInclude, 1935 String theSystem, 1936 boolean theAdd, 1937 TermConcept theConcept) { 1938 for (TermConcept nextChild : theConcept.getChildCodes()) { 1939 boolean added = addCodeIfNotAlreadyAdded( 1940 theValueSetCodeAccumulator, 1941 theAddedCodes, 1942 theAdd, 1943 theSystem, 1944 theInclude.getVersion(), 1945 nextChild.getCode(), 1946 nextChild.getDisplay(), 1947 nextChild.getId(), 1948 nextChild.getParentPidsAsString(), 1949 nextChild.getDesignations()); 1950 if (added) { 1951 addConceptAndChildren( 1952 theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild); 1953 } 1954 } 1955 } 1956 1957 @Override 1958 @Transactional 1959 public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) { 1960 IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails); 1961 ValueSet canonicalValueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet); 1962 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet); 1963 if (optionalTermValueSet.isEmpty()) { 1964 return myContext 1965 .getLocalizer() 1966 .getMessage(TermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId); 1967 } 1968 1969 ourLog.info( 1970 "Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl()); 1971 1972 TermValueSet termValueSet = optionalTermValueSet.get(); 1973 if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) { 1974 return myContext 1975 .getLocalizer() 1976 .getMessage( 1977 TermReadSvcImpl.class, 1978 "valueSetCantInvalidateNotYetPrecalculated", 1979 termValueSet.getUrl(), 1980 termValueSet.getExpansionStatus()); 1981 } 1982 1983 Long totalConcepts = termValueSet.getTotalConcepts(); 1984 1985 deletePreCalculatedValueSetContents(termValueSet); 1986 1987 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 1988 termValueSet.setExpansionTimestamp(null); 1989 myTermValueSetDao.save(termValueSet); 1990 1991 afterValueSetExpansionStatusChange(); 1992 1993 return myContext 1994 .getLocalizer() 1995 .getMessage( 1996 TermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts); 1997 } 1998 1999 @Override 2000 @Transactional 2001 public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) { 2002 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet); 2003 2004 if (optionalTermValueSet.isEmpty()) { 2005 ourLog.warn( 2006 "ValueSet is not present in terminology tables. Will perform in-memory code validation. {}", 2007 getValueSetInfo(theValueSet)); 2008 return false; 2009 } 2010 2011 TermValueSet termValueSet = optionalTermValueSet.get(); 2012 2013 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 2014 ourLog.warn( 2015 "{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}", 2016 getValueSetInfo(theValueSet), 2017 termValueSet.getExpansionStatus().name(), 2018 termValueSet.getExpansionStatus().getDescription()); 2019 return false; 2020 } 2021 2022 return true; 2023 } 2024 2025 private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) { 2026 JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 2027 return myTermValueSetDao.findByResourcePid(valueSetResourcePid.getId()); 2028 } 2029 2030 private JpaPid getValueSetResourcePersistentId(ValueSet theValueSet) { 2031 return myIdHelperService.resolveResourcePersistentIds( 2032 RequestPartitionId.allPartitions(), 2033 theValueSet.getIdElement().getResourceType(), 2034 theValueSet.getIdElement().getIdPart()); 2035 } 2036 2037 protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet( 2038 ConceptValidationOptions theValidationOptions, 2039 ValueSet theValueSet, 2040 String theSystem, 2041 String theCode, 2042 String theDisplay, 2043 Coding theCoding, 2044 CodeableConcept theCodeableConcept) { 2045 assert TransactionSynchronizationManager.isSynchronizationActive(); 2046 2047 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required"); 2048 JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 2049 2050 List<TermValueSetConcept> concepts = new ArrayList<>(); 2051 if (isNotBlank(theCode)) { 2052 if (theValidationOptions.isInferSystem()) { 2053 concepts.addAll( 2054 myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getId(), theCode)); 2055 } else if (isNotBlank(theSystem)) { 2056 concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode)); 2057 } 2058 } else if (theCoding != null) { 2059 if (theCoding.hasSystem() && theCoding.hasCode()) { 2060 concepts.addAll(findByValueSetResourcePidSystemAndCode( 2061 valueSetResourcePid, theCoding.getSystem(), theCoding.getCode())); 2062 } 2063 } else if (theCodeableConcept != null) { 2064 for (Coding coding : theCodeableConcept.getCoding()) { 2065 if (coding.hasSystem() && coding.hasCode()) { 2066 concepts.addAll(findByValueSetResourcePidSystemAndCode( 2067 valueSetResourcePid, coding.getSystem(), coding.getCode())); 2068 if (!concepts.isEmpty()) { 2069 break; 2070 } 2071 } 2072 } 2073 } else { 2074 return null; 2075 } 2076 2077 TermValueSet valueSetEntity = myTermValueSetDao 2078 .findByResourcePid(valueSetResourcePid.getId()) 2079 .orElseThrow(IllegalStateException::new); 2080 String timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity); 2081 String preExpansionMessage = myContext 2082 .getLocalizer() 2083 .getMessage(TermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription); 2084 2085 if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) { 2086 String systemVersion = null; 2087 for (TermValueSetConcept concept : concepts) { 2088 systemVersion = concept.getSystemVersion(); 2089 if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) { 2090 return new IValidationSupport.CodeValidationResult() 2091 .setCode(concept.getCode()) 2092 .setDisplay(concept.getDisplay()) 2093 .setCodeSystemVersion(concept.getSystemVersion()) 2094 .setSourceDetails(preExpansionMessage); 2095 } 2096 } 2097 2098 String expectedDisplay = concepts.get(0).getDisplay(); 2099 return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch( 2100 myContext, 2101 theCode, 2102 theDisplay, 2103 expectedDisplay, 2104 systemVersion, 2105 myStorageSettings.getIssueSeverityForCodeDisplayMismatch()); 2106 } 2107 2108 if (!concepts.isEmpty()) { 2109 return new IValidationSupport.CodeValidationResult() 2110 .setCode(concepts.get(0).getCode()) 2111 .setDisplay(concepts.get(0).getDisplay()) 2112 .setCodeSystemVersion(concepts.get(0).getSystemVersion()) 2113 .setMessage(preExpansionMessage); 2114 } 2115 2116 // Ok, we failed 2117 List<TermValueSetConcept> outcome = myValueSetConceptDao.findByTermValueSetIdSystemOnly( 2118 Pageable.ofSize(1), valueSetEntity.getId(), theSystem); 2119 String append; 2120 if (outcome.size() == 0) { 2121 append = " - No codes in ValueSet belong to CodeSystem with URL " + theSystem; 2122 } else { 2123 String unknownCodeMessage = myContext 2124 .getLocalizer() 2125 .getMessage(TermReadSvcImpl.class, "unknownCodeInSystem", theSystem, theCode); 2126 append = " - " + unknownCodeMessage + ". " + preExpansionMessage; 2127 } 2128 2129 return createFailureCodeValidationResult(theSystem, theCode, null, append); 2130 } 2131 2132 private CodeValidationResult createFailureCodeValidationResult( 2133 String theSystem, String theCode, String theCodeSystemVersion, String theAppend) { 2134 return new CodeValidationResult() 2135 .setSeverity(IssueSeverity.ERROR) 2136 .setCodeSystemVersion(theCodeSystemVersion) 2137 .setMessage("Unable to validate code " + theSystem + "#" + theCode + theAppend); 2138 } 2139 2140 private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode( 2141 JpaPid theResourcePid, String theSystem, String theCode) { 2142 assert TransactionSynchronizationManager.isSynchronizationActive(); 2143 2144 List<TermValueSetConcept> retVal = new ArrayList<>(); 2145 Optional<TermValueSetConcept> optionalTermValueSetConcept; 2146 int versionIndex = theSystem.indexOf(OUR_PIPE_CHARACTER); 2147 if (versionIndex >= 0) { 2148 String systemUrl = theSystem.substring(0, versionIndex); 2149 String systemVersion = theSystem.substring(versionIndex + 1); 2150 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion( 2151 theResourcePid.getId(), systemUrl, systemVersion, theCode); 2152 } else { 2153 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode( 2154 theResourcePid.getId(), theSystem, theCode); 2155 } 2156 optionalTermValueSetConcept.ifPresent(retVal::add); 2157 return retVal; 2158 } 2159 2160 private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 2161 for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) { 2162 TermConcept nextChild = nextChildLink.getChild(); 2163 if (addToSet(theSetToPopulate, nextChild)) { 2164 fetchChildren(nextChild, theSetToPopulate); 2165 } 2166 } 2167 } 2168 2169 private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) { 2170 TermCodeSystemVersion codeSystem = 2171 myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid); 2172 return myConceptDao.findByCodeSystemAndCode(codeSystem.getPid(), theCode); 2173 } 2174 2175 private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 2176 for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) { 2177 TermConcept nextChild = nextChildLink.getParent(); 2178 if (addToSet(theSetToPopulate, nextChild)) { 2179 fetchParents(nextChild, theSetToPopulate); 2180 } 2181 } 2182 } 2183 2184 @Override 2185 public Optional<TermConcept> findCode(String theCodeSystem, String theCode) { 2186 /* 2187 * Loading concepts without a transaction causes issues later on some 2188 * platforms (e.g. PSQL) so this transactiontemplate is here to make 2189 * sure that we always call this with an open transaction 2190 */ 2191 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2192 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_MANDATORY); 2193 txTemplate.setReadOnly(true); 2194 2195 return txTemplate.execute(t -> { 2196 TermCodeSystemVersionDetails csv = getCurrentCodeSystemVersion(theCodeSystem); 2197 if (csv == null) { 2198 return Optional.empty(); 2199 } 2200 return myConceptDao.findByCodeSystemAndCode(csv.myPid, theCode); 2201 }); 2202 } 2203 2204 @Override 2205 @Transactional(propagation = Propagation.MANDATORY) 2206 public List<TermConcept> findCodes(String theCodeSystem, List<String> theCodeList) { 2207 TermCodeSystemVersionDetails csv = getCurrentCodeSystemVersion(theCodeSystem); 2208 if (csv == null) { 2209 return Collections.emptyList(); 2210 } 2211 2212 return myConceptDao.findByCodeSystemAndCodeList(csv.myPid, theCodeList); 2213 } 2214 2215 @Nullable 2216 private TermCodeSystemVersionDetails getCurrentCodeSystemVersion(String theCodeSystemIdentifier) { 2217 String version = getVersionFromIdentifier(theCodeSystemIdentifier); 2218 TermCodeSystemVersionDetails retVal = myCodeSystemCurrentVersionCache.get( 2219 theCodeSystemIdentifier, 2220 t -> myTxTemplate.execute(tx -> { 2221 TermCodeSystemVersion csv = null; 2222 TermCodeSystem cs = 2223 myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier)); 2224 if (cs != null) { 2225 if (version != null) { 2226 csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version); 2227 } else if (cs.getCurrentVersion() != null) { 2228 csv = cs.getCurrentVersion(); 2229 } 2230 } 2231 if (csv != null) { 2232 return new TermCodeSystemVersionDetails(csv.getPid(), csv.getCodeSystemVersionId()); 2233 } else { 2234 return NO_CURRENT_VERSION; 2235 } 2236 })); 2237 if (retVal == NO_CURRENT_VERSION) { 2238 return null; 2239 } 2240 return retVal; 2241 } 2242 2243 private String getVersionFromIdentifier(String theUri) { 2244 String retVal = null; 2245 if (StringUtils.isNotEmpty((theUri))) { 2246 int versionSeparator = theUri.lastIndexOf('|'); 2247 if (versionSeparator != -1) { 2248 retVal = theUri.substring(versionSeparator + 1); 2249 } 2250 } 2251 return retVal; 2252 } 2253 2254 private String getUrlFromIdentifier(String theUri) { 2255 String retVal = theUri; 2256 if (StringUtils.isNotEmpty((theUri))) { 2257 int versionSeparator = theUri.lastIndexOf('|'); 2258 if (versionSeparator != -1) { 2259 retVal = theUri.substring(0, versionSeparator); 2260 } 2261 } 2262 return retVal; 2263 } 2264 2265 @Transactional(propagation = Propagation.REQUIRED) 2266 @Override 2267 public Set<TermConcept> findCodesAbove( 2268 Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 2269 StopWatch stopwatch = new StopWatch(); 2270 2271 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 2272 if (concept.isEmpty()) { 2273 return Collections.emptySet(); 2274 } 2275 2276 Set<TermConcept> retVal = new HashSet<>(); 2277 retVal.add(concept.get()); 2278 2279 fetchParents(concept.get(), retVal); 2280 2281 ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis()); 2282 return retVal; 2283 } 2284 2285 @Transactional 2286 @Override 2287 public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) { 2288 TermCodeSystem cs = getCodeSystem(theSystem); 2289 if (cs == null) { 2290 return findCodesAboveUsingBuiltInSystems(theSystem, theCode); 2291 } 2292 TermCodeSystemVersion csv = cs.getCurrentVersion(); 2293 2294 Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode); 2295 return toVersionIndependentConcepts(theSystem, codes); 2296 } 2297 2298 @Transactional(propagation = Propagation.REQUIRED) 2299 @Override 2300 public Set<TermConcept> findCodesBelow( 2301 Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 2302 Stopwatch stopwatch = Stopwatch.createStarted(); 2303 2304 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 2305 if (concept.isEmpty()) { 2306 return Collections.emptySet(); 2307 } 2308 2309 Set<TermConcept> retVal = new HashSet<>(); 2310 retVal.add(concept.get()); 2311 2312 fetchChildren(concept.get(), retVal); 2313 2314 ourLog.debug( 2315 "Fetched {} codes below code {} in {}ms", 2316 retVal.size(), 2317 theCode, 2318 stopwatch.elapsed(TimeUnit.MILLISECONDS)); 2319 return retVal; 2320 } 2321 2322 @Transactional 2323 @Override 2324 public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) { 2325 TermCodeSystem cs = getCodeSystem(theSystem); 2326 if (cs == null) { 2327 return findCodesBelowUsingBuiltInSystems(theSystem, theCode); 2328 } 2329 TermCodeSystemVersion csv = cs.getCurrentVersion(); 2330 2331 Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode); 2332 return toVersionIndependentConcepts(theSystem, codes); 2333 } 2334 2335 private TermCodeSystem getCodeSystem(String theSystem) { 2336 return myCodeSystemDao.findByCodeSystemUri(theSystem); 2337 } 2338 2339 @PostConstruct 2340 public void start() { 2341 RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute(); 2342 rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class)); 2343 myTxTemplate = new TransactionTemplate(myTransactionManager, rules); 2344 } 2345 2346 @Override 2347 public void scheduleJobs(ISchedulerService theSchedulerService) { 2348 // Register scheduled job to pre-expand ValueSets 2349 // In the future it would be great to make this a cluster-aware task somehow 2350 ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition(); 2351 vsJobDefinition.setId(getClass().getName()); 2352 vsJobDefinition.setJobClass(Job.class); 2353 theSchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition); 2354 } 2355 2356 @Override 2357 public synchronized void preExpandDeferredValueSetsToTerminologyTables() { 2358 if (!myStorageSettings.isEnableTaskPreExpandValueSets()) { 2359 return; 2360 } 2361 if (isNotSafeToPreExpandValueSets()) { 2362 ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded."); 2363 return; 2364 } 2365 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 2366 2367 while (true) { 2368 StopWatch sw = new StopWatch(); 2369 TermValueSet valueSetToExpand = txTemplate.execute(t -> { 2370 Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded(); 2371 if (optionalTermValueSet.isEmpty()) { 2372 return null; 2373 } 2374 2375 TermValueSet termValueSet = optionalTermValueSet.get(); 2376 termValueSet.setTotalConcepts(0L); 2377 termValueSet.setTotalConceptDesignations(0L); 2378 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS); 2379 return myTermValueSetDao.saveAndFlush(termValueSet); 2380 }); 2381 if (valueSetToExpand == null) { 2382 return; 2383 } 2384 2385 // We have a ValueSet to pre-expand. 2386 setPreExpandingValueSets(true); 2387 try { 2388 ValueSet valueSet = txTemplate.execute(t -> { 2389 TermValueSet refreshedValueSetToExpand = myTermValueSetDao 2390 .findById(valueSetToExpand.getId()) 2391 .orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId())); 2392 return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource()); 2393 }); 2394 assert valueSet != null; 2395 2396 ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator( 2397 valueSetToExpand, myTermValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao); 2398 ValueSetExpansionOptions options = new ValueSetExpansionOptions(); 2399 options.setIncludeHierarchy(true); 2400 expandValueSet(options, valueSet, accumulator); 2401 2402 // We are done with this ValueSet. 2403 txTemplate.executeWithoutResult(t -> { 2404 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED); 2405 valueSetToExpand.setExpansionTimestamp(new Date()); 2406 myTermValueSetDao.saveAndFlush(valueSetToExpand); 2407 }); 2408 2409 afterValueSetExpansionStatusChange(); 2410 2411 ourLog.info( 2412 "Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", 2413 valueSet.getId(), 2414 valueSet.getUrl(), 2415 accumulator.getConceptsSaved(), 2416 sw); 2417 2418 } catch (Exception e) { 2419 ourLog.error( 2420 "Failed to pre-expand ValueSet with URL[{}]: {}", valueSetToExpand.getUrl(), e.getMessage(), e); 2421 txTemplate.executeWithoutResult(t -> { 2422 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND); 2423 myTermValueSetDao.saveAndFlush(valueSetToExpand); 2424 }); 2425 2426 } finally { 2427 setPreExpandingValueSets(false); 2428 } 2429 } 2430 } 2431 2432 /* 2433 * If a ValueSet has just finished pre-expanding, let's flush the caches. This is 2434 * kind of a blunt tool, but it should ensure that users don't get unpredictable 2435 * results while they test changes, which is probably a worthwhile sacrifice 2436 */ 2437 private void afterValueSetExpansionStatusChange() { 2438 // TODO: JA2 - Move this caching into the memorycacheservice, and only purge the 2439 // relevant individual cache 2440 myCachingValidationSupport.invalidateCaches(); 2441 } 2442 2443 private synchronized boolean isPreExpandingValueSets() { 2444 return myPreExpandingValueSets; 2445 } 2446 2447 private synchronized void setPreExpandingValueSets(boolean thePreExpandingValueSets) { 2448 myPreExpandingValueSets = thePreExpandingValueSets; 2449 } 2450 2451 private boolean isNotSafeToPreExpandValueSets() { 2452 return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty(true); 2453 } 2454 2455 private Optional<TermValueSet> getNextTermValueSetNotExpanded() { 2456 Optional<TermValueSet> retVal = Optional.empty(); 2457 Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus( 2458 PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 2459 2460 if (!page.getContent().isEmpty()) { 2461 retVal = Optional.of(page.getContent().get(0)); 2462 } 2463 2464 return retVal; 2465 } 2466 2467 @Override 2468 @Transactional 2469 public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) { 2470 2471 ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied"); 2472 if (isPlaceholder(theValueSet)) { 2473 ourLog.info( 2474 "Not storing TermValueSet for placeholder {}", 2475 theValueSet.getIdElement().toVersionless().getValueAsString()); 2476 return; 2477 } 2478 2479 ValidateUtil.isNotBlankOrThrowUnprocessableEntity( 2480 theValueSet.getUrl(), "ValueSet has no value for ValueSet.url"); 2481 ourLog.info( 2482 "Storing TermValueSet for {}", 2483 theValueSet.getIdElement().toVersionless().getValueAsString()); 2484 2485 /* 2486 * Get CodeSystem and validate CodeSystemVersion 2487 */ 2488 TermValueSet termValueSet = new TermValueSet(); 2489 termValueSet.setResource(theResourceTable); 2490 termValueSet.setUrl(theValueSet.getUrl()); 2491 termValueSet.setVersion(theValueSet.getVersion()); 2492 termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null); 2493 2494 // Delete version being replaced 2495 deleteValueSetForResource(theResourceTable); 2496 2497 /* 2498 * Do the upload. 2499 */ 2500 String url = termValueSet.getUrl(); 2501 String version = termValueSet.getVersion(); 2502 Optional<TermValueSet> optionalExistingTermValueSetByUrl; 2503 if (version != null) { 2504 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version); 2505 } else { 2506 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url); 2507 } 2508 if (optionalExistingTermValueSetByUrl.isEmpty()) { 2509 2510 myTermValueSetDao.save(termValueSet); 2511 2512 } else { 2513 TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get(); 2514 String msg; 2515 if (version != null) { 2516 msg = myContext 2517 .getLocalizer() 2518 .getMessage( 2519 TermReadSvcImpl.class, 2520 "cannotCreateDuplicateValueSetUrlAndVersion", 2521 url, 2522 version, 2523 existingTermValueSet 2524 .getResource() 2525 .getIdDt() 2526 .toUnqualifiedVersionless() 2527 .getValue()); 2528 } else { 2529 msg = myContext 2530 .getLocalizer() 2531 .getMessage( 2532 TermReadSvcImpl.class, 2533 "cannotCreateDuplicateValueSetUrl", 2534 url, 2535 existingTermValueSet 2536 .getResource() 2537 .getIdDt() 2538 .toUnqualifiedVersionless() 2539 .getValue()); 2540 } 2541 throw new UnprocessableEntityException(Msg.code(902) + msg); 2542 } 2543 } 2544 2545 @Override 2546 @Transactional 2547 public IFhirResourceDaoCodeSystem.SubsumesResult subsumes( 2548 IPrimitiveType<String> theCodeA, 2549 IPrimitiveType<String> theCodeB, 2550 IPrimitiveType<String> theSystem, 2551 IBaseCoding theCodingA, 2552 IBaseCoding theCodingB) { 2553 FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA); 2554 FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB); 2555 2556 if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) { 2557 throw new InvalidRequestException( 2558 Msg.code(903) + "Unable to test subsumption across different code systems"); 2559 } 2560 2561 if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) { 2562 throw new InvalidRequestException( 2563 Msg.code(904) + "Unable to test subsumption across different code system versions"); 2564 } 2565 2566 String codeASystemIdentifier; 2567 if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) { 2568 codeASystemIdentifier = conceptA.getSystem() + OUR_PIPE_CHARACTER + conceptA.getSystemVersion(); 2569 } else { 2570 codeASystemIdentifier = conceptA.getSystem(); 2571 } 2572 TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode()) 2573 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA)); 2574 2575 String codeBSystemIdentifier; 2576 if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) { 2577 codeBSystemIdentifier = conceptB.getSystem() + OUR_PIPE_CHARACTER + conceptB.getSystemVersion(); 2578 } else { 2579 codeBSystemIdentifier = conceptB.getSystem(); 2580 } 2581 TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode()) 2582 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB)); 2583 2584 SearchSession searchSession = Search.session(myEntityManager); 2585 2586 ConceptSubsumptionOutcome subsumes; 2587 subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES); 2588 if (subsumes == null) { 2589 subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY); 2590 } 2591 if (subsumes == null) { 2592 subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED; 2593 } 2594 2595 return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes); 2596 } 2597 2598 @Override 2599 public IValidationSupport.LookupCodeResult lookupCode( 2600 ValidationSupportContext theValidationSupportContext, @Nonnull LookupCodeRequest theLookupCodeRequest) { 2601 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2602 return txTemplate.execute(t -> { 2603 final String theSystem = theLookupCodeRequest.getSystem(); 2604 final String theCode = theLookupCodeRequest.getCode(); 2605 Optional<TermConcept> codeOpt = findCode(theSystem, theCode); 2606 if (codeOpt.isPresent()) { 2607 TermConcept code = codeOpt.get(); 2608 2609 IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult(); 2610 result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName()); 2611 result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId()); 2612 result.setSearchedForSystem(theSystem); 2613 result.setSearchedForCode(theCode); 2614 result.setFound(true); 2615 result.setCodeDisplay(code.getDisplay()); 2616 2617 for (TermConceptDesignation next : code.getDesignations()) { 2618 // filter out the designation based on displayLanguage if any 2619 if (isDisplayLanguageMatch(theLookupCodeRequest.getDisplayLanguage(), next.getLanguage())) { 2620 IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation(); 2621 designation.setLanguage(next.getLanguage()); 2622 designation.setUseSystem(next.getUseSystem()); 2623 designation.setUseCode(next.getUseCode()); 2624 designation.setUseDisplay(next.getUseDisplay()); 2625 designation.setValue(next.getValue()); 2626 result.getDesignations().add(designation); 2627 } 2628 } 2629 2630 final Collection<String> propertyNames = theLookupCodeRequest.getPropertyNames(); 2631 for (TermConceptProperty next : code.getProperties()) { 2632 if (ObjectUtils.isNotEmpty(propertyNames) && !propertyNames.contains(next.getKey())) { 2633 continue; 2634 } 2635 if (next.getType() == TermConceptPropertyTypeEnum.CODING) { 2636 IValidationSupport.CodingConceptProperty property = 2637 new IValidationSupport.CodingConceptProperty( 2638 next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay()); 2639 result.getProperties().add(property); 2640 } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) { 2641 IValidationSupport.StringConceptProperty property = 2642 new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue()); 2643 result.getProperties().add(property); 2644 } else { 2645 throw new InternalErrorException(Msg.code(905) + "Unknown type: " + next.getType()); 2646 } 2647 } 2648 2649 return result; 2650 2651 } else { 2652 return new LookupCodeResult().setFound(false); 2653 } 2654 }); 2655 } 2656 2657 @Nullable 2658 private ConceptSubsumptionOutcome testForSubsumption( 2659 SearchSession theSearchSession, 2660 TermConcept theLeft, 2661 TermConcept theRight, 2662 ConceptSubsumptionOutcome theOutput) { 2663 List<TermConcept> fetch = theSearchSession 2664 .search(TermConcept.class) 2665 .where(f -> f.bool() 2666 .must(f.match().field("myId").matching(theRight.getId())) 2667 .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId())))) 2668 .fetchHits(1); 2669 2670 if (fetch.size() > 0) { 2671 return theOutput; 2672 } else { 2673 return null; 2674 } 2675 } 2676 2677 private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts( 2678 String theSystem, Set<TermConcept> codes) { 2679 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size()); 2680 for (TermConcept next : codes) { 2681 retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode())); 2682 } 2683 return retVal; 2684 } 2685 2686 @Override 2687 @Transactional 2688 public CodeValidationResult validateCodeInValueSet( 2689 ValidationSupportContext theValidationSupportContext, 2690 ConceptValidationOptions theOptions, 2691 String theCodeSystem, 2692 String theCode, 2693 String theDisplay, 2694 @Nonnull IBaseResource theValueSet) { 2695 invokeRunnableForUnitTest(); 2696 2697 IPrimitiveType<?> urlPrimitive; 2698 if (theValueSet instanceof org.hl7.fhir.dstu2.model.ValueSet) { 2699 urlPrimitive = FhirContext.forDstu2Hl7OrgCached() 2700 .newTerser() 2701 .getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); 2702 } else { 2703 urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); 2704 } 2705 String url = urlPrimitive.getValueAsString(); 2706 if (isNotBlank(url)) { 2707 return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url); 2708 } 2709 return null; 2710 } 2711 2712 @CoverageIgnore 2713 @Override 2714 public IValidationSupport.CodeValidationResult validateCode( 2715 @Nonnull ValidationSupportContext theValidationSupportContext, 2716 @Nonnull ConceptValidationOptions theOptions, 2717 String theCodeSystemUrl, 2718 String theCode, 2719 String theDisplay, 2720 String theValueSetUrl) { 2721 // TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS. 2722 invokeRunnableForUnitTest(); 2723 theOptions.setValidateDisplay(isNotBlank(theDisplay)); 2724 2725 if (isNotBlank(theValueSetUrl)) { 2726 return validateCodeInValueSet( 2727 theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystemUrl, theCode, theDisplay); 2728 } 2729 2730 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2731 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); 2732 txTemplate.setReadOnly(true); 2733 Optional<FhirVersionIndependentConcept> codeOpt = 2734 txTemplate.execute(tx -> findCode(theCodeSystemUrl, theCode).map(c -> { 2735 String codeSystemVersionId = getCurrentCodeSystemVersion(theCodeSystemUrl).myCodeSystemVersionId; 2736 return new FhirVersionIndependentConcept( 2737 theCodeSystemUrl, c.getCode(), c.getDisplay(), codeSystemVersionId); 2738 })); 2739 2740 if (codeOpt != null && codeOpt.isPresent()) { 2741 FhirVersionIndependentConcept code = codeOpt.get(); 2742 if (!theOptions.isValidateDisplay() 2743 || isBlank(code.getDisplay()) 2744 || isBlank(theDisplay) 2745 || code.getDisplay().equals(theDisplay)) { 2746 return new CodeValidationResult().setCode(code.getCode()).setDisplay(code.getDisplay()); 2747 } else { 2748 return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch( 2749 myContext, 2750 theCode, 2751 theDisplay, 2752 code.getDisplay(), 2753 code.getSystemVersion(), 2754 myStorageSettings.getIssueSeverityForCodeDisplayMismatch()); 2755 } 2756 } 2757 2758 return createFailureCodeValidationResult( 2759 theCodeSystemUrl, theCode, null, createMessageAppendForCodeNotFoundInCodeSystem(theCodeSystemUrl)); 2760 } 2761 2762 IValidationSupport.CodeValidationResult validateCodeInValueSet( 2763 ValidationSupportContext theValidationSupportContext, 2764 ConceptValidationOptions theValidationOptions, 2765 String theValueSetUrl, 2766 String theCodeSystem, 2767 String theCode, 2768 String theDisplay) { 2769 IBaseResource valueSet = 2770 theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl); 2771 CodeValidationResult retVal = null; 2772 2773 // If we don't have a PID, this came from some source other than the JPA 2774 // database, so we don't need to check if it's pre-expanded or not 2775 if (valueSet instanceof IAnyResource) { 2776 Long pid = IDao.RESOURCE_PID.get((IAnyResource) valueSet); 2777 if (pid != null) { 2778 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 2779 retVal = txTemplate.execute(tx -> { 2780 if (isValueSetPreExpandedForCodeValidation(valueSet)) { 2781 return validateCodeIsInPreExpandedValueSet( 2782 theValidationOptions, valueSet, theCodeSystem, theCode, theDisplay, null, null); 2783 } else { 2784 return null; 2785 } 2786 }); 2787 } 2788 } 2789 2790 if (retVal == null) { 2791 if (valueSet != null) { 2792 retVal = myInMemoryTerminologyServerValidationSupport.validateCodeInValueSet( 2793 theValidationSupportContext, 2794 theValidationOptions, 2795 theCodeSystem, 2796 theCode, 2797 theDisplay, 2798 valueSet); 2799 } else { 2800 String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]"; 2801 retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append); 2802 } 2803 } 2804 2805 // Check if someone is accidentally using a VS url where it should be a CS URL 2806 if (retVal != null 2807 && retVal.getCode() == null 2808 && theCodeSystem != null 2809 && myContext.getVersion().getVersion().isNewerThan(FhirVersionEnum.DSTU2)) { 2810 if (isValueSetSupported(theValidationSupportContext, theCodeSystem)) { 2811 if (!isCodeSystemSupported(theValidationSupportContext, theCodeSystem)) { 2812 String newMessage = "Unable to validate code " + theCodeSystem + "#" + theCode 2813 + " - Supplied system URL is a ValueSet URL and not a CodeSystem URL, check if it is correct: " 2814 + theCodeSystem; 2815 retVal.setMessage(newMessage); 2816 } 2817 } 2818 } 2819 2820 return retVal; 2821 } 2822 2823 @Override 2824 public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) { 2825 IValidationSupport validationSupport = provideValidationSupport(); 2826 IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem); 2827 if (codeSystem != null) { 2828 codeSystem = myVersionCanonicalizer.codeSystemToCanonical(codeSystem); 2829 } 2830 return (CodeSystem) codeSystem; 2831 } 2832 2833 @Nonnull 2834 private IValidationSupport provideJpaValidationSupport() { 2835 IValidationSupport jpaValidationSupport = myJpaValidationSupport; 2836 if (jpaValidationSupport == null) { 2837 jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class); 2838 myJpaValidationSupport = jpaValidationSupport; 2839 } 2840 return jpaValidationSupport; 2841 } 2842 2843 @Nonnull 2844 protected IValidationSupport provideValidationSupport() { 2845 IValidationSupport validationSupport = myValidationSupport; 2846 if (validationSupport == null) { 2847 validationSupport = myApplicationContext.getBean(IValidationSupport.class); 2848 myValidationSupport = validationSupport; 2849 } 2850 return validationSupport; 2851 } 2852 2853 public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) { 2854 IValidationSupport validationSupport = provideValidationSupport(); 2855 IBaseResource valueSet = validationSupport.fetchValueSet(theSystem); 2856 if (valueSet != null) { 2857 valueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet); 2858 } 2859 return (ValueSet) valueSet; 2860 } 2861 2862 @Override 2863 public IBaseResource fetchValueSet(String theValueSetUrl) { 2864 return provideJpaValidationSupport().fetchValueSet(theValueSetUrl); 2865 } 2866 2867 @Override 2868 public FhirContext getFhirContext() { 2869 return myContext; 2870 } 2871 2872 private void findCodesAbove( 2873 CodeSystem theSystem, 2874 String theSystemString, 2875 String theCode, 2876 List<FhirVersionIndependentConcept> theListToPopulate) { 2877 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 2878 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 2879 addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate); 2880 } 2881 } 2882 2883 @Override 2884 public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) { 2885 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 2886 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 2887 if (system != null) { 2888 findCodesAbove(system, theSystem, theCode, retVal); 2889 } 2890 return retVal; 2891 } 2892 2893 private void findCodesBelow( 2894 CodeSystem theSystem, 2895 String theSystemString, 2896 String theCode, 2897 List<FhirVersionIndependentConcept> theListToPopulate) { 2898 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 2899 findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList); 2900 } 2901 2902 private void findCodesBelow( 2903 String theSystemString, 2904 String theCode, 2905 List<FhirVersionIndependentConcept> theListToPopulate, 2906 List<CodeSystem.ConceptDefinitionComponent> conceptList) { 2907 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 2908 if (theCode.equals(next.getCode())) { 2909 addAllChildren(theSystemString, next, theListToPopulate); 2910 } else { 2911 findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept()); 2912 } 2913 } 2914 } 2915 2916 @Override 2917 public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) { 2918 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 2919 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 2920 if (system != null) { 2921 findCodesBelow(system, theSystem, theCode, retVal); 2922 } 2923 return retVal; 2924 } 2925 2926 private void addAllChildren( 2927 String theSystemString, 2928 CodeSystem.ConceptDefinitionComponent theCode, 2929 List<FhirVersionIndependentConcept> theListToPopulate) { 2930 if (isNotBlank(theCode.getCode())) { 2931 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode())); 2932 } 2933 for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) { 2934 addAllChildren(theSystemString, nextChild, theListToPopulate); 2935 } 2936 } 2937 2938 private boolean addTreeIfItContainsCode( 2939 String theSystemString, 2940 CodeSystem.ConceptDefinitionComponent theNext, 2941 String theCode, 2942 List<FhirVersionIndependentConcept> theListToPopulate) { 2943 boolean foundCodeInChild = false; 2944 for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) { 2945 foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate); 2946 } 2947 2948 if (theCode.equals(theNext.getCode()) || foundCodeInChild) { 2949 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode())); 2950 return true; 2951 } 2952 2953 return false; 2954 } 2955 2956 @Nonnull 2957 private FhirVersionIndependentConcept toConcept( 2958 IPrimitiveType<String> theCodeType, 2959 IPrimitiveType<String> theCodeSystemIdentifierType, 2960 IBaseCoding theCodingType) { 2961 String code = theCodeType != null ? theCodeType.getValueAsString() : null; 2962 String system = theCodeSystemIdentifierType != null 2963 ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) 2964 : null; 2965 String systemVersion = theCodeSystemIdentifierType != null 2966 ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) 2967 : null; 2968 if (theCodingType != null) { 2969 Coding canonicalizedCoding = myVersionCanonicalizer.codingToCanonical(theCodingType); 2970 assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't 2971 code = canonicalizedCoding.getCode(); 2972 system = canonicalizedCoding.getSystem(); 2973 systemVersion = canonicalizedCoding.getVersion(); 2974 } 2975 return new FhirVersionIndependentConcept(system, code, null, systemVersion); 2976 } 2977 2978 /** 2979 * When the search is for unversioned loinc system it uses the forcedId to obtain the current 2980 * version, as it is not necessarily the last one anymore. 2981 * For other cases it keeps on considering the last uploaded as the current 2982 */ 2983 @Override 2984 public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) { 2985 if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) { 2986 Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl); 2987 if (vsIdOpt.isEmpty()) { 2988 return Optional.empty(); 2989 } 2990 2991 return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get()); 2992 } 2993 2994 List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl); 2995 if (termValueSetList.isEmpty()) { 2996 return Optional.empty(); 2997 } 2998 2999 return Optional.of(termValueSetList.get(0)); 3000 } 3001 3002 @Override 3003 public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) { 3004 @SuppressWarnings("unchecked") 3005 List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager 3006 .createQuery("select r from ResourceTable r " 3007 + "where r.myResourceType = 'CodeSystem' and r.myFhirId = :fhirId") 3008 .setParameter("fhirId", theForcedId) 3009 .getResultList(); 3010 if (resultList.isEmpty()) return Optional.empty(); 3011 3012 if (resultList.size() > 1) 3013 throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " 3014 + theForcedId + ". Was constraint " + ResourceTable.IDX_RES_TYPE_FHIR_ID + " removed?"); 3015 3016 IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem"); 3017 IBaseResource cs = myJpaStorageResourceParser.toResource(resultList.get(0), false); 3018 return Optional.of(cs); 3019 } 3020 3021 @Transactional 3022 @Override 3023 public ReindexTerminologyResult reindexTerminology() throws InterruptedException { 3024 if (myFulltextSearchSvc == null) { 3025 return ReindexTerminologyResult.SEARCH_SVC_DISABLED; 3026 } 3027 3028 if (isBatchTerminologyTasksRunning()) { 3029 return ReindexTerminologyResult.OTHER_BATCH_TERMINOLOGY_TASKS_RUNNING; 3030 } 3031 3032 // disallow pre-expanding ValueSets while reindexing 3033 myDeferredStorageSvc.setProcessDeferred(false); 3034 3035 int objectLoadingThreadNumber = calculateObjectLoadingThreadNumber(); 3036 ourLog.info("Using {} threads to load objects", objectLoadingThreadNumber); 3037 3038 try { 3039 SearchSession searchSession = getSearchSession(); 3040 searchSession 3041 .massIndexer(TermConcept.class) 3042 .dropAndCreateSchemaOnStart(true) 3043 .purgeAllOnStart(false) 3044 .batchSizeToLoadObjects(100) 3045 .cacheMode(CacheMode.IGNORE) 3046 .threadsToLoadObjects(6) 3047 .transactionTimeout(60 * SECONDS_IN_MINUTE) 3048 .monitor(new PojoMassIndexingLoggingMonitor(INDEXED_ROOTS_LOGGING_COUNT)) 3049 .startAndWait(); 3050 } finally { 3051 myDeferredStorageSvc.setProcessDeferred(true); 3052 } 3053 3054 return ReindexTerminologyResult.SUCCESS; 3055 } 3056 3057 @VisibleForTesting 3058 boolean isBatchTerminologyTasksRunning() { 3059 return isNotSafeToPreExpandValueSets() || isPreExpandingValueSets(); 3060 } 3061 3062 @VisibleForTesting 3063 int calculateObjectLoadingThreadNumber() { 3064 IConnectionPoolInfoProvider connectionPoolInfoProvider = 3065 new ConnectionPoolInfoProvider(myHibernatePropertiesProvider.getDataSource()); 3066 Optional<Integer> maxConnectionsOpt = connectionPoolInfoProvider.getTotalConnectionSize(); 3067 if (maxConnectionsOpt.isEmpty()) { 3068 return DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS; 3069 } 3070 3071 int maxConnections = maxConnectionsOpt.get(); 3072 int usableThreads = maxConnections < 6 ? 1 : maxConnections - 5; 3073 int objectThreads = Math.min(usableThreads, MAX_MASS_INDEXER_OBJECT_LOADING_THREADS); 3074 ourLog.debug( 3075 "Data source connection pool has {} connections allocated, so reindexing will use {} object " 3076 + "loading threads (each using a connection)", 3077 maxConnections, 3078 objectThreads); 3079 return objectThreads; 3080 } 3081 3082 @VisibleForTesting 3083 SearchSession getSearchSession() { 3084 return Search.session(myEntityManager); 3085 } 3086 3087 @Override 3088 public ValueSetExpansionOutcome expandValueSet( 3089 ValidationSupportContext theValidationSupportContext, 3090 ValueSetExpansionOptions theExpansionOptions, 3091 @Nonnull IBaseResource theValueSetToExpand) { 3092 ValueSet canonicalInput = myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand); 3093 org.hl7.fhir.r4.model.ValueSet expandedR4 = expandValueSet(theExpansionOptions, canonicalInput); 3094 return new ValueSetExpansionOutcome(myVersionCanonicalizer.valueSetFromCanonical(expandedR4)); 3095 } 3096 3097 @Override 3098 public IBaseResource expandValueSet(ValueSetExpansionOptions theExpansionOptions, IBaseResource theInput) { 3099 org.hl7.fhir.r4.model.ValueSet valueSetToExpand = myVersionCanonicalizer.valueSetToCanonical(theInput); 3100 org.hl7.fhir.r4.model.ValueSet valueSetR4 = expandValueSet(theExpansionOptions, valueSetToExpand); 3101 return myVersionCanonicalizer.valueSetFromCanonical(valueSetR4); 3102 } 3103 3104 @Override 3105 public void expandValueSet( 3106 ValueSetExpansionOptions theExpansionOptions, 3107 IBaseResource theValueSetToExpand, 3108 IValueSetConceptAccumulator theValueSetCodeAccumulator) { 3109 org.hl7.fhir.r4.model.ValueSet valueSetToExpand = 3110 myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand); 3111 expandValueSet(theExpansionOptions, valueSetToExpand, theValueSetCodeAccumulator); 3112 } 3113 3114 private org.hl7.fhir.r4.model.ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) { 3115 Class<? extends IBaseResource> type = 3116 getFhirContext().getResourceDefinition("ValueSet").getImplementingClass(); 3117 IBaseResource valueSet = myJpaStorageResourceParser.toResource(type, theResourceTable, null, false); 3118 return myVersionCanonicalizer.valueSetToCanonical(valueSet); 3119 } 3120 3121 @Override 3122 public CodeValidationResult validateCodeIsInPreExpandedValueSet( 3123 ConceptValidationOptions theOptions, 3124 IBaseResource theValueSet, 3125 String theSystem, 3126 String theCode, 3127 String theDisplay, 3128 IBaseDatatype theCoding, 3129 IBaseDatatype theCodeableConcept) { 3130 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null"); 3131 org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet); 3132 org.hl7.fhir.r4.model.Coding codingR4 = myVersionCanonicalizer.codingToCanonical((IBaseCoding) theCoding); 3133 org.hl7.fhir.r4.model.CodeableConcept codeableConcept = 3134 myVersionCanonicalizer.codeableConceptToCanonical(theCodeableConcept); 3135 3136 return validateCodeIsInPreExpandedValueSet( 3137 theOptions, valueSetR4, theSystem, theCode, theDisplay, codingR4, codeableConcept); 3138 } 3139 3140 @Override 3141 public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) { 3142 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null"); 3143 org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet); 3144 return isValueSetPreExpandedForCodeValidation(valueSetR4); 3145 } 3146 3147 private static class TermCodeSystemVersionDetails { 3148 3149 private final long myPid; 3150 private final String myCodeSystemVersionId; 3151 3152 public TermCodeSystemVersionDetails(long thePid, String theCodeSystemVersionId) { 3153 myPid = thePid; 3154 myCodeSystemVersionId = theCodeSystemVersionId; 3155 } 3156 } 3157 3158 public static class Job implements HapiJob { 3159 @Autowired 3160 private ITermReadSvc myTerminologySvc; 3161 3162 @Override 3163 public void execute(JobExecutionContext theContext) { 3164 myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables(); 3165 } 3166 } 3167 3168 /** 3169 * Properties returned from method buildSearchScroll 3170 */ 3171 private static final class SearchProperties { 3172 private List<Supplier<SearchScroll<EntityReference>>> mySearchScroll = new ArrayList<>(); 3173 private List<String> myIncludeOrExcludeCodes; 3174 3175 public List<Supplier<SearchScroll<EntityReference>>> getSearchScroll() { 3176 return mySearchScroll; 3177 } 3178 3179 public void addSearchScroll(Supplier<SearchScroll<EntityReference>> theSearchScrollSupplier) { 3180 mySearchScroll.add(theSearchScrollSupplier); 3181 } 3182 3183 public List<String> getIncludeOrExcludeCodes() { 3184 return myIncludeOrExcludeCodes; 3185 } 3186 3187 public void setIncludeOrExcludeCodes(List<String> theIncludeOrExcludeCodes) { 3188 myIncludeOrExcludeCodes = theIncludeOrExcludeCodes; 3189 } 3190 3191 public boolean hasIncludeOrExcludeCodes() { 3192 return !myIncludeOrExcludeCodes.isEmpty(); 3193 } 3194 } 3195 3196 static boolean isValueSetDisplayLanguageMatch(ValueSetExpansionOptions theExpansionOptions, String theStoredLang) { 3197 if (theExpansionOptions == null) { 3198 return true; 3199 } 3200 3201 if (theExpansionOptions.getTheDisplayLanguage() == null || theStoredLang == null) { 3202 return true; 3203 } 3204 3205 return theExpansionOptions.getTheDisplayLanguage().equalsIgnoreCase(theStoredLang); 3206 } 3207 3208 @Nonnull 3209 private static String createMessageAppendForCodeNotFoundInCodeSystem(String theCodeSystemUrl) { 3210 return " - Code is not found in CodeSystem: " + theCodeSystemUrl; 3211 } 3212 3213 @VisibleForTesting 3214 public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) { 3215 ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest; 3216 } 3217 3218 static boolean isPlaceholder(DomainResource theResource) { 3219 boolean retVal = false; 3220 Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER); 3221 if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) { 3222 retVal = ((BooleanType) extension.getValue()).booleanValue(); 3223 } 3224 return retVal; 3225 } 3226 3227 /** 3228 * This is only used for unit tests to test failure conditions 3229 */ 3230 static void invokeRunnableForUnitTest() { 3231 if (myInvokeOnNextCallForUnitTest != null) { 3232 Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest; 3233 myInvokeOnNextCallForUnitTest = null; 3234 invokeOnNextCallForUnitTest.run(); 3235 } 3236 } 3237 3238 @VisibleForTesting 3239 public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) { 3240 myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest; 3241 } 3242 3243 static List<TermConcept> toPersistedConcepts( 3244 List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) { 3245 ArrayList<TermConcept> retVal = new ArrayList<>(); 3246 3247 for (CodeSystem.ConceptDefinitionComponent next : theConcept) { 3248 if (isNotBlank(next.getCode())) { 3249 TermConcept termConcept = toTermConcept(next, theCodeSystemVersion); 3250 retVal.add(termConcept); 3251 } 3252 } 3253 3254 return retVal; 3255 } 3256 3257 @Nonnull 3258 static TermConcept toTermConcept( 3259 CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) { 3260 TermConcept termConcept = new TermConcept(); 3261 termConcept.setCode(theConceptDefinition.getCode()); 3262 termConcept.setCodeSystemVersion(theCodeSystemVersion); 3263 termConcept.setDisplay(theConceptDefinition.getDisplay()); 3264 termConcept.addChildren( 3265 toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA); 3266 3267 for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent : 3268 theConceptDefinition.getDesignation()) { 3269 if (isNotBlank(designationComponent.getValue())) { 3270 TermConceptDesignation designation = termConcept.addDesignation(); 3271 designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null); 3272 if (designationComponent.hasUse()) { 3273 designation.setUseSystem( 3274 designationComponent.getUse().hasSystem() 3275 ? designationComponent.getUse().getSystem() 3276 : null); 3277 designation.setUseCode( 3278 designationComponent.getUse().hasCode() 3279 ? designationComponent.getUse().getCode() 3280 : null); 3281 designation.setUseDisplay( 3282 designationComponent.getUse().hasDisplay() 3283 ? designationComponent.getUse().getDisplay() 3284 : null); 3285 } 3286 designation.setValue(designationComponent.getValue()); 3287 } 3288 } 3289 3290 for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) { 3291 TermConceptProperty property = new TermConceptProperty(); 3292 3293 property.setKey(next.getCode()); 3294 property.setConcept(termConcept); 3295 property.setCodeSystemVersion(theCodeSystemVersion); 3296 3297 if (next.getValue() instanceof StringType) { 3298 property.setType(TermConceptPropertyTypeEnum.STRING); 3299 property.setValue(next.getValueStringType().getValue()); 3300 } else if (next.getValue() instanceof Coding) { 3301 Coding nextCoding = next.getValueCoding(); 3302 property.setType(TermConceptPropertyTypeEnum.CODING); 3303 property.setCodeSystem(nextCoding.getSystem()); 3304 property.setValue(nextCoding.getCode()); 3305 property.setDisplay(nextCoding.getDisplay()); 3306 } else if (next.getValue() != null) { 3307 // TODO: LOINC has properties of type BOOLEAN that we should handle 3308 ourLog.warn("Don't know how to handle properties of type: " 3309 + next.getValue().getClass()); 3310 continue; 3311 } 3312 3313 termConcept.getProperties().add(property); 3314 } 3315 return termConcept; 3316 } 3317 3318 static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) { 3319 // NOTE: return the designation when one of then is not specified. 3320 if (theReqLang == null || theStoredLang == null) return true; 3321 3322 return theReqLang.equalsIgnoreCase(theStoredLang); 3323 } 3324}