001/* 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.term; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.context.FhirVersionEnum; 024import ca.uhn.fhir.context.support.ConceptValidationOptions; 025import ca.uhn.fhir.context.support.IValidationSupport; 026import ca.uhn.fhir.context.support.LookupCodeRequest; 027import ca.uhn.fhir.context.support.ValidationSupportContext; 028import ca.uhn.fhir.context.support.ValueSetExpansionOptions; 029import ca.uhn.fhir.i18n.Msg; 030import ca.uhn.fhir.interceptor.model.RequestPartitionId; 031import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 032import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 033import ca.uhn.fhir.jpa.api.dao.IDao; 034import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem; 036import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 037import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode; 038import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider; 039import ca.uhn.fhir.jpa.config.util.ConnectionPoolInfoProvider; 040import ca.uhn.fhir.jpa.config.util.IConnectionPoolInfoProvider; 041import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; 042import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser; 043import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; 044import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; 045import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; 046import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao; 047import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao; 048import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao; 049import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao; 050import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao; 051import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao; 052import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao; 053import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; 054import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView; 055import ca.uhn.fhir.jpa.entity.TermCodeSystem; 056import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; 057import ca.uhn.fhir.jpa.entity.TermConcept; 058import ca.uhn.fhir.jpa.entity.TermConceptDesignation; 059import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; 060import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; 061import ca.uhn.fhir.jpa.entity.TermConceptProperty; 062import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum; 063import ca.uhn.fhir.jpa.entity.TermValueSet; 064import ca.uhn.fhir.jpa.entity.TermValueSetConcept; 065import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum; 066import ca.uhn.fhir.jpa.model.dao.JpaPid; 067import ca.uhn.fhir.jpa.model.entity.ResourceTable; 068import ca.uhn.fhir.jpa.model.sched.HapiJob; 069import ca.uhn.fhir.jpa.model.sched.IHasScheduledJobs; 070import ca.uhn.fhir.jpa.model.sched.ISchedulerService; 071import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition; 072import ca.uhn.fhir.jpa.model.util.JpaConstants; 073import ca.uhn.fhir.jpa.search.builder.SearchBuilder; 074import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; 075import ca.uhn.fhir.jpa.term.api.ITermReadSvc; 076import ca.uhn.fhir.jpa.term.api.ReindexTerminologyResult; 077import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException; 078import ca.uhn.fhir.rest.api.Constants; 079import ca.uhn.fhir.rest.api.server.RequestDetails; 080import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 081import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 082import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; 083import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; 084import ca.uhn.fhir.util.CoverageIgnore; 085import ca.uhn.fhir.util.FhirVersionIndependentConcept; 086import ca.uhn.fhir.util.HapiExtensions; 087import ca.uhn.fhir.util.StopWatch; 088import ca.uhn.fhir.util.UrlUtil; 089import ca.uhn.fhir.util.ValidateUtil; 090import ca.uhn.hapi.converters.canonical.VersionCanonicalizer; 091import com.google.common.annotations.VisibleForTesting; 092import com.google.common.base.Stopwatch; 093import com.google.common.collect.ArrayListMultimap; 094import jakarta.annotation.Nonnull; 095import jakarta.annotation.Nullable; 096import jakarta.annotation.PostConstruct; 097import jakarta.persistence.EntityManager; 098import jakarta.persistence.NonUniqueResultException; 099import jakarta.persistence.PersistenceContext; 100import jakarta.persistence.PersistenceContextType; 101import org.apache.commons.collections4.ListUtils; 102import org.apache.commons.lang3.ObjectUtils; 103import org.apache.commons.lang3.StringUtils; 104import org.apache.commons.lang3.Validate; 105import org.apache.commons.lang3.time.DateUtils; 106import org.apache.lucene.index.Term; 107import org.apache.lucene.search.IndexSearcher; 108import org.hibernate.CacheMode; 109import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep; 110import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep; 111import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; 112import org.hibernate.search.engine.search.query.SearchQuery; 113import org.hibernate.search.engine.search.query.SearchScroll; 114import org.hibernate.search.engine.search.query.SearchScrollResult; 115import org.hibernate.search.mapper.orm.Search; 116import org.hibernate.search.mapper.orm.common.EntityReference; 117import org.hibernate.search.mapper.orm.session.SearchSession; 118import org.hibernate.search.mapper.pojo.massindexing.impl.PojoMassIndexingLoggingMonitor; 119import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; 120import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50; 121import org.hl7.fhir.convertors.context.ConversionContext40_50; 122import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50; 123import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50; 124import org.hl7.fhir.instance.model.api.IAnyResource; 125import org.hl7.fhir.instance.model.api.IBaseCoding; 126import org.hl7.fhir.instance.model.api.IBaseDatatype; 127import org.hl7.fhir.instance.model.api.IBaseResource; 128import org.hl7.fhir.instance.model.api.IIdType; 129import org.hl7.fhir.instance.model.api.IPrimitiveType; 130import org.hl7.fhir.r4.model.BooleanType; 131import org.hl7.fhir.r4.model.CanonicalType; 132import org.hl7.fhir.r4.model.CodeSystem; 133import org.hl7.fhir.r4.model.CodeableConcept; 134import org.hl7.fhir.r4.model.Coding; 135import org.hl7.fhir.r4.model.DateTimeType; 136import org.hl7.fhir.r4.model.DecimalType; 137import org.hl7.fhir.r4.model.DomainResource; 138import org.hl7.fhir.r4.model.Enumerations; 139import org.hl7.fhir.r4.model.Extension; 140import org.hl7.fhir.r4.model.InstantType; 141import org.hl7.fhir.r4.model.IntegerType; 142import org.hl7.fhir.r4.model.StringType; 143import org.hl7.fhir.r4.model.ValueSet; 144import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome; 145import org.quartz.JobExecutionContext; 146import org.springframework.beans.factory.annotation.Autowired; 147import org.springframework.context.ApplicationContext; 148import org.springframework.data.domain.PageRequest; 149import org.springframework.data.domain.Pageable; 150import org.springframework.data.domain.Slice; 151import org.springframework.transaction.PlatformTransactionManager; 152import org.springframework.transaction.TransactionDefinition; 153import org.springframework.transaction.annotation.Propagation; 154import org.springframework.transaction.annotation.Transactional; 155import org.springframework.transaction.interceptor.NoRollbackRuleAttribute; 156import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute; 157import org.springframework.transaction.support.TransactionSynchronizationManager; 158import org.springframework.transaction.support.TransactionTemplate; 159import org.springframework.util.comparator.Comparators; 160 161import java.util.ArrayList; 162import java.util.Arrays; 163import java.util.Collection; 164import java.util.Collections; 165import java.util.Date; 166import java.util.HashMap; 167import java.util.HashSet; 168import java.util.LinkedHashMap; 169import java.util.List; 170import java.util.Map; 171import java.util.Objects; 172import java.util.Optional; 173import java.util.Set; 174import java.util.StringTokenizer; 175import java.util.UUID; 176import java.util.concurrent.TimeUnit; 177import java.util.function.Consumer; 178import java.util.function.Supplier; 179import java.util.stream.Collectors; 180 181import static ca.uhn.fhir.jpa.entity.TermConceptPropertyBinder.CONCEPT_PROPERTY_PREFIX_NAME; 182import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI; 183import static java.lang.String.join; 184import static java.util.stream.Collectors.joining; 185import static java.util.stream.Collectors.toList; 186import static java.util.stream.Collectors.toSet; 187import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; 188import static org.apache.commons.lang3.StringUtils.defaultString; 189import static org.apache.commons.lang3.StringUtils.isBlank; 190import static org.apache.commons.lang3.StringUtils.isEmpty; 191import static org.apache.commons.lang3.StringUtils.isNoneBlank; 192import static org.apache.commons.lang3.StringUtils.isNotBlank; 193import static org.apache.commons.lang3.StringUtils.lowerCase; 194import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase; 195 196public class TermReadSvcImpl implements ITermReadSvc, IHasScheduledJobs { 197 public static final int DEFAULT_FETCH_SIZE = 250; 198 public static final int DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS = 2; 199 // doesn't seem to be much gain by using more threads than this value 200 public static final int MAX_MASS_INDEXER_OBJECT_LOADING_THREADS = 6; 201 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TermReadSvcImpl.class); 202 private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions(); 203 private static final TermCodeSystemVersionDetails NO_CURRENT_VERSION = new TermCodeSystemVersionDetails(-1L, null); 204 private static final String OUR_PIPE_CHARACTER = "|"; 205 private static final int SECONDS_IN_MINUTE = 60; 206 private static final int INDEXED_ROOTS_LOGGING_COUNT = 50_000; 207 private static final String CS_USERDATA_CURRENT_VERSION = TermReadSvcImpl.class.getName() + "_CS_CURRENT_VERSION"; 208 private static final String VS_USERDATA_CURRENT_VERSION = TermReadSvcImpl.class.getName() + "_VS_CURRENT_VERSION"; 209 private static Runnable myInvokeOnNextCallForUnitTest; 210 private static boolean ourForceDisableHibernateSearchForUnitTest; 211 212 @Autowired 213 protected DaoRegistry myDaoRegistry; 214 215 @Autowired 216 protected ITermCodeSystemDao myCodeSystemDao; 217 218 @Autowired 219 protected ITermConceptDao myConceptDao; 220 221 @Autowired 222 protected ITermConceptPropertyDao myConceptPropertyDao; 223 224 @Autowired 225 protected ITermConceptDesignationDao myConceptDesignationDao; 226 227 @Autowired 228 protected ITermValueSetDao myTermValueSetDao; 229 230 @Autowired 231 protected ITermValueSetConceptDao myValueSetConceptDao; 232 233 @Autowired 234 protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao; 235 236 @Autowired 237 protected FhirContext myContext; 238 239 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 240 protected EntityManager myEntityManager; 241 242 private boolean myPreExpandingValueSets = false; 243 244 @Autowired 245 private ITermCodeSystemVersionDao myCodeSystemVersionDao; 246 247 @Autowired 248 private JpaStorageSettings myStorageSettings; 249 250 private TransactionTemplate myTxTemplate; 251 252 @Autowired 253 private PlatformTransactionManager myTransactionManager; 254 255 @Autowired(required = false) 256 private IFulltextSearchSvc myFulltextSearchSvc; 257 258 @Autowired 259 private PlatformTransactionManager myTxManager; 260 261 @Autowired 262 private ITermConceptDao myTermConceptDao; 263 264 @Autowired 265 private ITermValueSetConceptViewDao myTermValueSetConceptViewDao; 266 267 @Autowired 268 private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao; 269 270 @Autowired(required = false) 271 private ITermDeferredStorageSvc myDeferredStorageSvc; 272 273 @Autowired 274 private IIdHelperService<JpaPid> myIdHelperService; 275 276 @Autowired 277 private ApplicationContext myApplicationContext; 278 279 private volatile IValidationSupport myJpaValidationSupport; 280 private volatile IValidationSupport myValidationSupport; 281 // We need this bean so we can tell which mode hibernate search is running in. 282 @Autowired 283 private HibernatePropertiesProvider myHibernatePropertiesProvider; 284 285 @Autowired 286 private VersionCanonicalizer myVersionCanonicalizer; 287 288 @Autowired 289 private IJpaStorageResourceParser myJpaStorageResourceParser; 290 291 @Autowired 292 private InMemoryTerminologyServerValidationSupport myInMemoryTerminologyServerValidationSupport; 293 294 @Autowired 295 private ValueSetConceptAccumulatorFactory myValueSetConceptAccumulatorFactory; 296 297 @Override 298 public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) { 299 if (isBlank(theSystem)) { 300 return false; 301 } 302 TermCodeSystemVersionDetails cs = getCurrentCodeSystemVersion(theValidationSupportContext, theSystem); 303 return cs != null; 304 } 305 306 @Override 307 public boolean isValueSetSupported(ValidationSupportContext theValidationSupportContext, String theValueSetUrl) { 308 return fetchValueSet(theValueSetUrl) != null; 309 } 310 311 private boolean addCodeIfNotAlreadyAdded( 312 @Nullable ValueSetExpansionOptions theExpansionOptions, 313 IValueSetConceptAccumulator theValueSetCodeAccumulator, 314 Set<String> theAddedCodes, 315 TermConcept theConcept, 316 boolean theAdd, 317 String theValueSetIncludeVersion) { 318 String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); 319 String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId(); 320 String code = theConcept.getCode(); 321 String display = theConcept.getDisplay(); 322 Long sourceConceptPid = theConcept.getId(); 323 String directParentPids = ""; 324 325 if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) { 326 directParentPids = theConcept.getParents().stream() 327 .map(t -> t.getParent().getId().toString()) 328 .collect(joining(" ")); 329 } 330 331 Collection<TermConceptDesignation> designations = theConcept.getDesignations(); 332 if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) { 333 return addCodeIfNotAlreadyAdded( 334 theValueSetCodeAccumulator, 335 theAddedCodes, 336 designations, 337 theAdd, 338 codeSystem + OUR_PIPE_CHARACTER + theValueSetIncludeVersion, 339 code, 340 display, 341 sourceConceptPid, 342 directParentPids, 343 codeSystemVersion); 344 } else { 345 return addCodeIfNotAlreadyAdded( 346 theValueSetCodeAccumulator, 347 theAddedCodes, 348 designations, 349 theAdd, 350 codeSystem, 351 code, 352 display, 353 sourceConceptPid, 354 directParentPids, 355 codeSystemVersion); 356 } 357 } 358 359 private boolean addCodeIfNotAlreadyAdded( 360 IValueSetConceptAccumulator theValueSetCodeAccumulator, 361 Set<String> theAddedCodes, 362 boolean theAdd, 363 String theCodeSystem, 364 String theCodeSystemVersion, 365 String theCode, 366 String theDisplay, 367 Long theSourceConceptPid, 368 String theSourceConceptDirectParentPids, 369 Collection<TermConceptDesignation> theDesignations) { 370 if (StringUtils.isNotEmpty(theCodeSystemVersion)) { 371 if (isNoneBlank(theCodeSystem, theCode)) { 372 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 373 theValueSetCodeAccumulator.includeConceptWithDesignations( 374 theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion, 375 theCode, 376 theDisplay, 377 theDesignations, 378 theSourceConceptPid, 379 theSourceConceptDirectParentPids, 380 theCodeSystemVersion); 381 return true; 382 } 383 384 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 385 theValueSetCodeAccumulator.excludeConcept( 386 theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion, theCode); 387 return true; 388 } 389 } 390 } else { 391 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 392 theValueSetCodeAccumulator.includeConceptWithDesignations( 393 theCodeSystem, 394 theCode, 395 theDisplay, 396 theDesignations, 397 theSourceConceptPid, 398 theSourceConceptDirectParentPids, 399 theCodeSystemVersion); 400 return true; 401 } 402 403 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 404 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 405 return true; 406 } 407 } 408 409 return false; 410 } 411 412 private boolean addCodeIfNotAlreadyAdded( 413 IValueSetConceptAccumulator theValueSetCodeAccumulator, 414 Set<String> theAddedCodes, 415 Collection<TermConceptDesignation> theDesignations, 416 boolean theAdd, 417 String theCodeSystem, 418 String theCode, 419 String theDisplay, 420 Long theSourceConceptPid, 421 String theSourceConceptDirectParentPids, 422 String theSystemVersion) { 423 if (isNoneBlank(theCodeSystem, theCode)) { 424 if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 425 theValueSetCodeAccumulator.includeConceptWithDesignations( 426 theCodeSystem, 427 theCode, 428 theDisplay, 429 theDesignations, 430 theSourceConceptPid, 431 theSourceConceptDirectParentPids, 432 theSystemVersion); 433 return true; 434 } 435 436 if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) { 437 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 438 return true; 439 } 440 } 441 442 return false; 443 } 444 445 private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) { 446 boolean retVal = theSetToPopulate.add(theConcept); 447 if (retVal) { 448 if (theSetToPopulate.size() >= myStorageSettings.getMaximumExpansionSize()) { 449 String msg = myContext 450 .getLocalizer() 451 .getMessage( 452 TermReadSvcImpl.class, 453 "expansionTooLarge", 454 myStorageSettings.getMaximumExpansionSize()); 455 throw new ExpansionTooCostlyException(Msg.code(885) + msg); 456 } 457 } 458 return retVal; 459 } 460 461 public Optional<TermValueSet> deleteValueSetForResource(ResourceTable theResourceTable) { 462 // Get existing entity so it can be deleted. 463 Optional<TermValueSet> optionalExistingTermValueSetById = 464 myTermValueSetDao.findByResourcePid(theResourceTable.getId()); 465 466 if (optionalExistingTermValueSetById.isPresent()) { 467 TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get(); 468 469 ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId()); 470 deletePreCalculatedValueSetContents(existingTermValueSet); 471 myTermValueSetDao.deleteById(existingTermValueSet.getId()); 472 473 /* 474 * If we're updating an existing ValueSet within a transaction, we need to make 475 * sure to manually flush now since otherwise we'll try to create a new 476 * TermValueSet entity and fail with a constraint error on the URL, since 477 * this one won't be deleted yet 478 */ 479 myTermValueSetDao.flush(); 480 481 ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId()); 482 } 483 484 return optionalExistingTermValueSetById; 485 } 486 487 private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) { 488 myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId()); 489 myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId()); 490 } 491 492 @Override 493 @Transactional 494 public void deleteValueSetAndChildren(ResourceTable theResourceTable) { 495 deleteValueSetForResource(theResourceTable); 496 } 497 498 @Override 499 @Transactional 500 public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList( 501 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 502 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found 503 // in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may 504 // time-out. 505 506 ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl); 507 508 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 509 for (ValueSet.ValueSetExpansionContainsComponent nextContains : 510 expanded.getExpansion().getContains()) { 511 retVal.add(new FhirVersionIndependentConcept( 512 nextContains.getSystem(), 513 nextContains.getCode(), 514 nextContains.getDisplay(), 515 nextContains.getVersion())); 516 } 517 return retVal; 518 } 519 520 @Override 521 public ValueSet expandValueSet( 522 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 523 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl); 524 if (valueSet == null) { 525 throw new ResourceNotFoundException( 526 Msg.code(886) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl)); 527 } 528 529 return expandValueSet(theExpansionOptions, valueSet); 530 } 531 532 @Override 533 public ValueSet expandValueSet( 534 @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) { 535 String filter = null; 536 if (theExpansionOptions != null) { 537 filter = theExpansionOptions.getFilter(); 538 } 539 return doExpandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter)); 540 } 541 542 private ValueSet doExpandValueSet( 543 @Nullable ValueSetExpansionOptions theExpansionOptions, 544 ValueSet theValueSetToExpand, 545 ExpansionFilter theFilter) { 546 Set<String> addedCodes = new HashSet<>(); 547 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null"); 548 549 ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions); 550 int offset = expansionOptions.getOffset(); 551 int count = expansionOptions.getCount(); 552 553 ValueSetExpansionComponentWithConceptAccumulator accumulator = 554 new ValueSetExpansionComponentWithConceptAccumulator( 555 myContext, count, expansionOptions.isIncludeHierarchy()); 556 accumulator.setHardExpansionMaximumSize(myStorageSettings.getMaximumExpansionSize()); 557 accumulator.setSkipCountRemaining(offset); 558 accumulator.setIdentifier(UUID.randomUUID().toString()); 559 accumulator.setTimestamp(new Date()); 560 accumulator.setOffset(offset); 561 562 if (theExpansionOptions != null && isHibernateSearchEnabled()) { 563 accumulator.addParameter().setName("offset").setValue(new IntegerType(offset)); 564 accumulator.addParameter().setName("count").setValue(new IntegerType(count)); 565 } 566 567 myTxTemplate.executeWithoutResult(tx -> expandValueSetIntoAccumulator( 568 theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true, addedCodes)); 569 570 if (accumulator.getTotalConcepts() != null) { 571 accumulator.setTotal(accumulator.getTotalConcepts()); 572 } 573 574 ValueSet valueSet = new ValueSet(); 575 valueSet.setUrl(theValueSetToExpand.getUrl()); 576 valueSet.setId(theValueSetToExpand.getId()); 577 valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE); 578 valueSet.setCompose(theValueSetToExpand.getCompose()); 579 valueSet.setExpansion(accumulator); 580 581 for (String next : accumulator.getMessages()) { 582 valueSet.getMeta() 583 .addExtension() 584 .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE) 585 .setValue(new StringType(next)); 586 } 587 588 if (expansionOptions.isIncludeHierarchy()) { 589 accumulator.applyHierarchy(); 590 } 591 592 return valueSet; 593 } 594 595 private void expandValueSetIntoAccumulator( 596 ValueSet theValueSetToExpand, 597 ValueSetExpansionOptions theExpansionOptions, 598 IValueSetConceptAccumulator theAccumulator, 599 ExpansionFilter theFilter, 600 boolean theAdd, 601 Set<String> theAddedCodes) { 602 Optional<TermValueSet> optionalTermValueSet; 603 if (theValueSetToExpand.hasUrl()) { 604 if (theValueSetToExpand.hasVersion()) { 605 optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion( 606 theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion()); 607 } else { 608 optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl()); 609 } 610 } else { 611 optionalTermValueSet = Optional.empty(); 612 } 613 614 /* 615 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion 616 */ 617 if (optionalTermValueSet.isEmpty()) { 618 ourLog.debug( 619 "ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", 620 getValueSetInfo(theValueSetToExpand)); 621 String msg = myContext 622 .getLocalizer() 623 .getMessage( 624 TermReadSvcImpl.class, 625 "valueSetExpandedUsingInMemoryExpansion", 626 getValueSetInfo(theValueSetToExpand)); 627 theAccumulator.addMessage(msg); 628 doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes); 629 return; 630 } 631 632 /* 633 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion 634 */ 635 TermValueSet termValueSet = optionalTermValueSet.get(); 636 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 637 String msg = myContext 638 .getLocalizer() 639 .getMessage( 640 TermReadSvcImpl.class, 641 "valueSetNotYetExpanded", 642 getValueSetInfo(theValueSetToExpand), 643 termValueSet.getExpansionStatus().name(), 644 termValueSet.getExpansionStatus().getDescription()); 645 theAccumulator.addMessage(msg); 646 doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes); 647 return; 648 } 649 650 /* 651 * ValueSet is pre-expanded in database so let's use that 652 */ 653 String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet); 654 String msg = myContext 655 .getLocalizer() 656 .getMessage(TermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp); 657 theAccumulator.addMessage(msg); 658 expandConcepts( 659 theExpansionOptions, 660 theAccumulator, 661 termValueSet, 662 theFilter, 663 theAdd, 664 theAddedCodes, 665 myHibernatePropertiesProvider.isOracleDialect()); 666 } 667 668 @Nonnull 669 private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) { 670 String expansionTimestamp = "(unknown)"; 671 if (termValueSet.getExpansionTimestamp() != null) { 672 // Note: We used to append "123ms ago" to the timestamp, but we cache the 673 // results here, so it's just kind of weird to do that since the duration will 674 // be out of date when the entry comes back from cache 675 expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString(); 676 } 677 return expansionTimestamp; 678 } 679 680 private void expandConcepts( 681 ValueSetExpansionOptions theExpansionOptions, 682 IValueSetConceptAccumulator theAccumulator, 683 TermValueSet theTermValueSet, 684 ExpansionFilter theFilter, 685 boolean theAdd, 686 Set<String> theAddedCodes, 687 boolean theOracle) { 688 // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its 689 // copy pasted sibling 690 Integer offset = theAccumulator.getSkipCountRemaining(); 691 offset = ObjectUtils.defaultIfNull(offset, 0); 692 offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue()); 693 694 Integer count = theAccumulator.getCapacityRemaining(); 695 count = defaultIfNull(count, myStorageSettings.getMaximumExpansionSize()); 696 697 int conceptsExpanded = 0; 698 int designationsExpanded = 0; 699 int toIndex = offset + count; 700 701 Collection<? extends ITermValueSetConceptView> conceptViews; 702 boolean wasFilteredResult = false; 703 String filterDisplayValue = null; 704 if (!theFilter.getFilters().isEmpty() 705 && JpaConstants.VALUESET_FILTER_DISPLAY.equals( 706 theFilter.getFilters().get(0).getProperty()) 707 && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) { 708 filterDisplayValue = 709 lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]")); 710 String displayValue = "%" + lowerCase(filterDisplayValue) + "%"; 711 if (theOracle) { 712 conceptViews = 713 myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 714 } else { 715 conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 716 } 717 wasFilteredResult = true; 718 } else { 719 if (theOracle) { 720 conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId( 721 offset, toIndex, theTermValueSet.getId()); 722 } else { 723 conceptViews = 724 myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId()); 725 } 726 theAccumulator.consumeSkipCount(offset); 727 if (theAdd) { 728 theAccumulator.incrementOrDecrementTotalConcepts( 729 true, theTermValueSet.getTotalConcepts().intValue()); 730 } 731 } 732 733 if (conceptViews.isEmpty()) { 734 logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded); 735 return; 736 } 737 738 Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>(); 739 ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create(); 740 Map<Long, Long> pidToSourcePid = new HashMap<>(); 741 Map<Long, String> pidToSourceDirectParentPids = new HashMap<>(); 742 743 for (ITermValueSetConceptView conceptView : conceptViews) { 744 745 String system = conceptView.getConceptSystemUrl(); 746 String code = conceptView.getConceptCode(); 747 String display = conceptView.getConceptDisplay(); 748 String systemVersion = conceptView.getConceptSystemVersion(); 749 750 // -- this is quick solution, may need to revisit 751 if (!applyFilter(display, filterDisplayValue)) { 752 continue; 753 } 754 755 Long conceptPid = conceptView.getConceptPid(); 756 if (!pidToConcept.containsKey(conceptPid)) { 757 FhirVersionIndependentConcept concept = 758 new FhirVersionIndependentConcept(system, code, display, systemVersion); 759 pidToConcept.put(conceptPid, concept); 760 } 761 762 // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations 763 // optional. 764 if (conceptView.getDesignationPid() != null) { 765 TermConceptDesignation designation = new TermConceptDesignation(); 766 767 if (isValueSetDisplayLanguageMatch(theExpansionOptions, conceptView.getDesignationLang())) { 768 designation.setUseSystem(conceptView.getDesignationUseSystem()); 769 designation.setUseCode(conceptView.getDesignationUseCode()); 770 designation.setUseDisplay(conceptView.getDesignationUseDisplay()); 771 designation.setValue(conceptView.getDesignationVal()); 772 designation.setLanguage(conceptView.getDesignationLang()); 773 pidToDesignations.put(conceptPid, designation); 774 } 775 776 if (++designationsExpanded % 250 == 0) { 777 logDesignationsExpanded( 778 "Expansion of designations in progress. ", theTermValueSet, designationsExpanded); 779 } 780 } 781 782 if (theAccumulator.isTrackingHierarchy()) { 783 pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid()); 784 pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids()); 785 } 786 787 if (++conceptsExpanded % 250 == 0) { 788 logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded); 789 } 790 } 791 792 for (Long nextPid : pidToConcept.keySet()) { 793 FhirVersionIndependentConcept concept = pidToConcept.get(nextPid); 794 List<TermConceptDesignation> designations = pidToDesignations.get(nextPid); 795 String system = concept.getSystem(); 796 String code = concept.getCode(); 797 String display = concept.getDisplay(); 798 String systemVersion = concept.getSystemVersion(); 799 800 if (theAdd) { 801 if (theAccumulator.getCapacityRemaining() != null) { 802 if (theAccumulator.getCapacityRemaining() == 0) { 803 break; 804 } 805 } 806 807 Long sourceConceptPid = pidToSourcePid.get(nextPid); 808 String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid); 809 if (theAddedCodes.add(system + OUR_PIPE_CHARACTER + code)) { 810 theAccumulator.includeConceptWithDesignations( 811 system, 812 code, 813 display, 814 designations, 815 sourceConceptPid, 816 sourceConceptDirectParentPids, 817 systemVersion); 818 if (wasFilteredResult) { 819 theAccumulator.incrementOrDecrementTotalConcepts(true, 1); 820 } 821 } 822 } else { 823 if (theAddedCodes.remove(system + OUR_PIPE_CHARACTER + code)) { 824 theAccumulator.excludeConcept(system, code); 825 theAccumulator.incrementOrDecrementTotalConcepts(false, 1); 826 } 827 } 828 } 829 830 logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded); 831 logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded); 832 } 833 834 private void logConceptsExpanded( 835 String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) { 836 if (theConceptsExpanded > 0) { 837 ourLog.debug( 838 "{}Have expanded {} concepts in ValueSet[{}]", 839 theLogDescriptionPrefix, 840 theConceptsExpanded, 841 theTermValueSet.getUrl()); 842 } 843 } 844 845 private void logDesignationsExpanded( 846 String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) { 847 if (theDesignationsExpanded > 0) { 848 ourLog.debug( 849 "{}Have expanded {} designations in ValueSet[{}]", 850 theLogDescriptionPrefix, 851 theDesignationsExpanded, 852 theTermValueSet.getUrl()); 853 } 854 } 855 856 public boolean applyFilter(final String theDisplay, final String theFilterDisplay) { 857 858 // -- safety check only, no need to apply filter 859 if (theDisplay == null || theFilterDisplay == null) return true; 860 861 // -- sentence case 862 if (startsWithIgnoreCase(theDisplay, theFilterDisplay)) return true; 863 864 // -- token case 865 return startsWithByWordBoundaries(theDisplay, theFilterDisplay); 866 } 867 868 private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) { 869 // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo' 870 StringTokenizer tok = new StringTokenizer(theDisplay); 871 List<String> tokens = new ArrayList<>(); 872 while (tok.hasMoreTokens()) { 873 String token = tok.nextToken(); 874 if (startsWithIgnoreCase(token, theFilterDisplay)) return true; 875 tokens.add(token); 876 } 877 878 // Allow to search by the end of the phrase. E.g. "working proficiency" will match "Limited working 879 // proficiency" 880 for (int start = 0; start <= tokens.size() - 1; ++start) { 881 for (int end = start + 1; end <= tokens.size(); ++end) { 882 String sublist = String.join(" ", tokens.subList(start, end)); 883 if (startsWithIgnoreCase(sublist, theFilterDisplay)) return true; 884 } 885 } 886 return false; 887 } 888 889 @Override 890 public void expandValueSet( 891 ValueSetExpansionOptions theExpansionOptions, 892 ValueSet theValueSetToExpand, 893 IValueSetConceptAccumulator theValueSetCodeAccumulator) { 894 Set<String> addedCodes = new HashSet<>(); 895 doExpandValueSet( 896 theExpansionOptions, 897 theValueSetToExpand, 898 theValueSetCodeAccumulator, 899 ExpansionFilter.NO_FILTER, 900 addedCodes); 901 } 902 903 /** 904 * Note: Not transactional because specific calls within this method 905 * get executed in a transaction 906 */ 907 private void doExpandValueSet( 908 ValueSetExpansionOptions theExpansionOptions, 909 ValueSet theValueSetToExpand, 910 IValueSetConceptAccumulator theValueSetCodeAccumulator, 911 @Nonnull ExpansionFilter theExpansionFilter, 912 Set<String> theAddedCodes) { 913 914 StopWatch sw = new StopWatch(); 915 String valueSetInfo = getValueSetInfo(theValueSetToExpand); 916 ourLog.debug("Working with {}", valueSetInfo); 917 918 // Offset can't be combined with excludes 919 Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining(); 920 if (skipCountRemaining != null && skipCountRemaining > 0) { 921 if (!theValueSetToExpand.getCompose().getExclude().isEmpty()) { 922 String msg = myContext 923 .getLocalizer() 924 .getMessage(TermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo); 925 throw new InvalidRequestException(Msg.code(887) + msg); 926 } 927 } 928 929 // Handle includes 930 ourLog.debug("Handling includes"); 931 for (ValueSet.ConceptSetComponent include : 932 theValueSetToExpand.getCompose().getInclude()) { 933 myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude( 934 theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, include, true, theExpansionFilter)); 935 } 936 937 // Handle excludes 938 ourLog.debug("Handling excludes"); 939 for (ValueSet.ConceptSetComponent exclude : 940 theValueSetToExpand.getCompose().getExclude()) { 941 myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude( 942 theExpansionOptions, 943 theValueSetCodeAccumulator, 944 theAddedCodes, 945 exclude, 946 false, 947 ExpansionFilter.NO_FILTER)); 948 } 949 950 if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) { 951 myTxTemplate.execute( 952 t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder()); 953 } 954 955 ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis()); 956 } 957 958 private String getValueSetInfo(ValueSet theValueSet) { 959 StringBuilder sb = new StringBuilder(); 960 boolean isIdentified = false; 961 if (theValueSet.hasUrl()) { 962 isIdentified = true; 963 sb.append("ValueSet.url[").append(theValueSet.getUrl()).append("]"); 964 } else if (theValueSet.hasId()) { 965 isIdentified = true; 966 sb.append("ValueSet.id[").append(theValueSet.getId()).append("]"); 967 } 968 969 if (!isIdentified) { 970 sb.append("Unidentified ValueSet"); 971 } 972 973 return sb.toString(); 974 } 975 976 /** 977 * Returns true if there are potentially more results to process. 978 */ 979 private void expandValueSetHandleIncludeOrExclude( 980 @Nullable ValueSetExpansionOptions theExpansionOptions, 981 IValueSetConceptAccumulator theValueSetCodeAccumulator, 982 Set<String> theAddedCodes, 983 ValueSet.ConceptSetComponent theIncludeOrExclude, 984 boolean theAdd, 985 @Nonnull ExpansionFilter theExpansionFilter) { 986 987 String system = theIncludeOrExclude.getSystem(); 988 boolean hasSystem = isNotBlank(system); 989 boolean hasValueSet = !theIncludeOrExclude.getValueSet().isEmpty(); 990 991 if (hasSystem) { 992 993 if (theExpansionFilter.hasCode() 994 && theExpansionFilter.getSystem() != null 995 && !system.equals(theExpansionFilter.getSystem())) { 996 return; 997 } 998 999 ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system); 1000 1001 Optional<TermCodeSystemVersion> termCodeSystemVersion = 1002 optionalFindTermCodeSystemVersion(theIncludeOrExclude); 1003 if (termCodeSystemVersion.isPresent()) { 1004 1005 expandValueSetHandleIncludeOrExcludeUsingDatabase( 1006 theExpansionOptions, 1007 theValueSetCodeAccumulator, 1008 theAddedCodes, 1009 theIncludeOrExclude, 1010 theAdd, 1011 theExpansionFilter, 1012 system, 1013 termCodeSystemVersion.get()); 1014 1015 } else { 1016 1017 if (!theIncludeOrExclude.getConcept().isEmpty() && theExpansionFilter.hasCode()) { 1018 if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) { 1019 if (theIncludeOrExclude.getConcept().stream() 1020 .noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) { 1021 return; 1022 } 1023 } 1024 } 1025 1026 Consumer<FhirVersionIndependentConcept> consumer = c -> addOrRemoveCode( 1027 theValueSetCodeAccumulator, 1028 theAddedCodes, 1029 theAdd, 1030 system, 1031 c.getCode(), 1032 c.getDisplay(), 1033 c.getSystemVersion()); 1034 1035 try { 1036 ConversionContext40_50.INSTANCE.init( 1037 new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet"); 1038 org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude = 1039 ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude); 1040 myInMemoryTerminologyServerValidationSupport.expandValueSetIncludeOrExclude( 1041 new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude); 1042 } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) { 1043 if (theExpansionOptions != null 1044 && !theExpansionOptions.isFailOnMissingCodeSystem() 1045 // Code system is unknown, therefore NOT_FOUND 1046 && e.getCodeValidationIssue() 1047 .hasIssueDetailCode(CodeValidationIssueCoding.NOT_FOUND.getCode())) { 1048 return; 1049 } 1050 throw new InternalErrorException(Msg.code(888) + e); 1051 } finally { 1052 ConversionContext40_50.INSTANCE.close("ValueSet"); 1053 } 1054 } 1055 1056 } else if (hasValueSet) { 1057 1058 for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) { 1059 String valueSetUrl = nextValueSet.getValueAsString(); 1060 ourLog.debug( 1061 "Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl); 1062 1063 ExpansionFilter subExpansionFilter = new ExpansionFilter( 1064 theExpansionFilter, 1065 theIncludeOrExclude.getFilter(), 1066 theValueSetCodeAccumulator.getCapacityRemaining()); 1067 1068 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not 1069 // be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the 1070 // expansion may time-out. 1071 1072 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl); 1073 if (valueSet == null) { 1074 throw new ResourceNotFoundException( 1075 Msg.code(889) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl)); 1076 } 1077 1078 expandValueSetIntoAccumulator( 1079 valueSet, 1080 theExpansionOptions, 1081 theValueSetCodeAccumulator, 1082 subExpansionFilter, 1083 theAdd, 1084 theAddedCodes); 1085 } 1086 1087 } else { 1088 throw new InvalidRequestException(Msg.code(890) + "ValueSet contains " + (theAdd ? "include" : "exclude") 1089 + " criteria with no system defined"); 1090 } 1091 } 1092 1093 private Optional<TermCodeSystemVersion> optionalFindTermCodeSystemVersion( 1094 ValueSet.ConceptSetComponent theIncludeOrExclude) { 1095 if (isEmpty(theIncludeOrExclude.getVersion())) { 1096 return Optional.ofNullable(myCodeSystemDao.findByCodeSystemUri(theIncludeOrExclude.getSystem())) 1097 .map(TermCodeSystem::getCurrentVersion); 1098 } else { 1099 return Optional.ofNullable(myCodeSystemVersionDao.findByCodeSystemUriAndVersion( 1100 theIncludeOrExclude.getSystem(), theIncludeOrExclude.getVersion())); 1101 } 1102 } 1103 1104 private boolean isHibernateSearchEnabled() { 1105 return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest; 1106 } 1107 1108 private void expandValueSetHandleIncludeOrExcludeUsingDatabase( 1109 ValueSetExpansionOptions theExpansionOptions, 1110 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1111 Set<String> theAddedCodes, 1112 ValueSet.ConceptSetComponent theIncludeOrExclude, 1113 boolean theAdd, 1114 @Nonnull ExpansionFilter theExpansionFilter, 1115 String theSystem, 1116 TermCodeSystemVersion theTermCodeSystemVersion) { 1117 1118 StopWatch fullOperationSw = new StopWatch(); 1119 String includeOrExcludeVersion = theIncludeOrExclude.getVersion(); 1120 1121 /* 1122 * If FullText searching is not enabled, we can handle only basic expansions 1123 * since we're going to do it without the database. 1124 */ 1125 if (!isHibernateSearchEnabled()) { 1126 expandWithoutHibernateSearch( 1127 theValueSetCodeAccumulator, 1128 theTermCodeSystemVersion, 1129 theAddedCodes, 1130 theIncludeOrExclude, 1131 theSystem, 1132 theAdd); 1133 return; 1134 } 1135 1136 /* 1137 * Ok, let's use hibernate search to build the expansion 1138 */ 1139 1140 int count = 0; 1141 1142 Optional<Integer> chunkSizeOpt = getScrollChunkSize(theAdd, theValueSetCodeAccumulator); 1143 if (chunkSizeOpt.isEmpty()) { 1144 return; 1145 } 1146 int chunkSize = chunkSizeOpt.get(); 1147 1148 /* 1149 * Turn the filter into one or more Hibernate Search queries. Ideally we want it 1150 * to be handled by a single query, but Lucene/ES don't like it when we exceed 1151 * 1024 different terms in a single query. So if we have that many terms (which 1152 * can happen if a ValueSet has a lot of explicitly enumerated codes that it's 1153 * including) we split this into multiple searches. The method below builds these 1154 * searches lazily, returning a Supplier that creates and executes the search 1155 * when it's actually time to. 1156 */ 1157 SearchProperties searchProps = buildSearchScrolls( 1158 theTermCodeSystemVersion, 1159 theExpansionFilter, 1160 theSystem, 1161 theIncludeOrExclude, 1162 chunkSize, 1163 includeOrExcludeVersion); 1164 int accumulatedBatchesSoFar = 0; 1165 for (var next : searchProps.getSearchScroll()) { 1166 try (SearchScroll<EntityReference> scroll = next.get()) { 1167 ourLog.debug( 1168 "Beginning batch expansion for {} with max results per batch: {}", 1169 (theAdd ? "inclusion" : "exclusion"), 1170 chunkSize); 1171 for (SearchScrollResult<EntityReference> chunk = scroll.next(); 1172 chunk.hasHits(); 1173 chunk = scroll.next()) { 1174 int countForBatch = 0; 1175 1176 List<Long> pids = 1177 chunk.hits().stream().map(t -> (Long) t.id()).collect(Collectors.toList()); 1178 1179 List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids); 1180 1181 // If the include section had multiple codes, return the codes in the same order 1182 termConcepts = sortTermConcepts(searchProps, termConcepts); 1183 1184 // int firstResult = theQueryIndex * maxResultsPerBatch; 1185 // TODO GGG HS we lose the ability to check the 1186 // index of the first result, so just best-guessing it here. 1187 int delta = 0; 1188 for (TermConcept concept : termConcepts) { 1189 count++; 1190 countForBatch++; 1191 if (theAdd && searchProps.hasIncludeOrExcludeCodes()) { 1192 ValueSet.ConceptReferenceComponent theIncludeConcept = 1193 getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept); 1194 if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) { 1195 concept.setDisplay(theIncludeConcept.getDisplay()); 1196 } 1197 } 1198 boolean added = addCodeIfNotAlreadyAdded( 1199 theExpansionOptions, 1200 theValueSetCodeAccumulator, 1201 theAddedCodes, 1202 concept, 1203 theAdd, 1204 includeOrExcludeVersion); 1205 if (added) { 1206 delta++; 1207 } 1208 } 1209 1210 ourLog.debug( 1211 "Batch expansion scroll for {} with offset {} produced {} results in {}ms", 1212 (theAdd ? "inclusion" : "exclusion"), 1213 accumulatedBatchesSoFar, 1214 chunk.hits().size(), 1215 chunk.took().toMillis()); 1216 1217 theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta); 1218 accumulatedBatchesSoFar += countForBatch; 1219 1220 // keep session bounded 1221 myEntityManager.flush(); 1222 myEntityManager.clear(); 1223 } 1224 1225 ourLog.debug( 1226 "Expansion for {} produced {} results in {}ms", 1227 (theAdd ? "inclusion" : "exclusion"), 1228 count, 1229 fullOperationSw.getMillis()); 1230 } 1231 } 1232 } 1233 1234 private List<TermConcept> sortTermConcepts(SearchProperties searchProps, List<TermConcept> termConcepts) { 1235 List<String> codes = searchProps.getIncludeOrExcludeCodes(); 1236 if (codes.size() > 1) { 1237 termConcepts = new ArrayList<>(termConcepts); 1238 Map<String, Integer> codeToIndex = new HashMap<>(codes.size()); 1239 for (int i = 0; i < codes.size(); i++) { 1240 codeToIndex.put(codes.get(i), i); 1241 } 1242 termConcepts.sort(((o1, o2) -> { 1243 Integer idx1 = codeToIndex.get(o1.getCode()); 1244 Integer idx2 = codeToIndex.get(o2.getCode()); 1245 return Comparators.nullsHigh().compare(idx1, idx2); 1246 })); 1247 } 1248 return termConcepts; 1249 } 1250 1251 private Optional<Integer> getScrollChunkSize( 1252 boolean theAdd, IValueSetConceptAccumulator theValueSetCodeAccumulator) { 1253 int maxResultsPerBatch = SearchBuilder.getMaximumPageSize(); 1254 1255 /* 1256 * If the accumulator is bounded, we may reduce the size of the query to 1257 * Lucene in order to be more efficient. 1258 */ 1259 if (theAdd) { 1260 Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining(); 1261 if (accumulatorCapacityRemaining != null) { 1262 maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1); 1263 } 1264 } 1265 return maxResultsPerBatch > 0 ? Optional.of(maxResultsPerBatch) : Optional.empty(); 1266 } 1267 1268 private SearchProperties buildSearchScrolls( 1269 TermCodeSystemVersion theTermCodeSystemVersion, 1270 ExpansionFilter theExpansionFilter, 1271 String theSystem, 1272 ValueSet.ConceptSetComponent theIncludeOrExclude, 1273 Integer theScrollChunkSize, 1274 String theIncludeOrExcludeVersion) { 1275 SearchSession searchSession = Search.session(myEntityManager); 1276 // Manually building a predicate since we need to throw it around. 1277 SearchPredicateFactory predicate = 1278 searchSession.scope(TermConcept.class).predicate(); 1279 1280 List<String> allCodes = theIncludeOrExclude.getConcept().stream() 1281 .filter(Objects::nonNull) 1282 .map(ValueSet.ConceptReferenceComponent::getCode) 1283 .filter(StringUtils::isNotBlank) 1284 .collect(Collectors.toList()); 1285 SearchProperties returnProps = new SearchProperties(); 1286 returnProps.setIncludeOrExcludeCodes(allCodes); 1287 1288 /* 1289 * Lucene/ES can't typically handle more than 1024 clauses per search, so if 1290 * we have more than that number (e.g. because of a ValueSet that explicitly 1291 * includes thousands of codes), we break this up into multiple searches. 1292 */ 1293 List<List<String>> partitionedCodes = ListUtils.partition(allCodes, IndexSearcher.getMaxClauseCount() - 10); 1294 if (partitionedCodes.isEmpty()) { 1295 partitionedCodes = List.of(List.of()); 1296 } 1297 1298 for (List<String> nextCodePartition : partitionedCodes) { 1299 Supplier<SearchScroll<EntityReference>> nextScroll = () -> { 1300 // Build the top-level expansion on filters. 1301 PredicateFinalStep step = predicate.bool(b -> { 1302 b.must(predicate 1303 .match() 1304 .field("myCodeSystemVersionPid") 1305 .matching(theTermCodeSystemVersion.getPid())); 1306 1307 if (theExpansionFilter.hasCode()) { 1308 b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode())); 1309 } 1310 1311 String codeSystemUrlAndVersion = 1312 buildCodeSystemUrlAndVersion(theSystem, theIncludeOrExcludeVersion); 1313 for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) { 1314 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 1315 } 1316 for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) { 1317 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 1318 } 1319 }); 1320 1321 // Add a selector on any explicitly enumerated codes in the VS component 1322 final PredicateFinalStep finishedQuery; 1323 if (nextCodePartition.isEmpty()) { 1324 finishedQuery = step; 1325 } else { 1326 PredicateFinalStep expansionStep = buildExpansionPredicate(nextCodePartition, predicate); 1327 finishedQuery = predicate.bool().must(step).must(expansionStep); 1328 } 1329 1330 SearchQuery<EntityReference> termConceptsQuery = searchSession 1331 .search(TermConcept.class) 1332 .selectEntityReference() 1333 .where(f -> finishedQuery) 1334 .toQuery(); 1335 1336 return termConceptsQuery.scroll(theScrollChunkSize); 1337 }; 1338 1339 returnProps.addSearchScroll(nextScroll); 1340 } 1341 1342 return returnProps; 1343 } 1344 1345 private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet( 1346 ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) { 1347 return theIncludeOrExclude.getConcept().stream() 1348 .filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode())) 1349 .findFirst() 1350 .orElse(null); 1351 } 1352 1353 /** 1354 * Helper method which builds a predicate for the expansion 1355 */ 1356 private PredicateFinalStep buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) { 1357 assert !theCodes.isEmpty(); 1358 return thePredicate.simpleQueryString().field("myCode").matching(String.join(" | ", theCodes)); 1359 } 1360 1361 private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) { 1362 String codeSystemUrlAndVersion; 1363 if (theIncludeOrExcludeVersion != null) { 1364 codeSystemUrlAndVersion = theSystem + OUR_PIPE_CHARACTER + theIncludeOrExcludeVersion; 1365 } else { 1366 codeSystemUrlAndVersion = theSystem; 1367 } 1368 return codeSystemUrlAndVersion; 1369 } 1370 1371 private @Nonnull ValueSetExpansionOptions provideExpansionOptions( 1372 @Nullable ValueSetExpansionOptions theExpansionOptions) { 1373 return Objects.requireNonNullElse(theExpansionOptions, DEFAULT_EXPANSION_OPTIONS); 1374 } 1375 1376 private void addOrRemoveCode( 1377 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1378 Set<String> theAddedCodes, 1379 boolean theAdd, 1380 String theSystem, 1381 String theCode, 1382 String theDisplay, 1383 String theSystemVersion) { 1384 if (theAdd && theAddedCodes.add(theSystem + OUR_PIPE_CHARACTER + theCode)) { 1385 theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion); 1386 } 1387 if (!theAdd && theAddedCodes.remove(theSystem + OUR_PIPE_CHARACTER + theCode)) { 1388 theValueSetCodeAccumulator.excludeConcept(theSystem, theCode); 1389 } 1390 } 1391 1392 private void handleFilter( 1393 String theCodeSystemIdentifier, 1394 SearchPredicateFactory theF, 1395 BooleanPredicateClausesStep<?> theB, 1396 ValueSet.ConceptSetFilterComponent theFilter) { 1397 if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) { 1398 return; 1399 } 1400 1401 // if filter type is EXISTS, there's no reason to worry about the value (we won't set it anyways) 1402 if ((isBlank(theFilter.getValue()) && theFilter.getOp() != ValueSet.FilterOperator.EXISTS) 1403 || theFilter.getOp() == null 1404 || isBlank(theFilter.getProperty())) { 1405 throw new InvalidRequestException( 1406 Msg.code(891) + "Invalid filter, must have fields populated: property op value"); 1407 } 1408 1409 switch (theFilter.getProperty()) { 1410 case "display:exact": 1411 case "display": 1412 handleFilterDisplay(theF, theB, theFilter); 1413 break; 1414 case "concept": 1415 case "code": 1416 handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter); 1417 break; 1418 case "parent": 1419 case "child": 1420 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1421 handleFilterLoincParentChild(theF, theB, theFilter); 1422 break; 1423 case "ancestor": 1424 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1425 handleFilterLoincAncestor(theCodeSystemIdentifier, theF, theB, theFilter); 1426 break; 1427 case "descendant": 1428 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1429 handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter); 1430 break; 1431 case "copyright": 1432 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1433 handleFilterLoincCopyright(theF, theB, theFilter); 1434 break; 1435 default: 1436 if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) { 1437 handleFilterRegex(theF, theB, theFilter); 1438 } else { 1439 handleFilterPropertyDefault(theF, theB, theFilter); 1440 } 1441 break; 1442 } 1443 } 1444 1445 private void handleFilterPropertyDefault( 1446 SearchPredicateFactory theF, 1447 BooleanPredicateClausesStep<?> theB, 1448 ValueSet.ConceptSetFilterComponent theFilter) { 1449 1450 String value = theFilter.getValue(); 1451 if (theFilter.getOp() == ValueSet.FilterOperator.EXISTS) { 1452 // EXISTS has no value and is thus handled differently 1453 Term term = new Term(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty()); 1454 theB.must(theF.exists().field(term.field())); 1455 } else { 1456 Term term = new Term(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty(), value); 1457 switch (theFilter.getOp()) { 1458 case EQUAL: 1459 theB.must(theF.match().field(term.field()).matching(term.text())); 1460 break; 1461 case IN: 1462 case NOTIN: 1463 boolean isNotFilter = theFilter.getOp() == ValueSet.FilterOperator.NOTIN; 1464 // IN and NOTIN expect comma separated lists 1465 String[] values = term.text().split(","); 1466 Set<String> valueSet = new HashSet<>(Arrays.asList(values)); 1467 if (isNotFilter) { 1468 theB.filter(theF.not(theF.terms().field(term.field()).matchingAny(valueSet))); 1469 } else { 1470 theB.filter(theF.terms().field(term.field()).matchingAny(valueSet)); 1471 } 1472 break; 1473 case ISA: 1474 case ISNOTA: 1475 case DESCENDENTOF: 1476 case GENERALIZES: 1477 default: 1478 /* 1479 * We do not need to handle REGEX, because that's handled in parent 1480 * We also don't handle EXISTS because that's a separate area (with different term). 1481 * We add a match-none filter because otherwise it matches everything (not desired). 1482 */ 1483 ourLog.error( 1484 "Unsupported property filter {}. This may affect expansion, but will not cause errors.", 1485 theFilter.getOp().getDisplay()); 1486 theB.must(theF.matchNone()); 1487 break; 1488 } 1489 } 1490 } 1491 1492 private void handleFilterRegex( 1493 SearchPredicateFactory theF, 1494 BooleanPredicateClausesStep<?> theB, 1495 ValueSet.ConceptSetFilterComponent theFilter) { 1496 /* 1497 * We treat the regex filter as a match on the regex 1498 * anywhere in the property string. The spec does not 1499 * say whether this is the right behaviour or not, but 1500 * there are examples that seem to suggest that it is. 1501 */ 1502 String value = theFilter.getValue(); 1503 if (value.endsWith("$")) { 1504 value = value.substring(0, value.length() - 1); 1505 } else if (!value.endsWith(".*")) { 1506 value = value + ".*"; 1507 } 1508 if (!value.startsWith("^") && !value.startsWith(".*")) { 1509 value = ".*" + value; 1510 } else if (value.startsWith("^")) { 1511 value = value.substring(1); 1512 } 1513 1514 theB.must(theF.regexp() 1515 .field(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty()) 1516 .matching(value)); 1517 } 1518 1519 private void handleFilterLoincCopyright( 1520 SearchPredicateFactory theF, 1521 BooleanPredicateClausesStep<?> theB, 1522 ValueSet.ConceptSetFilterComponent theFilter) { 1523 1524 if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1525 1526 String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase(); 1527 switch (copyrightFilterValue) { 1528 case "3rdparty": 1529 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1530 addFilterLoincCopyright3rdParty(theF, theB); 1531 break; 1532 case "loinc": 1533 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1534 addFilterLoincCopyrightLoinc(theF, theB); 1535 break; 1536 default: 1537 throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1538 } 1539 1540 } else { 1541 throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty()); 1542 } 1543 } 1544 1545 private void addFilterLoincCopyrightLoinc(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) { 1546 theB.mustNot(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE")); 1547 } 1548 1549 private void addFilterLoincCopyright3rdParty(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) { 1550 theB.must(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE")); 1551 } 1552 1553 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1554 private void handleFilterLoincAncestor( 1555 String theSystem, 1556 SearchPredicateFactory f, 1557 BooleanPredicateClausesStep<?> b, 1558 ValueSet.ConceptSetFilterComponent theFilter) { 1559 switch (theFilter.getOp()) { 1560 case EQUAL: 1561 addLoincFilterAncestorEqual(theSystem, f, b, theFilter); 1562 break; 1563 case IN: 1564 addLoincFilterAncestorIn(theSystem, f, b, theFilter); 1565 break; 1566 default: 1567 throw new InvalidRequestException(Msg.code(892) + "Don't know how to handle op=" + theFilter.getOp() 1568 + " on property " + theFilter.getProperty()); 1569 } 1570 } 1571 1572 private void addLoincFilterAncestorEqual( 1573 String theSystem, 1574 SearchPredicateFactory f, 1575 BooleanPredicateClausesStep<?> b, 1576 ValueSet.ConceptSetFilterComponent theFilter) { 1577 addLoincFilterAncestorEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue()); 1578 } 1579 1580 private void addLoincFilterAncestorEqual( 1581 String theSystem, 1582 SearchPredicateFactory f, 1583 BooleanPredicateClausesStep<?> b, 1584 String theProperty, 1585 String theValue) { 1586 List<Term> terms = getAncestorTerms(theSystem, theProperty, theValue); 1587 b.must(f.bool(innerB -> terms.forEach( 1588 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1589 } 1590 1591 private void addLoincFilterAncestorIn( 1592 String theSystem, 1593 SearchPredicateFactory f, 1594 BooleanPredicateClausesStep<?> b, 1595 ValueSet.ConceptSetFilterComponent theFilter) { 1596 String[] values = theFilter.getValue().split(","); 1597 List<Term> terms = new ArrayList<>(); 1598 for (String value : values) { 1599 terms.addAll(getAncestorTerms(theSystem, theFilter.getProperty(), value)); 1600 } 1601 b.must(f.bool(innerB -> terms.forEach( 1602 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1603 } 1604 1605 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1606 private void handleFilterLoincParentChild( 1607 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1608 switch (theFilter.getOp()) { 1609 case EQUAL: 1610 addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue()); 1611 break; 1612 case IN: 1613 addLoincFilterParentChildIn(f, b, theFilter); 1614 break; 1615 default: 1616 throw new InvalidRequestException(Msg.code(893) + "Don't know how to handle op=" + theFilter.getOp() 1617 + " on property " + theFilter.getProperty()); 1618 } 1619 } 1620 1621 private void addLoincFilterParentChildIn( 1622 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1623 String[] values = theFilter.getValue().split(","); 1624 List<Term> terms = new ArrayList<>(); 1625 for (String value : values) { 1626 logFilteringValueOnProperty(value, theFilter.getProperty()); 1627 terms.add(getPropertyTerm(theFilter.getProperty(), value)); 1628 } 1629 1630 b.must(f.bool(innerB -> terms.forEach( 1631 term -> innerB.should(f.match().field(term.field()).matching(term.text()))))); 1632 } 1633 1634 private void addLoincFilterParentChildEqual( 1635 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) { 1636 logFilteringValueOnProperty(theValue, theProperty); 1637 b.must(f.match().field(CONCEPT_PROPERTY_PREFIX_NAME + theProperty).matching(theValue)); 1638 } 1639 1640 private void handleFilterConceptAndCode( 1641 String theSystem, 1642 SearchPredicateFactory f, 1643 BooleanPredicateClausesStep<?> b, 1644 ValueSet.ConceptSetFilterComponent theFilter) { 1645 TermConcept code = findCodeForFilterCriteriaCodeOrConcept(theSystem, theFilter); 1646 1647 if (theFilter.getOp() == ValueSet.FilterOperator.ISA) { 1648 ourLog.debug( 1649 " * Filtering on specific code and codes with a parent of {}/{}/{}", 1650 code.getId(), 1651 code.getCode(), 1652 code.getDisplay()); 1653 1654 b.must(f.bool() 1655 .should(f.match().field("myParentPids").matching("" + code.getId())) 1656 .should(f.match().field("myId").matching(code.getId()))); 1657 } else if (theFilter.getOp() == ValueSet.FilterOperator.DESCENDENTOF) { 1658 ourLog.debug( 1659 " * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay()); 1660 1661 b.must(f.match().field("myParentPids").matching("" + code.getId())); 1662 } else { 1663 throwInvalidFilter(theFilter, ""); 1664 } 1665 } 1666 1667 @Nonnull 1668 private TermConcept findCodeForFilterCriteriaCodeOrConcept( 1669 String theSystem, ValueSet.ConceptSetFilterComponent theFilter) { 1670 return findCode(theSystem, theFilter.getValue()) 1671 .orElseThrow(() -> 1672 new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" 1673 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue())); 1674 } 1675 1676 private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) { 1677 throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() 1678 + " on property " + theFilter.getProperty() + theErrorSuffix); 1679 } 1680 1681 private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) { 1682 String systemUrl = getUrlFromIdentifier(theSystemIdentifier); 1683 if (!isCodeSystemLoinc(systemUrl)) { 1684 throw new InvalidRequestException(Msg.code(895) + "Invalid filter, property " + theProperty 1685 + " is LOINC-specific and cannot be used with system: " + systemUrl); 1686 } 1687 } 1688 1689 private boolean isCodeSystemLoinc(String theSystem) { 1690 return LOINC_URI.equals(theSystem); 1691 } 1692 1693 private void handleFilterDisplay( 1694 SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1695 if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1696 addDisplayFilterExact(f, b, theFilter); 1697 } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1698 if (theFilter.getValue().trim().contains(" ")) { 1699 addDisplayFilterExact(f, b, theFilter); 1700 } else { 1701 addDisplayFilterInexact(f, b, theFilter); 1702 } 1703 } 1704 } 1705 1706 private void addDisplayFilterExact( 1707 SearchPredicateFactory f, 1708 BooleanPredicateClausesStep<?> bool, 1709 ValueSet.ConceptSetFilterComponent nextFilter) { 1710 bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue())); 1711 } 1712 1713 private void addDisplayFilterInexact( 1714 SearchPredicateFactory f, 1715 BooleanPredicateClausesStep<?> bool, 1716 ValueSet.ConceptSetFilterComponent nextFilter) { 1717 bool.must(f.phrase() 1718 .field("myDisplay") 1719 .boost(4.0f) 1720 .field("myDisplayWordEdgeNGram") 1721 .boost(1.0f) 1722 .field("myDisplayEdgeNGram") 1723 .boost(1.0f) 1724 .matching(nextFilter.getValue().toLowerCase()) 1725 .slop(2)); 1726 } 1727 1728 private Term getPropertyTerm(String theProperty, String theValue) { 1729 return new Term(CONCEPT_PROPERTY_PREFIX_NAME + theProperty, theValue); 1730 } 1731 1732 private List<Term> getAncestorTerms(String theSystem, String theProperty, String theValue) { 1733 List<Term> retVal = new ArrayList<>(); 1734 1735 TermConcept code = findCode(theSystem, theValue) 1736 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" 1737 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1738 1739 retVal.add(new Term("myParentPids", "" + code.getId())); 1740 logFilteringValueOnProperty(theValue, theProperty); 1741 1742 return retVal; 1743 } 1744 1745 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1746 private void handleFilterLoincDescendant( 1747 String theSystem, 1748 SearchPredicateFactory f, 1749 BooleanPredicateClausesStep<?> b, 1750 ValueSet.ConceptSetFilterComponent theFilter) { 1751 switch (theFilter.getOp()) { 1752 case EQUAL: 1753 addLoincFilterDescendantEqual(theSystem, f, b, theFilter); 1754 break; 1755 case IN: 1756 addLoincFilterDescendantIn(theSystem, f, b, theFilter); 1757 break; 1758 default: 1759 throw new InvalidRequestException(Msg.code(896) + "Don't know how to handle op=" + theFilter.getOp() 1760 + " on property " + theFilter.getProperty()); 1761 } 1762 } 1763 1764 private void addLoincFilterDescendantEqual( 1765 String theSystem, 1766 SearchPredicateFactory f, 1767 BooleanPredicateClausesStep<?> b, 1768 ValueSet.ConceptSetFilterComponent theFilter) { 1769 1770 List<Long> parentPids = getCodeParentPids(theSystem, theFilter.getProperty(), theFilter.getValue()); 1771 if (parentPids.isEmpty()) { 1772 // Can't return empty must, because it wil match according to other predicates. 1773 // Some day there will be a 'matchNone' predicate 1774 // (https://discourse.hibernate.org/t/fail-fast-predicate/6062) 1775 b.mustNot(f.matchAll()); 1776 return; 1777 } 1778 1779 b.must(f.bool(innerB -> { 1780 innerB.minimumShouldMatchNumber(1); 1781 parentPids.forEach(pid -> innerB.should(f.match().field("myId").matching(pid))); 1782 })); 1783 } 1784 1785 /** 1786 * We are looking for codes which have codes indicated in theFilter.getValue() as descendants. 1787 * Strategy is to find codes which have their pId(s) in the list of the parentId(s) of all the TermConcept(s) 1788 * representing the codes in theFilter.getValue() 1789 */ 1790 private void addLoincFilterDescendantIn( 1791 String theSystem, 1792 SearchPredicateFactory f, 1793 BooleanPredicateClausesStep<?> b, 1794 ValueSet.ConceptSetFilterComponent theFilter) { 1795 1796 String[] values = theFilter.getValue().split(","); 1797 if (values.length == 0) { 1798 throw new InvalidRequestException(Msg.code(2062) + "Invalid filter criteria - no codes specified"); 1799 } 1800 1801 List<Long> descendantCodePidList = getMultipleCodeParentPids(theSystem, theFilter.getProperty(), values); 1802 1803 b.must(f.bool(innerB -> descendantCodePidList.forEach( 1804 pId -> innerB.should(f.match().field("myId").matching(pId))))); 1805 } 1806 1807 /** 1808 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1809 */ 1810 private List<Long> getCodeParentPids(String theSystem, String theProperty, String theValue) { 1811 TermConcept code = findCode(theSystem, theValue) 1812 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" 1813 + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1814 1815 String[] parentPids = code.getParentPidsAsString().split(" "); 1816 List<Long> retVal = Arrays.stream(parentPids) 1817 .filter(pid -> !StringUtils.equals(pid, "NONE")) 1818 .map(Long::parseLong) 1819 .collect(Collectors.toList()); 1820 logFilteringValueOnProperty(theValue, theProperty); 1821 return retVal; 1822 } 1823 1824 /** 1825 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1826 */ 1827 private List<Long> getMultipleCodeParentPids(String theSystem, String theProperty, String[] theValues) { 1828 List<String> valuesList = Arrays.asList(theValues); 1829 List<TermConcept> termConcepts = findCodes(theSystem, valuesList); 1830 if (valuesList.size() != termConcepts.size()) { 1831 String exMsg = getTermConceptsFetchExceptionMsg(termConcepts, valuesList); 1832 throw new InvalidRequestException(Msg.code(2064) + "Invalid filter criteria - {" 1833 + Constants.codeSystemWithDefaultDescription(theSystem) + "}: " + exMsg); 1834 } 1835 1836 List<Long> retVal = termConcepts.stream() 1837 .flatMap(tc -> Arrays.stream(tc.getParentPidsAsString().split(" "))) 1838 .filter(pid -> !StringUtils.equals(pid, "NONE")) 1839 .map(Long::parseLong) 1840 .collect(Collectors.toList()); 1841 1842 logFilteringValueOnProperties(valuesList, theProperty); 1843 1844 return retVal; 1845 } 1846 1847 /** 1848 * Generate message indicating for which of theValues a TermConcept was not found 1849 */ 1850 private String getTermConceptsFetchExceptionMsg(List<TermConcept> theTermConcepts, List<String> theValues) { 1851 // case: more TermConcept(s) retrieved than codes queried 1852 if (theTermConcepts.size() > theValues.size()) { 1853 return "Invalid filter criteria - More TermConcepts were found than indicated codes. Queried codes: [" 1854 + join( 1855 ",", 1856 theValues + "]; Obtained TermConcept IDs, codes: [" 1857 + theTermConcepts.stream() 1858 .map(tc -> tc.getId() + ", " + tc.getCode()) 1859 .collect(joining("; ")) 1860 + "]"); 1861 } 1862 1863 // case: less TermConcept(s) retrieved than codes queried 1864 Set<String> matchedCodes = 1865 theTermConcepts.stream().map(TermConcept::getCode).collect(toSet()); 1866 List<String> notMatchedValues = 1867 theValues.stream().filter(v -> !matchedCodes.contains(v)).collect(toList()); 1868 1869 return "Invalid filter criteria - No TermConcept(s) were found for the requested codes: [" 1870 + join(",", notMatchedValues + "]"); 1871 } 1872 1873 private void logFilteringValueOnProperty(String theValue, String theProperty) { 1874 ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty); 1875 } 1876 1877 private void logFilteringValueOnProperties(List<String> theValues, String theProperty) { 1878 ourLog.debug(" * Filtering with values={} on property {}", String.join(", ", theValues), theProperty); 1879 } 1880 1881 private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) { 1882 throw new InvalidRequestException( 1883 Msg.code(897) + "Don't know how to handle op=" + theOp + " on property " + theProperty); 1884 } 1885 1886 private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) { 1887 throw new InvalidRequestException( 1888 Msg.code(898) + "Don't know how to handle value=" + theValue + " on property " + theProperty); 1889 } 1890 1891 private void expandWithoutHibernateSearch( 1892 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1893 TermCodeSystemVersion theVersion, 1894 Set<String> theAddedCodes, 1895 ValueSet.ConceptSetComponent theInclude, 1896 String theSystem, 1897 boolean theAdd) { 1898 ourLog.trace("Hibernate search is not enabled"); 1899 1900 if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) { 1901 Validate.isTrue( 1902 ((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator) 1903 .getParameter() 1904 .isEmpty(), 1905 "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server."); 1906 } 1907 1908 Validate.isTrue( 1909 isNotBlank(theSystem), 1910 "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server."); 1911 1912 for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { 1913 boolean handled = false; 1914 switch (nextFilter.getProperty().toLowerCase()) { 1915 case "concept": 1916 case "code": 1917 if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) { 1918 theValueSetCodeAccumulator.addMessage( 1919 "Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient."); 1920 TermConcept code = findCodeForFilterCriteriaCodeOrConcept(theSystem, nextFilter); 1921 addConceptAndChildren( 1922 theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code); 1923 handled = true; 1924 } 1925 break; 1926 default: 1927 // TODO - we need to handle other properties (fields) 1928 // and other operations (not just is-a) 1929 // in some (preferably generic) way 1930 break; 1931 } 1932 1933 if (!handled) { 1934 throwInvalidFilter( 1935 nextFilter, 1936 " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion functionality is available."); 1937 } 1938 } 1939 1940 if (theInclude.getFilter().isEmpty() && theInclude.getConcept().isEmpty()) { 1941 Collection<TermConcept> concepts = 1942 myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid()); 1943 for (TermConcept next : concepts) { 1944 addCodeIfNotAlreadyAdded( 1945 theValueSetCodeAccumulator, 1946 theAddedCodes, 1947 theAdd, 1948 theSystem, 1949 theInclude.getVersion(), 1950 next.getCode(), 1951 next.getDisplay(), 1952 next.getId(), 1953 next.getParentPidsAsString(), 1954 next.getDesignations()); 1955 } 1956 } 1957 1958 for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) { 1959 if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) { 1960 continue; 1961 } 1962 Collection<TermConceptDesignation> designations = next.getDesignation().stream() 1963 .map(t -> new TermConceptDesignation() 1964 .setValue(t.getValue()) 1965 .setLanguage(t.getLanguage()) 1966 .setUseCode(t.getUse().getCode()) 1967 .setUseSystem(t.getUse().getSystem()) 1968 .setUseDisplay(t.getUse().getDisplay())) 1969 .collect(Collectors.toList()); 1970 addCodeIfNotAlreadyAdded( 1971 theValueSetCodeAccumulator, 1972 theAddedCodes, 1973 theAdd, 1974 theSystem, 1975 theInclude.getVersion(), 1976 next.getCode(), 1977 next.getDisplay(), 1978 null, 1979 null, 1980 designations); 1981 } 1982 } 1983 1984 private void addConceptAndChildren( 1985 IValueSetConceptAccumulator theValueSetCodeAccumulator, 1986 Set<String> theAddedCodes, 1987 ValueSet.ConceptSetComponent theInclude, 1988 String theSystem, 1989 boolean theAdd, 1990 TermConcept theConcept) { 1991 for (TermConcept nextChild : theConcept.getChildCodes()) { 1992 boolean added = addCodeIfNotAlreadyAdded( 1993 theValueSetCodeAccumulator, 1994 theAddedCodes, 1995 theAdd, 1996 theSystem, 1997 theInclude.getVersion(), 1998 nextChild.getCode(), 1999 nextChild.getDisplay(), 2000 nextChild.getId(), 2001 nextChild.getParentPidsAsString(), 2002 nextChild.getDesignations()); 2003 if (added) { 2004 addConceptAndChildren( 2005 theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild); 2006 } 2007 } 2008 } 2009 2010 @Override 2011 @Transactional 2012 public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) { 2013 IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails); 2014 ValueSet canonicalValueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet); 2015 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet); 2016 if (optionalTermValueSet.isEmpty()) { 2017 return myContext 2018 .getLocalizer() 2019 .getMessage(TermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId); 2020 } 2021 2022 ourLog.info( 2023 "Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl()); 2024 2025 TermValueSet termValueSet = optionalTermValueSet.get(); 2026 if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) { 2027 return myContext 2028 .getLocalizer() 2029 .getMessage( 2030 TermReadSvcImpl.class, 2031 "valueSetCantInvalidateNotYetPrecalculated", 2032 termValueSet.getUrl(), 2033 termValueSet.getExpansionStatus()); 2034 } 2035 2036 Long totalConcepts = termValueSet.getTotalConcepts(); 2037 2038 deletePreCalculatedValueSetContents(termValueSet); 2039 2040 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 2041 termValueSet.setExpansionTimestamp(null); 2042 myTermValueSetDao.save(termValueSet); 2043 2044 afterValueSetExpansionStatusChange(); 2045 2046 return myContext 2047 .getLocalizer() 2048 .getMessage( 2049 TermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts); 2050 } 2051 2052 @Override 2053 @Transactional(readOnly = true) 2054 public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) { 2055 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet); 2056 2057 if (optionalTermValueSet.isEmpty()) { 2058 ourLog.warn( 2059 "ValueSet is not present in terminology tables. Will perform in-memory code validation. {}", 2060 getValueSetInfo(theValueSet)); 2061 return false; 2062 } 2063 2064 TermValueSet termValueSet = optionalTermValueSet.get(); 2065 2066 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 2067 ourLog.warn( 2068 "{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}", 2069 getValueSetInfo(theValueSet), 2070 termValueSet.getExpansionStatus().name(), 2071 termValueSet.getExpansionStatus().getDescription()); 2072 return false; 2073 } 2074 2075 return true; 2076 } 2077 2078 @SuppressWarnings({"OptionalAssignedToNull", "unchecked"}) 2079 private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) { 2080 Optional<TermValueSet> retVal = (Optional<TermValueSet>) theValueSet.getUserData(VS_USERDATA_CURRENT_VERSION); 2081 if (retVal == null) { 2082 synchronized (theValueSet) { 2083 JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 2084 retVal = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getId()); 2085 theValueSet.setUserData(VS_USERDATA_CURRENT_VERSION, retVal); 2086 } 2087 } 2088 2089 return retVal; 2090 } 2091 2092 private JpaPid getValueSetResourcePersistentId(ValueSet theValueSet) { 2093 return myIdHelperService.resolveResourceIdentityPid( 2094 RequestPartitionId.allPartitions(), 2095 theValueSet.getIdElement().getResourceType(), 2096 theValueSet.getIdElement().getIdPart(), 2097 ResolveIdentityMode.includeDeleted().cacheOk()); 2098 } 2099 2100 protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet( 2101 ValidationSupportContext theValidationSupportContext, 2102 ConceptValidationOptions theValidationOptions, 2103 ValueSet theValueSet, 2104 String theSystem, 2105 String theCode, 2106 String theDisplay, 2107 Coding theCoding, 2108 CodeableConcept theCodeableConcept) { 2109 assert TransactionSynchronizationManager.isSynchronizationActive(); 2110 2111 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required"); 2112 JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 2113 2114 List<TermValueSetConcept> concepts = new ArrayList<>(); 2115 if (isNotBlank(theCode)) { 2116 if (theValidationOptions.isInferSystem()) { 2117 concepts.addAll( 2118 myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getId(), theCode)); 2119 } else if (isNotBlank(theSystem)) { 2120 concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode)); 2121 } 2122 } else if (theCoding != null) { 2123 if (theCoding.hasSystem() && theCoding.hasCode()) { 2124 concepts.addAll(findByValueSetResourcePidSystemAndCode( 2125 valueSetResourcePid, theCoding.getSystem(), theCoding.getCode())); 2126 } 2127 } else if (theCodeableConcept != null) { 2128 for (Coding coding : theCodeableConcept.getCoding()) { 2129 if (coding.hasSystem() && coding.hasCode()) { 2130 concepts.addAll(findByValueSetResourcePidSystemAndCode( 2131 valueSetResourcePid, coding.getSystem(), coding.getCode())); 2132 if (!concepts.isEmpty()) { 2133 break; 2134 } 2135 } 2136 } 2137 } else { 2138 return null; 2139 } 2140 2141 TermValueSet valueSetEntity = fetchValueSetEntity(theValueSet).orElseThrow(IllegalStateException::new); 2142 String timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity); 2143 String preExpansionMessage = myContext 2144 .getLocalizer() 2145 .getMessage(TermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription); 2146 2147 if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) { 2148 String systemVersion = null; 2149 for (TermValueSetConcept concept : concepts) { 2150 systemVersion = concept.getSystemVersion(); 2151 if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) { 2152 return new IValidationSupport.CodeValidationResult() 2153 .setCode(concept.getCode()) 2154 .setDisplay(concept.getDisplay()) 2155 .setCodeSystemVersion(concept.getSystemVersion()) 2156 .setSourceDetails(preExpansionMessage); 2157 } 2158 } 2159 2160 String expectedDisplay = concepts.get(0).getDisplay(); 2161 return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch( 2162 myContext, 2163 theCode, 2164 theDisplay, 2165 expectedDisplay, 2166 theSystem, 2167 systemVersion, 2168 myStorageSettings.getIssueSeverityForCodeDisplayMismatch()); 2169 } 2170 2171 if (!concepts.isEmpty()) { 2172 return new IValidationSupport.CodeValidationResult() 2173 .setCode(concepts.get(0).getCode()) 2174 .setDisplay(concepts.get(0).getDisplay()) 2175 .setCodeSystemVersion(concepts.get(0).getSystemVersion()) 2176 .setMessage(preExpansionMessage); 2177 } 2178 2179 // Ok, we failed 2180 List<TermValueSetConcept> outcome = myValueSetConceptDao.findByTermValueSetIdSystemOnly( 2181 Pageable.ofSize(1), valueSetEntity.getId(), theSystem); 2182 String append; 2183 if (outcome.size() == 0) { 2184 append = " - No codes in ValueSet belong to CodeSystem with URL " + theSystem; 2185 } else { 2186 String unknownCodeMessage = myContext 2187 .getLocalizer() 2188 .getMessage(TermReadSvcImpl.class, "unknownCodeInSystem", theSystem, theCode); 2189 append = " - " + unknownCodeMessage + ". " + preExpansionMessage; 2190 } 2191 2192 return createFailureCodeValidationResult(theSystem, theCode, null, append); 2193 } 2194 2195 private CodeValidationResult createFailureCodeValidationResult( 2196 String theSystem, String theCode, String theCodeSystemVersion, String theAppend) { 2197 String theMessage = "Unable to validate code " + theSystem + "#" + theCode + theAppend; 2198 return new CodeValidationResult() 2199 .setSeverity(IssueSeverity.ERROR) 2200 .setCodeSystemVersion(theCodeSystemVersion) 2201 .setMessage(theMessage) 2202 .addIssue(new CodeValidationIssue( 2203 theMessage, 2204 IssueSeverity.ERROR, 2205 CodeValidationIssueCode.CODE_INVALID, 2206 CodeValidationIssueCoding.INVALID_CODE)); 2207 } 2208 2209 private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode( 2210 JpaPid theResourcePid, String theSystem, String theCode) { 2211 assert TransactionSynchronizationManager.isSynchronizationActive(); 2212 2213 List<TermValueSetConcept> retVal = new ArrayList<>(); 2214 Optional<TermValueSetConcept> optionalTermValueSetConcept; 2215 int versionIndex = theSystem.indexOf(OUR_PIPE_CHARACTER); 2216 if (versionIndex >= 0) { 2217 String systemUrl = theSystem.substring(0, versionIndex); 2218 String systemVersion = theSystem.substring(versionIndex + 1); 2219 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion( 2220 theResourcePid.getId(), systemUrl, systemVersion, theCode); 2221 } else { 2222 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode( 2223 theResourcePid.getId(), theSystem, theCode); 2224 } 2225 optionalTermValueSetConcept.ifPresent(retVal::add); 2226 return retVal; 2227 } 2228 2229 private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 2230 for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) { 2231 TermConcept nextChild = nextChildLink.getChild(); 2232 if (addToSet(theSetToPopulate, nextChild)) { 2233 fetchChildren(nextChild, theSetToPopulate); 2234 } 2235 } 2236 } 2237 2238 private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) { 2239 TermCodeSystemVersion codeSystem = 2240 myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid); 2241 return myConceptDao.findByCodeSystemAndCode(codeSystem.getPid(), theCode); 2242 } 2243 2244 private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 2245 for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) { 2246 TermConcept nextChild = nextChildLink.getParent(); 2247 if (addToSet(theSetToPopulate, nextChild)) { 2248 fetchParents(nextChild, theSetToPopulate); 2249 } 2250 } 2251 } 2252 2253 @Override 2254 public Optional<TermConcept> findCode(String theCodeSystem, String theCode) { 2255 /* 2256 * Loading concepts without a transaction causes issues later on some 2257 * platforms (e.g. PSQL) so make sure that we always call this with an open transaction 2258 */ 2259 HapiTransactionService.requireTransaction(); 2260 2261 TermCodeSystemVersionDetails csv = 2262 getCurrentCodeSystemVersion(new ValidationSupportContext(provideValidationSupport()), theCodeSystem); 2263 if (csv == null) { 2264 return Optional.empty(); 2265 } 2266 return myConceptDao.findByCodeSystemAndCode(csv.myPid, theCode); 2267 } 2268 2269 @Override 2270 public List<TermConcept> findCodes(String theCodeSystem, List<String> theCodeList) { 2271 HapiTransactionService.requireTransaction(); 2272 2273 TermCodeSystemVersionDetails csv = 2274 getCurrentCodeSystemVersion(new ValidationSupportContext(provideValidationSupport()), theCodeSystem); 2275 if (csv == null) { 2276 return Collections.emptyList(); 2277 } 2278 2279 return myConceptDao.findByCodeSystemAndCodeList(csv.myPid, theCodeList); 2280 } 2281 2282 @Nullable 2283 private TermCodeSystemVersionDetails getCurrentCodeSystemVersion( 2284 ValidationSupportContext theValidationSupportContext, String theCodeSystemIdentifier) { 2285 String version = getVersionFromIdentifier(theCodeSystemIdentifier); 2286 2287 // Fetch the CodeSystem from ValidationSupport, which should return a cached copy. We 2288 // keep a copy of the current version entity in userData in that cached copy 2289 // to avoid repeated lookups 2290 TermCodeSystemVersionDetails retVal; 2291 IBaseResource codeSystem = 2292 theValidationSupportContext.getRootValidationSupport().fetchCodeSystem(theCodeSystemIdentifier); 2293 if (codeSystem != null) { 2294 2295 synchronized (codeSystem) { 2296 retVal = (TermCodeSystemVersionDetails) codeSystem.getUserData(CS_USERDATA_CURRENT_VERSION); 2297 if (retVal == null) { 2298 retVal = getCurrentCodeSystemVersion(theCodeSystemIdentifier, version); 2299 codeSystem.setUserData(CS_USERDATA_CURRENT_VERSION, retVal); 2300 } 2301 } 2302 } else { 2303 retVal = getCurrentCodeSystemVersion(theCodeSystemIdentifier, version); 2304 } 2305 2306 return retVal; 2307 } 2308 2309 @Nullable 2310 private TermCodeSystemVersionDetails getCurrentCodeSystemVersion(String theCodeSystemIdentifier, String version) { 2311 TermCodeSystemVersionDetails retVal; 2312 retVal = myTxTemplate.execute(tx -> { 2313 TermCodeSystemVersion csv = null; 2314 TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier)); 2315 if (cs != null) { 2316 if (version != null) { 2317 csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version); 2318 } else if (cs.getCurrentVersion() != null) { 2319 csv = cs.getCurrentVersion(); 2320 } 2321 } 2322 if (csv != null) { 2323 return new TermCodeSystemVersionDetails(csv.getPid(), csv.getCodeSystemVersionId()); 2324 } else { 2325 return null; 2326 } 2327 }); 2328 return retVal; 2329 } 2330 2331 private String getVersionFromIdentifier(String theUri) { 2332 String retVal = null; 2333 if (StringUtils.isNotEmpty((theUri))) { 2334 int versionSeparator = theUri.lastIndexOf('|'); 2335 if (versionSeparator != -1) { 2336 retVal = theUri.substring(versionSeparator + 1); 2337 } 2338 } 2339 return retVal; 2340 } 2341 2342 private String getUrlFromIdentifier(String theUri) { 2343 String retVal = theUri; 2344 if (StringUtils.isNotEmpty((theUri))) { 2345 int versionSeparator = theUri.lastIndexOf('|'); 2346 if (versionSeparator != -1) { 2347 retVal = theUri.substring(0, versionSeparator); 2348 } 2349 } 2350 return retVal; 2351 } 2352 2353 @Transactional(propagation = Propagation.REQUIRED, readOnly = true) 2354 @Override 2355 public Set<TermConcept> findCodesAbove( 2356 Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 2357 StopWatch stopwatch = new StopWatch(); 2358 2359 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 2360 if (concept.isEmpty()) { 2361 return Collections.emptySet(); 2362 } 2363 2364 Set<TermConcept> retVal = new HashSet<>(); 2365 retVal.add(concept.get()); 2366 2367 fetchParents(concept.get(), retVal); 2368 2369 ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis()); 2370 return retVal; 2371 } 2372 2373 @Transactional(readOnly = true) 2374 @Override 2375 public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) { 2376 TermCodeSystem cs = getCodeSystem(theSystem); 2377 if (cs == null) { 2378 return findCodesAboveUsingBuiltInSystems(theSystem, theCode); 2379 } 2380 TermCodeSystemVersion csv = cs.getCurrentVersion(); 2381 2382 Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode); 2383 return toVersionIndependentConcepts(theSystem, codes); 2384 } 2385 2386 @Transactional(propagation = Propagation.REQUIRED, readOnly = true) 2387 @Override 2388 public Set<TermConcept> findCodesBelow( 2389 Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 2390 Stopwatch stopwatch = Stopwatch.createStarted(); 2391 2392 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 2393 if (concept.isEmpty()) { 2394 return Collections.emptySet(); 2395 } 2396 2397 Set<TermConcept> retVal = new HashSet<>(); 2398 retVal.add(concept.get()); 2399 2400 fetchChildren(concept.get(), retVal); 2401 2402 ourLog.debug( 2403 "Fetched {} codes below code {} in {}ms", 2404 retVal.size(), 2405 theCode, 2406 stopwatch.elapsed(TimeUnit.MILLISECONDS)); 2407 return retVal; 2408 } 2409 2410 @Transactional(readOnly = true) 2411 @Override 2412 public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) { 2413 TermCodeSystem cs = getCodeSystem(theSystem); 2414 if (cs == null) { 2415 return findCodesBelowUsingBuiltInSystems(theSystem, theCode); 2416 } 2417 TermCodeSystemVersion csv = cs.getCurrentVersion(); 2418 2419 Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode); 2420 return toVersionIndependentConcepts(theSystem, codes); 2421 } 2422 2423 private TermCodeSystem getCodeSystem(String theSystem) { 2424 return myCodeSystemDao.findByCodeSystemUri(theSystem); 2425 } 2426 2427 @PostConstruct 2428 public void start() { 2429 RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute(); 2430 rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class)); 2431 myTxTemplate = new TransactionTemplate(myTransactionManager, rules); 2432 } 2433 2434 @Override 2435 public void scheduleJobs(ISchedulerService theSchedulerService) { 2436 // Register scheduled job to pre-expand ValueSets 2437 // In the future it would be great to make this a cluster-aware task somehow 2438 ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition(); 2439 vsJobDefinition.setId(getClass().getName()); 2440 vsJobDefinition.setJobClass(Job.class); 2441 theSchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition); 2442 } 2443 2444 @Override 2445 public synchronized void preExpandDeferredValueSetsToTerminologyTables() { 2446 if (!myStorageSettings.isEnableTaskPreExpandValueSets()) { 2447 return; 2448 } 2449 if (isNotSafeToPreExpandValueSets()) { 2450 ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded."); 2451 return; 2452 } 2453 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 2454 2455 while (true) { 2456 StopWatch sw = new StopWatch(); 2457 TermValueSet valueSetToExpand = txTemplate.execute(t -> { 2458 Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded(); 2459 if (optionalTermValueSet.isEmpty()) { 2460 return null; 2461 } 2462 2463 TermValueSet termValueSet = optionalTermValueSet.get(); 2464 termValueSet.setTotalConcepts(0L); 2465 termValueSet.setTotalConceptDesignations(0L); 2466 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS); 2467 return myTermValueSetDao.saveAndFlush(termValueSet); 2468 }); 2469 if (valueSetToExpand == null) { 2470 return; 2471 } 2472 2473 // We have a ValueSet to pre-expand. 2474 setPreExpandingValueSets(true); 2475 try { 2476 ValueSet valueSet = txTemplate.execute(t -> { 2477 TermValueSet refreshedValueSetToExpand = myTermValueSetDao 2478 .findById(valueSetToExpand.getId()) 2479 .orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId())); 2480 return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource()); 2481 }); 2482 assert valueSet != null; 2483 2484 ValueSetConceptAccumulator valueSetConceptAccumulator = 2485 myValueSetConceptAccumulatorFactory.create(valueSetToExpand); 2486 ValueSetExpansionOptions options = new ValueSetExpansionOptions(); 2487 options.setIncludeHierarchy(true); 2488 expandValueSet(options, valueSet, valueSetConceptAccumulator); 2489 2490 // We are done with this ValueSet. 2491 txTemplate.executeWithoutResult(t -> { 2492 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED); 2493 valueSetToExpand.setExpansionTimestamp(new Date()); 2494 myTermValueSetDao.saveAndFlush(valueSetToExpand); 2495 }); 2496 2497 afterValueSetExpansionStatusChange(); 2498 2499 ourLog.info( 2500 "Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", 2501 valueSet.getId(), 2502 valueSet.getUrl(), 2503 valueSetConceptAccumulator.getConceptsSaved(), 2504 sw); 2505 2506 } catch (Exception e) { 2507 ourLog.error( 2508 "Failed to pre-expand ValueSet with URL[{}]: {}", valueSetToExpand.getUrl(), e.getMessage(), e); 2509 txTemplate.executeWithoutResult(t -> { 2510 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND); 2511 myTermValueSetDao.saveAndFlush(valueSetToExpand); 2512 }); 2513 2514 } finally { 2515 setPreExpandingValueSets(false); 2516 } 2517 } 2518 } 2519 2520 /* 2521 * If a ValueSet has just finished pre-expanding, let's flush the caches. This is 2522 * kind of a blunt tool, but it should ensure that users don't get unpredictable 2523 * results while they test changes, which is probably a worthwhile sacrifice 2524 */ 2525 private void afterValueSetExpansionStatusChange() { 2526 provideValidationSupport().invalidateCaches(); 2527 } 2528 2529 @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") 2530 @Override 2531 public void invalidateCaches() { 2532 /* 2533 * Clear out anything left in the userdata caches. We do this mostly because it messes 2534 * up unit tests to have these things stick around between test runs, since many of 2535 * these resources come from DefaultProfileValidationSupport and therefore live beyond 2536 * any single test execution. 2537 */ 2538 for (IBaseResource next : provideValidationSupport().fetchAllConformanceResources()) { 2539 if (next != null) { 2540 synchronized (next) { 2541 if (next.getUserData(CS_USERDATA_CURRENT_VERSION) != null) { 2542 next.setUserData(CS_USERDATA_CURRENT_VERSION, null); 2543 } 2544 if (next.getUserData(VS_USERDATA_CURRENT_VERSION) != null) { 2545 next.setUserData(VS_USERDATA_CURRENT_VERSION, null); 2546 } 2547 } 2548 } 2549 } 2550 } 2551 2552 private synchronized boolean isPreExpandingValueSets() { 2553 return myPreExpandingValueSets; 2554 } 2555 2556 private synchronized void setPreExpandingValueSets(boolean thePreExpandingValueSets) { 2557 myPreExpandingValueSets = thePreExpandingValueSets; 2558 } 2559 2560 private boolean isNotSafeToPreExpandValueSets() { 2561 return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty(true); 2562 } 2563 2564 private Optional<TermValueSet> getNextTermValueSetNotExpanded() { 2565 Optional<TermValueSet> retVal = Optional.empty(); 2566 Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus( 2567 PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 2568 2569 if (!page.getContent().isEmpty()) { 2570 retVal = Optional.of(page.getContent().get(0)); 2571 } 2572 2573 return retVal; 2574 } 2575 2576 @Override 2577 @Transactional 2578 public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) { 2579 // If we're in a transaction, we need to flush now so that we can correctly detect 2580 // duplicates if there are multiple ValueSets in the same TX with the same URL 2581 // (which is an error, but we need to catch it). It'd be better to catch this by 2582 // inspecting the URLs in the bundle or something, since flushing hurts performance 2583 // but it's not expected that loading valuesets is going to be a huge high frequency 2584 // thing so it probably doesn't matter 2585 myEntityManager.flush(); 2586 2587 ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied"); 2588 if (isPlaceholder(theValueSet)) { 2589 ourLog.info( 2590 "Not storing TermValueSet for placeholder {}", 2591 theValueSet.getIdElement().toVersionless().getValueAsString()); 2592 return; 2593 } 2594 2595 ValidateUtil.isNotBlankOrThrowUnprocessableEntity( 2596 theValueSet.getUrl(), "ValueSet has no value for ValueSet.url"); 2597 ourLog.info( 2598 "Storing TermValueSet for {}", 2599 theValueSet.getIdElement().toVersionless().getValueAsString()); 2600 2601 /* 2602 * Get CodeSystem and validate CodeSystemVersion 2603 */ 2604 TermValueSet termValueSet = new TermValueSet(); 2605 termValueSet.setResource(theResourceTable); 2606 termValueSet.setUrl(theValueSet.getUrl()); 2607 termValueSet.setVersion(theValueSet.getVersion()); 2608 termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null); 2609 2610 // Delete version being replaced 2611 Optional<TermValueSet> deletedTrmValueSet = deleteValueSetForResource(theResourceTable); 2612 2613 /* 2614 * Do the upload. 2615 */ 2616 String url = termValueSet.getUrl(); 2617 String version = termValueSet.getVersion(); 2618 Optional<TermValueSet> optionalExistingTermValueSetByUrl; 2619 2620 if (deletedTrmValueSet.isPresent() 2621 && Objects.equals(deletedTrmValueSet.get().getUrl(), url) 2622 && Objects.equals(deletedTrmValueSet.get().getVersion(), version)) { 2623 // If we just deleted the valueset marker, we don't need to check if it exists 2624 // in the database 2625 optionalExistingTermValueSetByUrl = Optional.empty(); 2626 } else { 2627 optionalExistingTermValueSetByUrl = getTermValueSet(version, url); 2628 } 2629 2630 if (optionalExistingTermValueSetByUrl.isEmpty()) { 2631 2632 myTermValueSetDao.save(termValueSet); 2633 2634 } else { 2635 TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get(); 2636 String msg; 2637 if (version != null) { 2638 msg = myContext 2639 .getLocalizer() 2640 .getMessage( 2641 TermReadSvcImpl.class, 2642 "cannotCreateDuplicateValueSetUrlAndVersion", 2643 url, 2644 version, 2645 existingTermValueSet 2646 .getResource() 2647 .getIdDt() 2648 .toUnqualifiedVersionless() 2649 .getValue()); 2650 } else { 2651 msg = myContext 2652 .getLocalizer() 2653 .getMessage( 2654 TermReadSvcImpl.class, 2655 "cannotCreateDuplicateValueSetUrl", 2656 url, 2657 existingTermValueSet 2658 .getResource() 2659 .getIdDt() 2660 .toUnqualifiedVersionless() 2661 .getValue()); 2662 } 2663 throw new UnprocessableEntityException(Msg.code(902) + msg); 2664 } 2665 } 2666 2667 private Optional<TermValueSet> getTermValueSet(String version, String url) { 2668 Optional<TermValueSet> optionalExistingTermValueSetByUrl; 2669 if (version != null) { 2670 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version); 2671 } else { 2672 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url); 2673 } 2674 return optionalExistingTermValueSetByUrl; 2675 } 2676 2677 @Override 2678 @Transactional 2679 public IFhirResourceDaoCodeSystem.SubsumesResult subsumes( 2680 IPrimitiveType<String> theCodeA, 2681 IPrimitiveType<String> theCodeB, 2682 IPrimitiveType<String> theSystem, 2683 IBaseCoding theCodingA, 2684 IBaseCoding theCodingB) { 2685 FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA); 2686 FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB); 2687 2688 if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) { 2689 throw new InvalidRequestException( 2690 Msg.code(903) + "Unable to test subsumption across different code systems"); 2691 } 2692 2693 if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) { 2694 throw new InvalidRequestException( 2695 Msg.code(904) + "Unable to test subsumption across different code system versions"); 2696 } 2697 2698 String codeASystemIdentifier; 2699 if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) { 2700 codeASystemIdentifier = conceptA.getSystem() + OUR_PIPE_CHARACTER + conceptA.getSystemVersion(); 2701 } else { 2702 codeASystemIdentifier = conceptA.getSystem(); 2703 } 2704 TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode()) 2705 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA)); 2706 2707 String codeBSystemIdentifier; 2708 if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) { 2709 codeBSystemIdentifier = conceptB.getSystem() + OUR_PIPE_CHARACTER + conceptB.getSystemVersion(); 2710 } else { 2711 codeBSystemIdentifier = conceptB.getSystem(); 2712 } 2713 TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode()) 2714 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB)); 2715 2716 SearchSession searchSession = Search.session(myEntityManager); 2717 2718 ConceptSubsumptionOutcome subsumes; 2719 subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES); 2720 if (subsumes == null) { 2721 subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY); 2722 } 2723 if (subsumes == null) { 2724 subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED; 2725 } 2726 2727 return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes); 2728 } 2729 2730 @Override 2731 public IValidationSupport.LookupCodeResult lookupCode( 2732 ValidationSupportContext theValidationSupportContext, @Nonnull LookupCodeRequest theLookupCodeRequest) { 2733 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2734 return txTemplate.execute(t -> { 2735 final String theSystem = theLookupCodeRequest.getSystem(); 2736 final String theCode = theLookupCodeRequest.getCode(); 2737 Optional<TermConcept> codeOpt = findCode(theSystem, theCode); 2738 if (codeOpt.isPresent()) { 2739 TermConcept code = codeOpt.get(); 2740 2741 IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult(); 2742 result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName()); 2743 result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId()); 2744 result.setSearchedForSystem(theSystem); 2745 result.setSearchedForCode(theCode); 2746 result.setFound(true); 2747 result.setCodeDisplay(code.getDisplay()); 2748 2749 for (TermConceptDesignation next : code.getDesignations()) { 2750 // filter out the designation based on displayLanguage if any 2751 if (isDisplayLanguageMatch(theLookupCodeRequest.getDisplayLanguage(), next.getLanguage())) { 2752 IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation(); 2753 designation.setLanguage(next.getLanguage()); 2754 designation.setUseSystem(next.getUseSystem()); 2755 designation.setUseCode(next.getUseCode()); 2756 designation.setUseDisplay(next.getUseDisplay()); 2757 designation.setValue(next.getValue()); 2758 result.getDesignations().add(designation); 2759 } 2760 } 2761 2762 final Collection<String> propertyNames = theLookupCodeRequest.getPropertyNames(); 2763 for (TermConceptProperty next : code.getProperties()) { 2764 if (ObjectUtils.isNotEmpty(propertyNames) && !propertyNames.contains(next.getKey())) { 2765 continue; 2766 } 2767 if (next.getType() == TermConceptPropertyTypeEnum.CODING) { 2768 IValidationSupport.CodingConceptProperty property = 2769 new IValidationSupport.CodingConceptProperty( 2770 next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay()); 2771 result.getProperties().add(property); 2772 } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) { 2773 IValidationSupport.StringConceptProperty property = 2774 new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue()); 2775 result.getProperties().add(property); 2776 } else { 2777 throw new InternalErrorException(Msg.code(905) + "Unknown type: " + next.getType()); 2778 } 2779 } 2780 2781 return result; 2782 2783 } else { 2784 return new LookupCodeResult().setFound(false); 2785 } 2786 }); 2787 } 2788 2789 @Nullable 2790 private ConceptSubsumptionOutcome testForSubsumption( 2791 SearchSession theSearchSession, 2792 TermConcept theLeft, 2793 TermConcept theRight, 2794 ConceptSubsumptionOutcome theOutput) { 2795 List<TermConcept> fetch = theSearchSession 2796 .search(TermConcept.class) 2797 .where(f -> f.bool() 2798 .must(f.match().field("myId").matching(theRight.getId())) 2799 .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId())))) 2800 .fetchHits(1); 2801 2802 if (fetch.size() > 0) { 2803 return theOutput; 2804 } else { 2805 return null; 2806 } 2807 } 2808 2809 private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts( 2810 String theSystem, Set<TermConcept> codes) { 2811 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size()); 2812 for (TermConcept next : codes) { 2813 retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode())); 2814 } 2815 return retVal; 2816 } 2817 2818 @Override 2819 @Transactional 2820 public CodeValidationResult validateCodeInValueSet( 2821 ValidationSupportContext theValidationSupportContext, 2822 ConceptValidationOptions theOptions, 2823 String theCodeSystem, 2824 String theCode, 2825 String theDisplay, 2826 @Nonnull IBaseResource theValueSet) { 2827 invokeRunnableForUnitTest(); 2828 2829 IPrimitiveType<?> urlPrimitive; 2830 if (theValueSet instanceof org.hl7.fhir.dstu2.model.ValueSet) { 2831 urlPrimitive = FhirContext.forDstu2Hl7OrgCached() 2832 .newTerser() 2833 .getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); 2834 } else { 2835 urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); 2836 } 2837 String url = urlPrimitive.getValueAsString(); 2838 if (isNotBlank(url)) { 2839 return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url); 2840 } 2841 return null; 2842 } 2843 2844 @CoverageIgnore 2845 @Override 2846 public IValidationSupport.CodeValidationResult validateCode( 2847 @Nonnull ValidationSupportContext theValidationSupportContext, 2848 @Nonnull ConceptValidationOptions theOptions, 2849 String theCodeSystemUrl, 2850 String theCode, 2851 String theDisplay, 2852 String theValueSetUrl) { 2853 // TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS. 2854 invokeRunnableForUnitTest(); 2855 theOptions.setValidateDisplay(isNotBlank(theDisplay)); 2856 2857 if (isNotBlank(theValueSetUrl)) { 2858 return validateCodeInValueSet( 2859 theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystemUrl, theCode, theDisplay); 2860 } 2861 2862 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2863 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); 2864 txTemplate.setReadOnly(true); 2865 Optional<FhirVersionIndependentConcept> codeOpt = 2866 txTemplate.execute(tx -> findCode(theCodeSystemUrl, theCode).map(c -> { 2867 String codeSystemVersionId = getCurrentCodeSystemVersion( 2868 theValidationSupportContext, theCodeSystemUrl) 2869 .myCodeSystemVersionId; 2870 return new FhirVersionIndependentConcept( 2871 theCodeSystemUrl, c.getCode(), c.getDisplay(), codeSystemVersionId); 2872 })); 2873 2874 if (codeOpt != null && codeOpt.isPresent()) { 2875 FhirVersionIndependentConcept code = codeOpt.get(); 2876 if (!theOptions.isValidateDisplay() 2877 || isBlank(code.getDisplay()) 2878 || isBlank(theDisplay) 2879 || code.getDisplay().equals(theDisplay)) { 2880 return new CodeValidationResult().setCode(code.getCode()).setDisplay(code.getDisplay()); 2881 } else { 2882 return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch( 2883 myContext, 2884 theCode, 2885 theDisplay, 2886 code.getDisplay(), 2887 code.getSystem(), 2888 code.getSystemVersion(), 2889 myStorageSettings.getIssueSeverityForCodeDisplayMismatch()); 2890 } 2891 } 2892 2893 return createFailureCodeValidationResult( 2894 theCodeSystemUrl, theCode, null, createMessageAppendForCodeNotFoundInCodeSystem(theCodeSystemUrl)); 2895 } 2896 2897 IValidationSupport.CodeValidationResult validateCodeInValueSet( 2898 ValidationSupportContext theValidationSupportContext, 2899 ConceptValidationOptions theValidationOptions, 2900 String theValueSetUrl, 2901 String theCodeSystem, 2902 String theCode, 2903 String theDisplay) { 2904 IBaseResource valueSet = 2905 theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl); 2906 CodeValidationResult retVal = null; 2907 2908 // If we don't have a PID, this came from some source other than the JPA 2909 // database, so we don't need to check if it's pre-expanded or not 2910 if (valueSet instanceof IAnyResource) { 2911 Long pid = IDao.RESOURCE_PID.get(valueSet); 2912 if (pid != null) { 2913 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 2914 retVal = txTemplate.execute(tx -> { 2915 if (isValueSetPreExpandedForCodeValidation(valueSet)) { 2916 return validateCodeIsInPreExpandedValueSet( 2917 theValidationSupportContext, 2918 theValidationOptions, 2919 valueSet, 2920 theCodeSystem, 2921 theCode, 2922 theDisplay, 2923 null, 2924 null); 2925 } else { 2926 return null; 2927 } 2928 }); 2929 } 2930 } 2931 2932 if (retVal == null) { 2933 if (valueSet != null) { 2934 retVal = myInMemoryTerminologyServerValidationSupport.validateCodeInValueSet( 2935 theValidationSupportContext, 2936 theValidationOptions, 2937 theCodeSystem, 2938 theCode, 2939 theDisplay, 2940 valueSet); 2941 } else { 2942 String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]"; 2943 retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append); 2944 } 2945 } 2946 2947 // Check if someone is accidentally using a VS url where it should be a CS URL 2948 if (retVal != null 2949 && retVal.getCode() == null 2950 && theCodeSystem != null 2951 && myContext.getVersion().getVersion().isNewerThan(FhirVersionEnum.DSTU2)) { 2952 if (isValueSetSupported(theValidationSupportContext, theCodeSystem)) { 2953 if (!isCodeSystemSupported(theValidationSupportContext, theCodeSystem)) { 2954 String newMessage = "Unable to validate code " + theCodeSystem + "#" + theCode 2955 + " - Supplied system URL is a ValueSet URL and not a CodeSystem URL, check if it is correct: " 2956 + theCodeSystem; 2957 retVal.setMessage(newMessage); 2958 } 2959 } 2960 } 2961 2962 return retVal; 2963 } 2964 2965 @Override 2966 public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) { 2967 IValidationSupport validationSupport = provideValidationSupport(); 2968 IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem); 2969 if (codeSystem != null) { 2970 codeSystem = myVersionCanonicalizer.codeSystemToCanonical(codeSystem); 2971 } 2972 return (CodeSystem) codeSystem; 2973 } 2974 2975 @Nonnull 2976 private IValidationSupport provideJpaValidationSupport() { 2977 IValidationSupport jpaValidationSupport = myJpaValidationSupport; 2978 if (jpaValidationSupport == null) { 2979 jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class); 2980 myJpaValidationSupport = jpaValidationSupport; 2981 } 2982 return jpaValidationSupport; 2983 } 2984 2985 @Nonnull 2986 protected IValidationSupport provideValidationSupport() { 2987 IValidationSupport validationSupport = myValidationSupport; 2988 if (validationSupport == null) { 2989 validationSupport = myApplicationContext.getBean(IValidationSupport.class); 2990 myValidationSupport = validationSupport; 2991 } 2992 return validationSupport; 2993 } 2994 2995 public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) { 2996 IValidationSupport validationSupport = provideValidationSupport(); 2997 IBaseResource valueSet = validationSupport.fetchValueSet(theSystem); 2998 if (valueSet != null) { 2999 valueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet); 3000 } 3001 return (ValueSet) valueSet; 3002 } 3003 3004 @Override 3005 public IBaseResource fetchValueSet(String theValueSetUrl) { 3006 return provideJpaValidationSupport().fetchValueSet(theValueSetUrl); 3007 } 3008 3009 @Override 3010 public FhirContext getFhirContext() { 3011 return myContext; 3012 } 3013 3014 private void findCodesAbove( 3015 CodeSystem theSystem, 3016 String theSystemString, 3017 String theCode, 3018 List<FhirVersionIndependentConcept> theListToPopulate) { 3019 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 3020 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 3021 addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate); 3022 } 3023 } 3024 3025 @Override 3026 public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) { 3027 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 3028 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 3029 if (system != null) { 3030 findCodesAbove(system, theSystem, theCode, retVal); 3031 } 3032 return retVal; 3033 } 3034 3035 private void findCodesBelow( 3036 CodeSystem theSystem, 3037 String theSystemString, 3038 String theCode, 3039 List<FhirVersionIndependentConcept> theListToPopulate) { 3040 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 3041 findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList); 3042 } 3043 3044 private void findCodesBelow( 3045 String theSystemString, 3046 String theCode, 3047 List<FhirVersionIndependentConcept> theListToPopulate, 3048 List<CodeSystem.ConceptDefinitionComponent> conceptList) { 3049 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 3050 if (theCode.equals(next.getCode())) { 3051 addAllChildren(theSystemString, next, theListToPopulate); 3052 } else { 3053 findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept()); 3054 } 3055 } 3056 } 3057 3058 @Override 3059 public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) { 3060 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 3061 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 3062 if (system != null) { 3063 findCodesBelow(system, theSystem, theCode, retVal); 3064 } 3065 return retVal; 3066 } 3067 3068 private void addAllChildren( 3069 String theSystemString, 3070 CodeSystem.ConceptDefinitionComponent theCode, 3071 List<FhirVersionIndependentConcept> theListToPopulate) { 3072 if (isNotBlank(theCode.getCode())) { 3073 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode())); 3074 } 3075 for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) { 3076 addAllChildren(theSystemString, nextChild, theListToPopulate); 3077 } 3078 } 3079 3080 private boolean addTreeIfItContainsCode( 3081 String theSystemString, 3082 CodeSystem.ConceptDefinitionComponent theNext, 3083 String theCode, 3084 List<FhirVersionIndependentConcept> theListToPopulate) { 3085 boolean foundCodeInChild = false; 3086 for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) { 3087 foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate); 3088 } 3089 3090 if (theCode.equals(theNext.getCode()) || foundCodeInChild) { 3091 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode())); 3092 return true; 3093 } 3094 3095 return false; 3096 } 3097 3098 @Nonnull 3099 private FhirVersionIndependentConcept toConcept( 3100 IPrimitiveType<String> theCodeType, 3101 IPrimitiveType<String> theCodeSystemIdentifierType, 3102 IBaseCoding theCodingType) { 3103 String code = theCodeType != null ? theCodeType.getValueAsString() : null; 3104 String system = theCodeSystemIdentifierType != null 3105 ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) 3106 : null; 3107 String systemVersion = theCodeSystemIdentifierType != null 3108 ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) 3109 : null; 3110 if (theCodingType != null) { 3111 Coding canonicalizedCoding = myVersionCanonicalizer.codingToCanonical(theCodingType); 3112 assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't 3113 code = canonicalizedCoding.getCode(); 3114 system = canonicalizedCoding.getSystem(); 3115 systemVersion = canonicalizedCoding.getVersion(); 3116 } 3117 return new FhirVersionIndependentConcept(system, code, null, systemVersion); 3118 } 3119 3120 /** 3121 * When the search is for unversioned loinc system it uses the forcedId to obtain the current 3122 * version, as it is not necessarily the last one anymore. 3123 * For other cases it keeps on considering the last uploaded as the current 3124 */ 3125 @Override 3126 public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) { 3127 if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) { 3128 Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl); 3129 if (vsIdOpt.isEmpty()) { 3130 return Optional.empty(); 3131 } 3132 3133 return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get()); 3134 } 3135 3136 List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl); 3137 if (termValueSetList.isEmpty()) { 3138 return Optional.empty(); 3139 } 3140 3141 return Optional.of(termValueSetList.get(0)); 3142 } 3143 3144 @Override 3145 public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) { 3146 @SuppressWarnings("unchecked") 3147 List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager 3148 .createQuery("select r from ResourceTable r " 3149 + "where r.myResourceType = 'CodeSystem' and r.myFhirId = :fhirId") 3150 .setParameter("fhirId", theForcedId) 3151 .getResultList(); 3152 if (resultList.isEmpty()) return Optional.empty(); 3153 3154 if (resultList.size() > 1) 3155 throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " 3156 + theForcedId + ". Was constraint " + ResourceTable.IDX_RES_TYPE_FHIR_ID + " removed?"); 3157 3158 IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem"); 3159 IBaseResource cs = myJpaStorageResourceParser.toResource(resultList.get(0), false); 3160 return Optional.of(cs); 3161 } 3162 3163 @Transactional 3164 @Override 3165 public ReindexTerminologyResult reindexTerminology() throws InterruptedException { 3166 if (myFulltextSearchSvc == null) { 3167 return ReindexTerminologyResult.SEARCH_SVC_DISABLED; 3168 } 3169 3170 if (isBatchTerminologyTasksRunning()) { 3171 return ReindexTerminologyResult.OTHER_BATCH_TERMINOLOGY_TASKS_RUNNING; 3172 } 3173 3174 // disallow pre-expanding ValueSets while reindexing 3175 myDeferredStorageSvc.setProcessDeferred(false); 3176 3177 int objectLoadingThreadNumber = calculateObjectLoadingThreadNumber(); 3178 ourLog.info("Using {} threads to load objects", objectLoadingThreadNumber); 3179 3180 try { 3181 SearchSession searchSession = getSearchSession(); 3182 searchSession 3183 .massIndexer(TermConcept.class) 3184 .dropAndCreateSchemaOnStart(true) 3185 .purgeAllOnStart(false) 3186 .batchSizeToLoadObjects(100) 3187 .cacheMode(CacheMode.IGNORE) 3188 .threadsToLoadObjects(6) 3189 .transactionTimeout(60 * SECONDS_IN_MINUTE) 3190 .monitor(new PojoMassIndexingLoggingMonitor(INDEXED_ROOTS_LOGGING_COUNT)) 3191 .startAndWait(); 3192 } finally { 3193 myDeferredStorageSvc.setProcessDeferred(true); 3194 } 3195 3196 return ReindexTerminologyResult.SUCCESS; 3197 } 3198 3199 @VisibleForTesting 3200 boolean isBatchTerminologyTasksRunning() { 3201 return isNotSafeToPreExpandValueSets() || isPreExpandingValueSets(); 3202 } 3203 3204 @VisibleForTesting 3205 int calculateObjectLoadingThreadNumber() { 3206 IConnectionPoolInfoProvider connectionPoolInfoProvider = 3207 new ConnectionPoolInfoProvider(myHibernatePropertiesProvider.getDataSource()); 3208 Optional<Integer> maxConnectionsOpt = connectionPoolInfoProvider.getTotalConnectionSize(); 3209 if (maxConnectionsOpt.isEmpty()) { 3210 return DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS; 3211 } 3212 3213 int maxConnections = maxConnectionsOpt.get(); 3214 int usableThreads = maxConnections < 6 ? 1 : maxConnections - 5; 3215 int objectThreads = Math.min(usableThreads, MAX_MASS_INDEXER_OBJECT_LOADING_THREADS); 3216 ourLog.debug( 3217 "Data source connection pool has {} connections allocated, so reindexing will use {} object " 3218 + "loading threads (each using a connection)", 3219 maxConnections, 3220 objectThreads); 3221 return objectThreads; 3222 } 3223 3224 @VisibleForTesting 3225 SearchSession getSearchSession() { 3226 return Search.session(myEntityManager); 3227 } 3228 3229 @Override 3230 public ValueSetExpansionOutcome expandValueSet( 3231 ValidationSupportContext theValidationSupportContext, 3232 ValueSetExpansionOptions theExpansionOptions, 3233 @Nonnull IBaseResource theValueSetToExpand) { 3234 ValueSet canonicalInput = myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand); 3235 org.hl7.fhir.r4.model.ValueSet expandedR4 = expandValueSet(theExpansionOptions, canonicalInput); 3236 return new ValueSetExpansionOutcome(myVersionCanonicalizer.valueSetFromCanonical(expandedR4)); 3237 } 3238 3239 @Override 3240 public IBaseResource expandValueSet(ValueSetExpansionOptions theExpansionOptions, IBaseResource theInput) { 3241 org.hl7.fhir.r4.model.ValueSet valueSetToExpand = myVersionCanonicalizer.valueSetToCanonical(theInput); 3242 org.hl7.fhir.r4.model.ValueSet valueSetR4 = expandValueSet(theExpansionOptions, valueSetToExpand); 3243 return myVersionCanonicalizer.valueSetFromCanonical(valueSetR4); 3244 } 3245 3246 @Override 3247 public void expandValueSet( 3248 ValueSetExpansionOptions theExpansionOptions, 3249 IBaseResource theValueSetToExpand, 3250 IValueSetConceptAccumulator theValueSetCodeAccumulator) { 3251 org.hl7.fhir.r4.model.ValueSet valueSetToExpand = 3252 myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand); 3253 expandValueSet(theExpansionOptions, valueSetToExpand, theValueSetCodeAccumulator); 3254 } 3255 3256 private org.hl7.fhir.r4.model.ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) { 3257 Class<? extends IBaseResource> type = 3258 getFhirContext().getResourceDefinition("ValueSet").getImplementingClass(); 3259 IBaseResource valueSet = myJpaStorageResourceParser.toResource(type, theResourceTable, null, false); 3260 return myVersionCanonicalizer.valueSetToCanonical(valueSet); 3261 } 3262 3263 @Override 3264 public CodeValidationResult validateCodeIsInPreExpandedValueSet( 3265 ValidationSupportContext theValidationSupportContext, 3266 ConceptValidationOptions theOptions, 3267 IBaseResource theValueSet, 3268 String theSystem, 3269 String theCode, 3270 String theDisplay, 3271 IBaseDatatype theCoding, 3272 IBaseDatatype theCodeableConcept) { 3273 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null"); 3274 org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet); 3275 org.hl7.fhir.r4.model.Coding codingR4 = myVersionCanonicalizer.codingToCanonical((IBaseCoding) theCoding); 3276 org.hl7.fhir.r4.model.CodeableConcept codeableConcept = 3277 myVersionCanonicalizer.codeableConceptToCanonical(theCodeableConcept); 3278 3279 return validateCodeIsInPreExpandedValueSet( 3280 theValidationSupportContext, 3281 theOptions, 3282 valueSetR4, 3283 theSystem, 3284 theCode, 3285 theDisplay, 3286 codingR4, 3287 codeableConcept); 3288 } 3289 3290 @Override 3291 public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) { 3292 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null"); 3293 org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet); 3294 return isValueSetPreExpandedForCodeValidation(valueSetR4); 3295 } 3296 3297 private static class TermCodeSystemVersionDetails { 3298 3299 private final long myPid; 3300 private final String myCodeSystemVersionId; 3301 3302 public TermCodeSystemVersionDetails(long thePid, String theCodeSystemVersionId) { 3303 myPid = thePid; 3304 myCodeSystemVersionId = theCodeSystemVersionId; 3305 } 3306 } 3307 3308 public static class Job implements HapiJob { 3309 @Autowired 3310 private ITermReadSvc myTerminologySvc; 3311 3312 @Override 3313 public void execute(JobExecutionContext theContext) { 3314 myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables(); 3315 } 3316 } 3317 3318 /** 3319 * Properties returned from method buildSearchScroll 3320 */ 3321 private static final class SearchProperties { 3322 private final List<Supplier<SearchScroll<EntityReference>>> mySearchScroll = new ArrayList<>(); 3323 private List<String> myIncludeOrExcludeCodes; 3324 3325 public List<Supplier<SearchScroll<EntityReference>>> getSearchScroll() { 3326 return mySearchScroll; 3327 } 3328 3329 public void addSearchScroll(Supplier<SearchScroll<EntityReference>> theSearchScrollSupplier) { 3330 mySearchScroll.add(theSearchScrollSupplier); 3331 } 3332 3333 public List<String> getIncludeOrExcludeCodes() { 3334 return myIncludeOrExcludeCodes; 3335 } 3336 3337 public void setIncludeOrExcludeCodes(List<String> theIncludeOrExcludeCodes) { 3338 myIncludeOrExcludeCodes = theIncludeOrExcludeCodes; 3339 } 3340 3341 public boolean hasIncludeOrExcludeCodes() { 3342 return !myIncludeOrExcludeCodes.isEmpty(); 3343 } 3344 } 3345 3346 static boolean isValueSetDisplayLanguageMatch(ValueSetExpansionOptions theExpansionOptions, String theStoredLang) { 3347 if (theExpansionOptions == null) { 3348 return true; 3349 } 3350 3351 if (theExpansionOptions.getTheDisplayLanguage() == null || theStoredLang == null) { 3352 return true; 3353 } 3354 3355 return theExpansionOptions.getTheDisplayLanguage().equalsIgnoreCase(theStoredLang); 3356 } 3357 3358 @Nonnull 3359 private static String createMessageAppendForCodeNotFoundInCodeSystem(String theCodeSystemUrl) { 3360 return " - Code is not found in CodeSystem: " + theCodeSystemUrl; 3361 } 3362 3363 @VisibleForTesting 3364 public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) { 3365 ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest; 3366 } 3367 3368 static boolean isPlaceholder(DomainResource theResource) { 3369 boolean retVal = false; 3370 Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER); 3371 if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) { 3372 retVal = ((BooleanType) extension.getValue()).booleanValue(); 3373 } 3374 return retVal; 3375 } 3376 3377 /** 3378 * This is only used for unit tests to test failure conditions 3379 */ 3380 static void invokeRunnableForUnitTest() { 3381 if (myInvokeOnNextCallForUnitTest != null) { 3382 Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest; 3383 myInvokeOnNextCallForUnitTest = null; 3384 invokeOnNextCallForUnitTest.run(); 3385 } 3386 } 3387 3388 @VisibleForTesting 3389 public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) { 3390 myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest; 3391 } 3392 3393 static List<TermConcept> toPersistedConcepts( 3394 List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) { 3395 ArrayList<TermConcept> retVal = new ArrayList<>(); 3396 3397 for (CodeSystem.ConceptDefinitionComponent next : theConcept) { 3398 if (isNotBlank(next.getCode())) { 3399 TermConcept termConcept = toTermConcept(next, theCodeSystemVersion); 3400 retVal.add(termConcept); 3401 } 3402 } 3403 3404 return retVal; 3405 } 3406 3407 @Nonnull 3408 static TermConcept toTermConcept( 3409 CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) { 3410 TermConcept termConcept = new TermConcept(); 3411 termConcept.setCode(theConceptDefinition.getCode()); 3412 termConcept.setCodeSystemVersion(theCodeSystemVersion); 3413 termConcept.setDisplay(theConceptDefinition.getDisplay()); 3414 termConcept.addChildren( 3415 toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA); 3416 3417 for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent : 3418 theConceptDefinition.getDesignation()) { 3419 if (isNotBlank(designationComponent.getValue())) { 3420 TermConceptDesignation designation = termConcept.addDesignation(); 3421 designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null); 3422 if (designationComponent.hasUse()) { 3423 designation.setUseSystem( 3424 designationComponent.getUse().hasSystem() 3425 ? designationComponent.getUse().getSystem() 3426 : null); 3427 designation.setUseCode( 3428 designationComponent.getUse().hasCode() 3429 ? designationComponent.getUse().getCode() 3430 : null); 3431 designation.setUseDisplay( 3432 designationComponent.getUse().hasDisplay() 3433 ? designationComponent.getUse().getDisplay() 3434 : null); 3435 } 3436 designation.setValue(designationComponent.getValue()); 3437 } 3438 } 3439 3440 for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) { 3441 TermConceptProperty property = new TermConceptProperty(); 3442 3443 property.setKey(next.getCode()); 3444 property.setConcept(termConcept); 3445 property.setCodeSystemVersion(theCodeSystemVersion); 3446 3447 if (next.getValue() instanceof StringType) { 3448 property.setType(TermConceptPropertyTypeEnum.STRING); 3449 property.setValue(next.getValueStringType().getValue()); 3450 } else if (next.getValue() instanceof BooleanType) { 3451 property.setType(TermConceptPropertyTypeEnum.BOOLEAN); 3452 property.setValue(((BooleanType) next.getValue()).getValueAsString()); 3453 } else if (next.getValue() instanceof IntegerType) { 3454 property.setType(TermConceptPropertyTypeEnum.INTEGER); 3455 property.setValue(((IntegerType) next.getValue()).getValueAsString()); 3456 } else if (next.getValue() instanceof DecimalType) { 3457 property.setType(TermConceptPropertyTypeEnum.DECIMAL); 3458 property.setValue(((DecimalType) next.getValue()).getValueAsString()); 3459 } else if (next.getValue() instanceof DateTimeType) { 3460 // DateType is not supported because it's not 3461 // supported in CodeSystem.setValue 3462 property.setType(TermConceptPropertyTypeEnum.DATETIME); 3463 property.setValue(((DateTimeType) next.getValue()).getValueAsString()); 3464 } else if (next.getValue() instanceof Coding) { 3465 Coding nextCoding = next.getValueCoding(); 3466 property.setType(TermConceptPropertyTypeEnum.CODING); 3467 property.setCodeSystem(nextCoding.getSystem()); 3468 property.setValue(nextCoding.getCode()); 3469 property.setDisplay(nextCoding.getDisplay()); 3470 } else if (next.getValue() != null) { 3471 ourLog.warn("Don't know how to handle properties of type: " 3472 + next.getValue().getClass()); 3473 continue; 3474 } 3475 3476 termConcept.getProperties().add(property); 3477 } 3478 return termConcept; 3479 } 3480 3481 static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) { 3482 // NOTE: return the designation when one of then is not specified. 3483 if (theReqLang == null || theStoredLang == null) return true; 3484 3485 return theReqLang.equalsIgnoreCase(theStoredLang); 3486 } 3487}