
001package ca.uhn.fhir.jpa.term; 002 003/* 004 * #%L 005 * HAPI FHIR JPA Server 006 * %% 007 * Copyright (C) 2014 - 2022 Smile CDR, Inc. 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023import ca.uhn.fhir.context.FhirContext; 024import ca.uhn.fhir.context.support.ConceptValidationOptions; 025import ca.uhn.fhir.context.support.IValidationSupport; 026import ca.uhn.fhir.context.support.ValidationSupportContext; 027import ca.uhn.fhir.context.support.ValueSetExpansionOptions; 028import ca.uhn.fhir.i18n.Msg; 029import ca.uhn.fhir.interceptor.model.RequestPartitionId; 030import ca.uhn.fhir.jpa.api.config.DaoConfig; 031import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 032import ca.uhn.fhir.jpa.api.dao.IDao; 033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 034import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem; 035import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 036import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider; 037import ca.uhn.fhir.jpa.config.util.ConnectionPoolInfoProvider; 038import ca.uhn.fhir.jpa.config.util.IConnectionPoolInfoProvider; 039import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; 040import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao; 041import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao; 042import ca.uhn.fhir.jpa.dao.data.ITermConceptDao; 043import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao; 044import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao; 045import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao; 046import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao; 047import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao; 048import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao; 049import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao; 050import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView; 051import ca.uhn.fhir.jpa.entity.TermCodeSystem; 052import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion; 053import ca.uhn.fhir.jpa.entity.TermConcept; 054import ca.uhn.fhir.jpa.entity.TermConceptDesignation; 055import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink; 056import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum; 057import ca.uhn.fhir.jpa.entity.TermConceptProperty; 058import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum; 059import ca.uhn.fhir.jpa.entity.TermValueSet; 060import ca.uhn.fhir.jpa.entity.TermValueSetConcept; 061import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum; 062import ca.uhn.fhir.jpa.model.entity.ForcedId; 063import ca.uhn.fhir.jpa.model.entity.ResourceTable; 064import ca.uhn.fhir.jpa.model.sched.HapiJob; 065import ca.uhn.fhir.jpa.model.sched.ISchedulerService; 066import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition; 067import ca.uhn.fhir.jpa.model.util.JpaConstants; 068import ca.uhn.fhir.jpa.search.ElasticsearchNestedQueryBuilderUtil; 069import ca.uhn.fhir.jpa.search.builder.SearchBuilder; 070import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc; 071import ca.uhn.fhir.jpa.term.api.ITermReadSvc; 072import ca.uhn.fhir.jpa.term.api.ReindexTerminologyResult; 073import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException; 074import ca.uhn.fhir.jpa.util.LogicUtil; 075import ca.uhn.fhir.rest.api.Constants; 076import ca.uhn.fhir.rest.api.server.RequestDetails; 077import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId; 078import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 079import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 080import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; 081import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException; 082import ca.uhn.fhir.util.CoverageIgnore; 083import ca.uhn.fhir.util.FhirVersionIndependentConcept; 084import ca.uhn.fhir.util.HapiExtensions; 085import ca.uhn.fhir.util.StopWatch; 086import ca.uhn.fhir.util.UrlUtil; 087import ca.uhn.fhir.util.ValidateUtil; 088import com.github.benmanes.caffeine.cache.Cache; 089import com.github.benmanes.caffeine.cache.Caffeine; 090import com.google.common.annotations.VisibleForTesting; 091import com.google.common.base.Stopwatch; 092import com.google.common.collect.ArrayListMultimap; 093import com.google.gson.JsonObject; 094import org.apache.commons.collections4.ListUtils; 095import org.apache.commons.lang3.ObjectUtils; 096import org.apache.commons.lang3.StringUtils; 097import org.apache.commons.lang3.Validate; 098import org.apache.commons.lang3.time.DateUtils; 099import org.apache.lucene.index.Term; 100import org.apache.lucene.search.BooleanQuery; 101import org.apache.lucene.search.Query; 102import org.apache.lucene.search.RegexpQuery; 103import org.apache.lucene.search.TermQuery; 104import org.hibernate.CacheMode; 105import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension; 106import org.hibernate.search.backend.lucene.LuceneExtension; 107import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep; 108import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep; 109import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; 110import org.hibernate.search.engine.search.query.SearchQuery; 111import org.hibernate.search.mapper.orm.Search; 112import org.hibernate.search.mapper.orm.common.EntityReference; 113import org.hibernate.search.mapper.orm.session.SearchSession; 114import org.hibernate.search.mapper.pojo.massindexing.impl.PojoMassIndexingLoggingMonitor; 115import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService; 116import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport; 117import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50; 118import org.hl7.fhir.convertors.context.ConversionContext40_50; 119import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50; 120import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50; 121import org.hl7.fhir.instance.model.api.IAnyResource; 122import org.hl7.fhir.instance.model.api.IBaseCoding; 123import org.hl7.fhir.instance.model.api.IBaseDatatype; 124import org.hl7.fhir.instance.model.api.IBaseResource; 125import org.hl7.fhir.instance.model.api.IIdType; 126import org.hl7.fhir.instance.model.api.IPrimitiveType; 127import org.hl7.fhir.r4.model.BooleanType; 128import org.hl7.fhir.r4.model.CanonicalType; 129import org.hl7.fhir.r4.model.CodeSystem; 130import org.hl7.fhir.r4.model.CodeableConcept; 131import org.hl7.fhir.r4.model.Coding; 132import org.hl7.fhir.r4.model.DomainResource; 133import org.hl7.fhir.r4.model.Enumerations; 134import org.hl7.fhir.r4.model.Extension; 135import org.hl7.fhir.r4.model.InstantType; 136import org.hl7.fhir.r4.model.IntegerType; 137import org.hl7.fhir.r4.model.StringType; 138import org.hl7.fhir.r4.model.ValueSet; 139import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome; 140import org.quartz.JobExecutionContext; 141import org.springframework.beans.factory.annotation.Autowired; 142import org.springframework.context.ApplicationContext; 143import org.springframework.context.annotation.Lazy; 144import org.springframework.data.domain.PageRequest; 145import org.springframework.data.domain.Pageable; 146import org.springframework.data.domain.Slice; 147import org.springframework.transaction.PlatformTransactionManager; 148import org.springframework.transaction.TransactionDefinition; 149import org.springframework.transaction.annotation.Propagation; 150import org.springframework.transaction.annotation.Transactional; 151import org.springframework.transaction.interceptor.NoRollbackRuleAttribute; 152import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute; 153import org.springframework.transaction.support.TransactionSynchronizationManager; 154import org.springframework.transaction.support.TransactionTemplate; 155import org.springframework.util.CollectionUtils; 156import org.springframework.util.comparator.Comparators; 157 158import javax.annotation.Nonnull; 159import javax.annotation.Nullable; 160import javax.annotation.PostConstruct; 161import javax.persistence.EntityManager; 162import javax.persistence.NonUniqueResultException; 163import javax.persistence.PersistenceContext; 164import javax.persistence.PersistenceContextType; 165import javax.persistence.TypedQuery; 166import javax.persistence.criteria.CriteriaBuilder; 167import javax.persistence.criteria.CriteriaQuery; 168import javax.persistence.criteria.Fetch; 169import javax.persistence.criteria.Join; 170import javax.persistence.criteria.JoinType; 171import javax.persistence.criteria.Predicate; 172import javax.persistence.criteria.Root; 173import java.util.ArrayList; 174import java.util.Arrays; 175import java.util.Collection; 176import java.util.Collections; 177import java.util.Date; 178import java.util.HashMap; 179import java.util.HashSet; 180import java.util.LinkedHashMap; 181import java.util.List; 182import java.util.Locale; 183import java.util.Map; 184import java.util.Objects; 185import java.util.Optional; 186import java.util.Set; 187import java.util.StringTokenizer; 188import java.util.UUID; 189import java.util.concurrent.TimeUnit; 190import java.util.concurrent.atomic.AtomicInteger; 191import java.util.function.Consumer; 192import java.util.function.Supplier; 193import java.util.stream.Collectors; 194 195import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI; 196import static java.lang.String.join; 197import static java.util.stream.Collectors.joining; 198import static java.util.stream.Collectors.toList; 199import static java.util.stream.Collectors.toSet; 200import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; 201import static org.apache.commons.lang3.StringUtils.defaultString; 202import static org.apache.commons.lang3.StringUtils.isBlank; 203import static org.apache.commons.lang3.StringUtils.isEmpty; 204import static org.apache.commons.lang3.StringUtils.isNoneBlank; 205import static org.apache.commons.lang3.StringUtils.isNotBlank; 206import static org.apache.commons.lang3.StringUtils.lowerCase; 207import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase; 208import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW; 209 210public abstract class BaseTermReadSvcImpl implements ITermReadSvc { 211 public static final int DEFAULT_FETCH_SIZE = 250; 212 private static final int SINGLE_FETCH_SIZE = 1; 213 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseTermReadSvcImpl.class); 214 private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions(); 215 private static final TermCodeSystemVersion NO_CURRENT_VERSION = new TermCodeSystemVersion().setId(-1L); 216 private static Runnable myInvokeOnNextCallForUnitTest; 217 private static boolean ourForceDisableHibernateSearchForUnitTest; 218 219 private static final String IDX_PROPERTIES = "myProperties"; 220 private static final String IDX_PROP_KEY = IDX_PROPERTIES + ".myKey"; 221 private static final String IDX_PROP_VALUE_STRING = IDX_PROPERTIES + ".myValueString"; 222 private static final String IDX_PROP_DISPLAY_STRING = IDX_PROPERTIES + ".myDisplayString"; 223 224 public static final int DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS = 2; 225 // doesn't seem to be much gain by using more threads than this value 226 public static final int MAX_MASS_INDEXER_OBJECT_LOADING_THREADS = 6; 227 228 private boolean myPreExpandingValueSets = false; 229 230 private final Cache<String, TermCodeSystemVersion> myCodeSystemCurrentVersionCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build(); 231 @Autowired 232 protected DaoRegistry myDaoRegistry; 233 @Autowired 234 protected ITermCodeSystemDao myCodeSystemDao; 235 @Autowired 236 protected ITermConceptDao myConceptDao; 237 @Autowired 238 protected ITermConceptPropertyDao myConceptPropertyDao; 239 @Autowired 240 protected ITermConceptDesignationDao myConceptDesignationDao; 241 @Autowired 242 protected ITermValueSetDao myTermValueSetDao; 243 @Autowired 244 protected ITermValueSetConceptDao myValueSetConceptDao; 245 @Autowired 246 protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao; 247 @Autowired 248 protected FhirContext myContext; 249 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 250 protected EntityManager myEntityManager; 251 @Autowired 252 private ITermCodeSystemVersionDao myCodeSystemVersionDao; 253 @Autowired 254 private DaoConfig myDaoConfig; 255 private TransactionTemplate myTxTemplate; 256 @Autowired 257 private PlatformTransactionManager myTransactionManager; 258 @Autowired(required = false) 259 private IFulltextSearchSvc myFulltextSearchSvc; 260 @Autowired 261 private PlatformTransactionManager myTxManager; 262 @Autowired 263 private ITermConceptDao myTermConceptDao; 264 @Autowired 265 private ITermValueSetConceptViewDao myTermValueSetConceptViewDao; 266 @Autowired 267 private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao; 268 @Autowired 269 private ISchedulerService mySchedulerService; 270 @Autowired(required = false) 271 private ITermDeferredStorageSvc myDeferredStorageSvc; 272 @Autowired 273 private IIdHelperService myIdHelperService; 274 275 @Autowired 276 private ApplicationContext myApplicationContext; 277 278 private volatile IValidationSupport myJpaValidationSupport; 279 private volatile IValidationSupport myValidationSupport; 280 281 //We need this bean so we can tell which mode hibernate search is running in. 282 @Autowired 283 private HibernatePropertiesProvider myHibernatePropertiesProvider; 284 285 286 private boolean isFullTextSetToUseElastic() { 287 return "elasticsearch".equalsIgnoreCase(myHibernatePropertiesProvider.getHibernateSearchBackend()); 288 } 289 290 @Override 291 public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) { 292 TermCodeSystemVersion cs = getCurrentCodeSystemVersion(theSystem); 293 return cs != null; 294 } 295 296 private boolean addCodeIfNotAlreadyAdded(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) { 297 String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri(); 298 String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId(); 299 String code = theConcept.getCode(); 300 String display = theConcept.getDisplay(); 301 Long sourceConceptPid = theConcept.getId(); 302 String directParentPids = ""; 303 304 if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) { 305 directParentPids = theConcept 306 .getParents() 307 .stream() 308 .map(t -> t.getParent().getId().toString()) 309 .collect(joining(" ")); 310 } 311 312 Collection<TermConceptDesignation> designations = theConcept.getDesignations(); 313 if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) { 314 return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem + "|" + theValueSetIncludeVersion, code, display, sourceConceptPid, directParentPids, codeSystemVersion); 315 } else { 316 return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem, code, display, sourceConceptPid, directParentPids, codeSystemVersion); 317 } 318 } 319 320 private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) { 321 if (StringUtils.isNotEmpty(theCodeSystemVersion)) { 322 if (isNoneBlank(theCodeSystem, theCode)) { 323 if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) { 324 theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion); 325 return true; 326 } 327 328 if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) { 329 theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode); 330 return true; 331 } 332 } 333 } else { 334 if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) { 335 theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion); 336 return true; 337 } 338 339 if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) { 340 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 341 return true; 342 } 343 } 344 345 return false; 346 } 347 348 private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, String theSystemVersion) { 349 if (isNoneBlank(theCodeSystem, theCode)) { 350 if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) { 351 theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theSystemVersion); 352 return true; 353 } 354 355 if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) { 356 theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode); 357 return true; 358 } 359 } 360 361 return false; 362 } 363 364 private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) { 365 boolean retVal = theSetToPopulate.add(theConcept); 366 if (retVal) { 367 if (theSetToPopulate.size() >= myDaoConfig.getMaximumExpansionSize()) { 368 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize()); 369 throw new ExpansionTooCostlyException(Msg.code(885) + msg); 370 } 371 } 372 return retVal; 373 } 374 375 /** 376 * This method is present only for unit tests, do not call from client code 377 */ 378 @VisibleForTesting 379 public void clearCaches() { 380 myCodeSystemCurrentVersionCache.invalidateAll(); 381 } 382 383 384 public void deleteValueSetForResource(ResourceTable theResourceTable) { 385 // Get existing entity so it can be deleted. 386 Optional<TermValueSet> optionalExistingTermValueSetById = myTermValueSetDao.findByResourcePid(theResourceTable.getId()); 387 388 if (optionalExistingTermValueSetById.isPresent()) { 389 TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get(); 390 391 ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId()); 392 deletePreCalculatedValueSetContents(existingTermValueSet); 393 myTermValueSetDao.deleteById(existingTermValueSet.getId()); 394 ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId()); 395 } 396 } 397 398 private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) { 399 myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId()); 400 myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId()); 401 } 402 403 @Override 404 @Transactional 405 public void deleteValueSetAndChildren(ResourceTable theResourceTable) { 406 deleteValueSetForResource(theResourceTable); 407 } 408 409 410 @Override 411 @Transactional 412 public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 413 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out. 414 415 ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl); 416 417 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 418 for (ValueSet.ValueSetExpansionContainsComponent nextContains : expanded.getExpansion().getContains()) { 419 retVal.add(new FhirVersionIndependentConcept(nextContains.getSystem(), nextContains.getCode(), nextContains.getDisplay(), nextContains.getVersion())); 420 } 421 return retVal; 422 } 423 424 @Override 425 @Transactional 426 public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) { 427 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl); 428 if (valueSet == null) { 429 throw new ResourceNotFoundException(Msg.code(886) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl)); 430 } 431 432 return expandValueSet(theExpansionOptions, valueSet); 433 } 434 435 @Override 436 @Transactional(propagation = Propagation.REQUIRED) 437 public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) { 438 String filter = null; 439 if (theExpansionOptions != null) { 440 filter = theExpansionOptions.getFilter(); 441 } 442 return expandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter)); 443 } 444 445 private ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, ExpansionFilter theFilter) { 446 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null"); 447 448 ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions); 449 int offset = expansionOptions.getOffset(); 450 int count = expansionOptions.getCount(); 451 452 ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, count, expansionOptions.isIncludeHierarchy()); 453 accumulator.setHardExpansionMaximumSize(myDaoConfig.getMaximumExpansionSize()); 454 accumulator.setSkipCountRemaining(offset); 455 accumulator.setIdentifier(UUID.randomUUID().toString()); 456 accumulator.setTimestamp(new Date()); 457 accumulator.setOffset(offset); 458 459 if (theExpansionOptions != null && isHibernateSearchEnabled()) { 460 accumulator.addParameter().setName("offset").setValue(new IntegerType(offset)); 461 accumulator.addParameter().setName("count").setValue(new IntegerType(count)); 462 } 463 464 expandValueSetIntoAccumulator(theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true); 465 466 if (accumulator.getTotalConcepts() != null) { 467 accumulator.setTotal(accumulator.getTotalConcepts()); 468 } 469 470 ValueSet valueSet = new ValueSet(); 471 valueSet.setUrl(theValueSetToExpand.getUrl()); 472 valueSet.setId(theValueSetToExpand.getId()); 473 valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE); 474 valueSet.setCompose(theValueSetToExpand.getCompose()); 475 valueSet.setExpansion(accumulator); 476 477 for (String next : accumulator.getMessages()) { 478 valueSet.getMeta().addExtension() 479 .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE) 480 .setValue(new StringType(next)); 481 } 482 483 if (expansionOptions.isIncludeHierarchy()) { 484 accumulator.applyHierarchy(); 485 } 486 487 return valueSet; 488 } 489 490 private void expandValueSetIntoAccumulator(ValueSet theValueSetToExpand, ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theAccumulator, ExpansionFilter theFilter, boolean theAdd) { 491 Optional<TermValueSet> optionalTermValueSet; 492 if (theValueSetToExpand.hasUrl()) { 493 if (theValueSetToExpand.hasVersion()) { 494 optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion(theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion()); 495 } else { 496 optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl()); 497 } 498 } else { 499 optionalTermValueSet = Optional.empty(); 500 } 501 502 /* 503 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion 504 */ 505 if (!optionalTermValueSet.isPresent()) { 506 ourLog.debug("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", getValueSetInfo(theValueSetToExpand)); 507 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingInMemoryExpansion", getValueSetInfo(theValueSetToExpand)); 508 theAccumulator.addMessage(msg); 509 expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter); 510 return; 511 } 512 513 /* 514 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion 515 */ 516 TermValueSet termValueSet = optionalTermValueSet.get(); 517 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 518 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded", getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription()); 519 theAccumulator.addMessage(msg); 520 expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter); 521 return; 522 } 523 524 /* 525 * ValueSet is pre-expanded in database so let's use that 526 */ 527 String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet); 528 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp); 529 theAccumulator.addMessage(msg); 530 expandConcepts(theAccumulator, termValueSet, theFilter, theAdd, isOracleDialect()); 531 } 532 533 @Nonnull 534 private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) { 535 String expansionTimestamp = "(unknown)"; 536 if (termValueSet.getExpansionTimestamp() != null) { 537 String timeElapsed = StopWatch.formatMillis(System.currentTimeMillis() - termValueSet.getExpansionTimestamp().getTime()); 538 expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString() + " (" + timeElapsed + " ago)"; 539 } 540 return expansionTimestamp; 541 } 542 543 private boolean isOracleDialect() { 544 return myHibernatePropertiesProvider.getDialect() instanceof org.hibernate.dialect.Oracle12cDialect; 545 } 546 547 private void expandConcepts(IValueSetConceptAccumulator theAccumulator, TermValueSet theTermValueSet, ExpansionFilter theFilter, boolean theAdd, boolean theOracle) { 548 // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its copy pasted sibling 549 Integer offset = theAccumulator.getSkipCountRemaining(); 550 offset = ObjectUtils.defaultIfNull(offset, 0); 551 offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue()); 552 553 Integer count = theAccumulator.getCapacityRemaining(); 554 count = defaultIfNull(count, myDaoConfig.getMaximumExpansionSize()); 555 556 int conceptsExpanded = 0; 557 int designationsExpanded = 0; 558 int toIndex = offset + count; 559 560 Collection<? extends ITermValueSetConceptView> conceptViews; 561 boolean wasFilteredResult = false; 562 String filterDisplayValue = null; 563 if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) { 564 filterDisplayValue = lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]")); 565 String displayValue = "%" + lowerCase(filterDisplayValue) + "%"; 566 if (theOracle) { 567 conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 568 } else { 569 conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue); 570 } 571 wasFilteredResult = true; 572 } else { 573 // TODO JA HS: I'm pretty sure we are overfetching here. test says offset 3, count 4, but we are fetching index 3 -> 10 here, grabbing 7 concepts. 574 //Specifically this test testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectRange 575 if (theOracle) { 576 conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId()); 577 } else { 578 conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId()); 579 } 580 theAccumulator.consumeSkipCount(offset); 581 if (theAdd) { 582 theAccumulator.incrementOrDecrementTotalConcepts(true, theTermValueSet.getTotalConcepts().intValue()); 583 } 584 } 585 586 if (conceptViews.isEmpty()) { 587 logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded); 588 return; 589 } 590 591 Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>(); 592 ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create(); 593 Map<Long, Long> pidToSourcePid = new HashMap<>(); 594 Map<Long, String> pidToSourceDirectParentPids = new HashMap<>(); 595 596 for (ITermValueSetConceptView conceptView : conceptViews) { 597 598 String system = conceptView.getConceptSystemUrl(); 599 String code = conceptView.getConceptCode(); 600 String display = conceptView.getConceptDisplay(); 601 String systemVersion = conceptView.getConceptSystemVersion(); 602 603 //-- this is quick solution, may need to revisit 604 if (!applyFilter(display, filterDisplayValue)) { 605 continue;} 606 607 Long conceptPid = conceptView.getConceptPid(); 608 if (!pidToConcept.containsKey(conceptPid)) { 609 FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display, systemVersion); 610 pidToConcept.put(conceptPid, concept); 611 } 612 613 // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations optional. 614 if (conceptView.getDesignationPid() != null) { 615 TermConceptDesignation designation = new TermConceptDesignation(); 616 designation.setUseSystem(conceptView.getDesignationUseSystem()); 617 designation.setUseCode(conceptView.getDesignationUseCode()); 618 designation.setUseDisplay(conceptView.getDesignationUseDisplay()); 619 designation.setValue(conceptView.getDesignationVal()); 620 designation.setLanguage(conceptView.getDesignationLang()); 621 pidToDesignations.put(conceptPid, designation); 622 623 if (++designationsExpanded % 250 == 0) { 624 logDesignationsExpanded("Expansion of designations in progress. ", theTermValueSet, designationsExpanded); 625 } 626 } 627 628 if (theAccumulator.isTrackingHierarchy()) { 629 pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid()); 630 pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids()); 631 } 632 633 if (++conceptsExpanded % 250 == 0) { 634 logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded); 635 } 636 } 637 638 for (Long nextPid : pidToConcept.keySet()) { 639 FhirVersionIndependentConcept concept = pidToConcept.get(nextPid); 640 List<TermConceptDesignation> designations = pidToDesignations.get(nextPid); 641 String system = concept.getSystem(); 642 String code = concept.getCode(); 643 String display = concept.getDisplay(); 644 String systemVersion = concept.getSystemVersion(); 645 646 if (theAdd) { 647 if (theAccumulator.getCapacityRemaining() != null) { 648 if (theAccumulator.getCapacityRemaining() == 0) { 649 break; 650 } 651 } 652 653 Long sourceConceptPid = pidToSourcePid.get(nextPid); 654 String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid); 655 theAccumulator.includeConceptWithDesignations(system, code, display, designations, sourceConceptPid, sourceConceptDirectParentPids, systemVersion); 656 } else { 657 boolean removed = theAccumulator.excludeConcept(system, code); 658 if (removed) { 659 theAccumulator.incrementOrDecrementTotalConcepts(false, 1); 660 } 661 } 662 } 663 664 if (wasFilteredResult && theAdd) { 665 theAccumulator.incrementOrDecrementTotalConcepts(true, pidToConcept.size()); 666 } 667 668 logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded); 669 logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded); 670 } 671 672 private void logConceptsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) { 673 if (theConceptsExpanded > 0) { 674 ourLog.debug("{}Have expanded {} concepts in ValueSet[{}]", theLogDescriptionPrefix, theConceptsExpanded, theTermValueSet.getUrl()); 675 } 676 } 677 678 private void logDesignationsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) { 679 if (theDesignationsExpanded > 0) { 680 ourLog.debug("{}Have expanded {} designations in ValueSet[{}]", theLogDescriptionPrefix, theDesignationsExpanded, theTermValueSet.getUrl()); 681 } 682 } 683 684 public boolean applyFilter(final String theDisplay, final String theFilterDisplay) { 685 686 //-- safety check only, no need to apply filter 687 if (theDisplay == null || theFilterDisplay == null) 688 return true; 689 690 // -- sentence case 691 if (startsWithIgnoreCase(theDisplay, theFilterDisplay)) 692 return true; 693 694 //-- token case 695 return startsWithByWordBoundaries(theDisplay, theFilterDisplay); 696 } 697 698 private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) { 699 // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo' 700 StringTokenizer tok = new StringTokenizer(theDisplay); 701 List<String> tokens = new ArrayList<>(); 702 while (tok.hasMoreTokens()) { 703 String token = tok.nextToken(); 704 if (startsWithIgnoreCase(token, theFilterDisplay)) 705 return true; 706 tokens.add(token); 707 } 708 709 // Allow to search by the end of the phrase. E.g. "working proficiency" will match "Limited working proficiency" 710 for (int start = 0; start <= tokens.size() - 1; ++start) { 711 for (int end = start + 1; end <= tokens.size(); ++end) { 712 String sublist = String.join(" ", tokens.subList(start, end)); 713 if (startsWithIgnoreCase(sublist, theFilterDisplay)) 714 return true; 715 } 716 } 717 return false; 718 } 719 720 @Override 721 @Transactional(propagation = Propagation.REQUIRED) 722 public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) { 723 expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, ExpansionFilter.NO_FILTER); 724 } 725 726 @SuppressWarnings("ConstantConditions") 727 private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, @Nonnull ExpansionFilter theExpansionFilter) { 728 Set<String> addedCodes = new HashSet<>(); 729 730 StopWatch sw = new StopWatch(); 731 String valueSetInfo = getValueSetInfo(theValueSetToExpand); 732 ourLog.debug("Working with {}", valueSetInfo); 733 734 // Offset can't be combined with excludes 735 Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining(); 736 if (skipCountRemaining != null && skipCountRemaining > 0) { 737 if (theValueSetToExpand.getCompose().getExclude().size() > 0) { 738 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo); 739 throw new InvalidRequestException(Msg.code(887) + msg); 740 } 741 } 742 743 // Handle includes 744 ourLog.debug("Handling includes"); 745 for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) { 746 for (int i = 0; ; i++) { 747 int queryIndex = i; 748 Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> { 749 boolean add = true; 750 return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, queryIndex, theExpansionFilter); 751 }); 752 if (!shouldContinue) { 753 break; 754 } 755 } 756 } 757 758 // Handle excludes 759 ourLog.debug("Handling excludes"); 760 for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) { 761 for (int i = 0; ; i++) { 762 int queryIndex = i; 763 Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> { 764 boolean add = false; 765 ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER; 766 return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, queryIndex, expansionFilter); 767 }); 768 if (!shouldContinue) { 769 break; 770 } 771 } 772 } 773 774 if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) { 775 myTxTemplate.execute(t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder()); 776 } 777 778 ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis()); 779 } 780 781 /** 782 * Execute in a new transaction only if we aren't already in one. We do this because in some cases 783 * when performing a VS expansion we throw an {@link ExpansionTooCostlyException} and we don't want 784 * this to cause the TX to be marked a rollback prematurely. 785 */ 786 private <T> T executeInNewTransactionIfNeeded(Supplier<T> theAction) { 787 if (TransactionSynchronizationManager.isSynchronizationActive()) { 788 return theAction.get(); 789 } 790 return myTxTemplate.execute(t -> theAction.get()); 791 } 792 793 private String getValueSetInfo(ValueSet theValueSet) { 794 StringBuilder sb = new StringBuilder(); 795 boolean isIdentified = false; 796 if (theValueSet.hasUrl()) { 797 isIdentified = true; 798 sb 799 .append("ValueSet.url[") 800 .append(theValueSet.getUrl()) 801 .append("]"); 802 } else if (theValueSet.hasId()) { 803 isIdentified = true; 804 sb 805 .append("ValueSet.id[") 806 .append(theValueSet.getId()) 807 .append("]"); 808 } 809 810 if (!isIdentified) { 811 sb.append("Unidentified ValueSet"); 812 } 813 814 return sb.toString(); 815 } 816 817 /** 818 * @return Returns true if there are potentially more results to process. 819 */ 820 private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter) { 821 822 String system = theIncludeOrExclude.getSystem(); 823 boolean hasSystem = isNotBlank(system); 824 boolean hasValueSet = theIncludeOrExclude.getValueSet().size() > 0; 825 826 if (hasSystem) { 827 828 if (theExpansionFilter.hasCode() && theExpansionFilter.getSystem() != null && !system.equals(theExpansionFilter.getSystem())) { 829 return false; 830 } 831 832 ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system); 833 834 TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system); 835 if (cs != null) { 836 837 return expandValueSetHandleIncludeOrExcludeUsingDatabase(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs); 838 839 } else { 840 841 if (theIncludeOrExclude.getConcept().size() > 0 && theExpansionFilter.hasCode()) { 842 if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) { 843 if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) { 844 return false; 845 } 846 } 847 } 848 849 Consumer<FhirVersionIndependentConcept> consumer = c -> { 850 addOrRemoveCode(theValueSetCodeAccumulator, theAddedCodes, theAdd, system, c.getCode(), c.getDisplay(), c.getSystemVersion()); 851 }; 852 853 try { 854 ConversionContext40_50.INSTANCE.init(new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet"); 855 org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude = ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude); 856 new InMemoryTerminologyServerValidationSupport(myContext).expandValueSetIncludeOrExclude(new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude); 857 } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) { 858 if (!theExpansionOptions.isFailOnMissingCodeSystem() && e.getFailureType() == InMemoryTerminologyServerValidationSupport.FailureType.UNKNOWN_CODE_SYSTEM) { 859 return false; 860 } 861 throw new InternalErrorException(Msg.code(888) + e); 862 } finally { 863 ConversionContext40_50.INSTANCE.close("ValueSet"); 864 } 865 866 return false; 867 } 868 869 } else if (hasValueSet) { 870 871 for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) { 872 String valueSetUrl = nextValueSet.getValueAsString(); 873 ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl); 874 875 ExpansionFilter subExpansionFilter = new ExpansionFilter(theExpansionFilter, theIncludeOrExclude.getFilter(), theValueSetCodeAccumulator.getCapacityRemaining()); 876 877 // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out. 878 879 ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl); 880 if (valueSet == null) { 881 throw new ResourceNotFoundException(Msg.code(889) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl)); 882 } 883 884 expandValueSetIntoAccumulator(valueSet, theExpansionOptions, theValueSetCodeAccumulator, subExpansionFilter, theAdd); 885 886 } 887 888 return false; 889 890 } else { 891 throw new InvalidRequestException(Msg.code(890) + "ValueSet contains " + (theAdd ? "include" : "exclude") + " criteria with no system defined"); 892 } 893 894 895 } 896 897 private boolean isHibernateSearchEnabled() { 898 return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest; 899 } 900 901 @Nonnull 902 private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) { 903 String includeOrExcludeVersion = theIncludeOrExclude.getVersion(); 904 TermCodeSystemVersion csv; 905 if (isEmpty(includeOrExcludeVersion)) { 906 csv = theCs.getCurrentVersion(); 907 } else { 908 csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCs.getPid(), includeOrExcludeVersion); 909 } 910 911 SearchSession searchSession = Search.session(myEntityManager); 912 /* 913 * If FullText searching is not enabled, we can handle only basic expansions 914 * since we're going to do it without the database. 915 */ 916 if (!isHibernateSearchEnabled()) { 917 expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd); 918 return false; 919 } 920 921 /* 922 * Ok, let's use hibernate search to build the expansion 923 */ 924 //Manually building a predicate since we need to throw it around. 925 SearchPredicateFactory predicate = searchSession.scope(TermConcept.class).predicate(); 926 927 //Build the top-level expansion on filters. 928 PredicateFinalStep step = predicate.bool(b -> { 929 b.must(predicate.match().field("myCodeSystemVersionPid").matching(csv.getPid())); 930 931 if (theExpansionFilter.hasCode()) { 932 b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode())); 933 } 934 935 String codeSystemUrlAndVersion = buildCodeSystemUrlAndVersion(theSystem, includeOrExcludeVersion); 936 for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) { 937 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 938 } 939 for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) { 940 handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter); 941 } 942 }); 943 944 List<String> codes = theIncludeOrExclude 945 .getConcept() 946 .stream() 947 .filter(Objects::nonNull) 948 .map(ValueSet.ConceptReferenceComponent::getCode) 949 .filter(StringUtils::isNotBlank) 950 .collect(Collectors.toList()); 951 952 Optional<PredicateFinalStep> expansionStepOpt = buildExpansionPredicate(codes, predicate); 953 final PredicateFinalStep finishedQuery = expansionStepOpt.isPresent() 954 ? predicate.bool().must(step).must(expansionStepOpt.get()) : step; 955 956 /* 957 * DM 2019-08-21 - Processing slows after any ValueSets with many codes explicitly identified. This might 958 * be due to the dark arts that is memory management. Will monitor but not do anything about this right now. 959 */ 960 961 //BooleanQuery.setMaxClauseCount(SearchBuilder.getMaximumPageSize()); 962 //TODO GGG HS looks like we can't set max clause count, but it can be set server side. 963 //BooleanQuery.setMaxClauseCount(10000); 964 // JM 2-22-02-15 - Hopefully increasing maxClauseCount should be not needed anymore 965 966 StopWatch sw = new StopWatch(); 967 AtomicInteger count = new AtomicInteger(0); 968 969 int maxResultsPerBatch = SearchBuilder.getMaximumPageSize(); 970 971 /* 972 * If the accumulator is bounded, we may reduce the size of the query to 973 * Lucene in order to be more efficient. 974 */ 975 if (theAdd) { 976 Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining(); 977 if (accumulatorCapacityRemaining != null) { 978 maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1); 979 } 980 if (maxResultsPerBatch <= 0) { 981 return false; 982 } 983 } 984 985 ourLog.debug("Beginning batch expansion for {} with max results per batch: {}", (theAdd ? "inclusion" : "exclusion"), maxResultsPerBatch); 986 987 StopWatch swForBatch = new StopWatch(); 988 AtomicInteger countForBatch = new AtomicInteger(0); 989 990 SearchQuery<EntityReference> termConceptsQuery = searchSession 991 .search(TermConcept.class) 992 .selectEntityReference() 993 .where(f -> finishedQuery) 994 .toQuery(); 995 996 ourLog.trace("About to query: {}", termConceptsQuery.queryString()); 997 List<EntityReference> termConceptRefs = termConceptsQuery.fetchHits(theQueryIndex * maxResultsPerBatch, maxResultsPerBatch); 998 List<Long> pids = termConceptRefs 999 .stream() 1000 .map(t -> (Long) t.id()) 1001 .collect(Collectors.toList()); 1002 1003 List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids); 1004 1005 // If the include section had multiple codes, return the codes in the same order 1006 if (codes.size() > 1) { 1007 termConcepts = new ArrayList<>(termConcepts); 1008 Map<String, Integer> codeToIndex = new HashMap<>(codes.size()); 1009 for (int i = 0; i < codes.size(); i++) { 1010 codeToIndex.put(codes.get(i), i); 1011 } 1012 termConcepts.sort(((o1, o2) -> { 1013 Integer idx1 = codeToIndex.get(o1.getCode()); 1014 Integer idx2 = codeToIndex.get(o2.getCode()); 1015 return Comparators.nullsHigh().compare(idx1, idx2); 1016 })); 1017 } 1018 1019 int resultsInBatch = termConcepts.size(); 1020 int firstResult = theQueryIndex * maxResultsPerBatch;// TODO GGG HS we lose the ability to check the index of the first result, so just best-guessing it here. 1021 int delta = 0; 1022 for (TermConcept concept : termConcepts) { 1023 count.incrementAndGet(); 1024 countForBatch.incrementAndGet(); 1025 if (theAdd && expansionStepOpt.isPresent()) { 1026 ValueSet.ConceptReferenceComponent theIncludeConcept = getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept); 1027 if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) { 1028 concept.setDisplay(theIncludeConcept.getDisplay()); 1029 } 1030 } 1031 boolean added = addCodeIfNotAlreadyAdded(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion); 1032 if (added) { 1033 delta++; 1034 } 1035 } 1036 1037 ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis()); 1038 theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta); 1039 1040 if (resultsInBatch < maxResultsPerBatch) { 1041 ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis()); 1042 return false; 1043 } else { 1044 return true; 1045 } 1046 } 1047 1048 private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet(ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) { 1049 return theIncludeOrExclude 1050 .getConcept() 1051 .stream().filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode())) 1052 .findFirst() 1053 .orElse(null); 1054 } 1055 1056 /** 1057 * Helper method which builds a predicate for the expansion 1058 */ 1059 private Optional<PredicateFinalStep> buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) { 1060 if (CollectionUtils.isEmpty(theCodes)) { return Optional.empty(); } 1061 1062 if (theCodes.size() < BooleanQuery.getMaxClauseCount()) { 1063 return Optional.of(thePredicate.simpleQueryString() 1064 .field( "myCode" ).matching( String.join(" | ", theCodes)) ); 1065 } 1066 1067 // Number of codes is larger than maxClauseCount, so we split the query in several clauses 1068 1069 // partition codes in lists of BooleanQuery.getMaxClauseCount() size 1070 List<List<String>> listOfLists = ListUtils.partition(theCodes, BooleanQuery.getMaxClauseCount()); 1071 1072 PredicateFinalStep step = thePredicate.bool(b -> { 1073 b.minimumShouldMatchNumber(1); 1074 for (List<String> codeList : listOfLists) { 1075 b.should(p -> p.simpleQueryString().field("myCode").matching(String.join(" | ", codeList))); 1076 } 1077 }); 1078 1079 return Optional.of(step); 1080 } 1081 1082 1083 private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) { 1084 String codeSystemUrlAndVersion; 1085 if (theIncludeOrExcludeVersion != null) { 1086 codeSystemUrlAndVersion = theSystem + "|" + theIncludeOrExcludeVersion; 1087 } else { 1088 codeSystemUrlAndVersion = theSystem; 1089 } 1090 return codeSystemUrlAndVersion; 1091 } 1092 1093 private @Nonnull 1094 ValueSetExpansionOptions provideExpansionOptions(@Nullable ValueSetExpansionOptions theExpansionOptions) { 1095 if (theExpansionOptions != null) { 1096 return theExpansionOptions; 1097 } else { 1098 return DEFAULT_EXPANSION_OPTIONS; 1099 } 1100 } 1101 1102 private void addOrRemoveCode(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theSystem, String theCode, String theDisplay, String theSystemVersion) { 1103 if (theAdd && theAddedCodes.add(theSystem + "|" + theCode)) { 1104 theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion); 1105 } 1106 if (!theAdd && theAddedCodes.remove(theSystem + "|" + theCode)) { 1107 theValueSetCodeAccumulator.excludeConcept(theSystem, theCode); 1108 } 1109 } 1110 1111 private void handleFilter(String theCodeSystemIdentifier, SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) { 1112 if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) { 1113 return; 1114 } 1115 1116 if (isBlank(theFilter.getValue()) || theFilter.getOp() == null || isBlank(theFilter.getProperty())) { 1117 throw new InvalidRequestException(Msg.code(891) + "Invalid filter, must have fields populated: property op value"); 1118 } 1119 1120 switch (theFilter.getProperty()) { 1121 case "display:exact": 1122 case "display": 1123 handleFilterDisplay(theF, theB, theFilter); 1124 break; 1125 case "concept": 1126 case "code": 1127 handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter); 1128 break; 1129 case "parent": 1130 case "child": 1131 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1132 handleFilterLoincParentChild(theF, theB, theFilter); 1133 break; 1134 case "ancestor": 1135 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1136 handleFilterLoincAncestor(theCodeSystemIdentifier, theF, theB, theFilter); 1137 break; 1138 case "descendant": 1139 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1140 handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter); 1141 break; 1142 case "copyright": 1143 isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty()); 1144 handleFilterLoincCopyright(theF, theB, theFilter); 1145 break; 1146 default: 1147 if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) { 1148 handleFilterRegex(theF, theB, theFilter); 1149 } else { 1150 handleFilterPropertyDefault(theF, theB, theFilter); 1151 } 1152 break; 1153 } 1154 } 1155 1156 private void handleFilterPropertyDefault(SearchPredicateFactory theF, 1157 BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) { 1158 1159 theB.must(getPropertyNameValueNestedPredicate(theF, theFilter.getProperty(), theFilter.getValue())); 1160 } 1161 1162 1163 private void handleFilterRegex(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) { 1164 /* 1165 * We treat the regex filter as a match on the regex 1166 * anywhere in the property string. The spec does not 1167 * say whether or not this is the right behaviour, but 1168 * there are examples that seem to suggest that it is. 1169 */ 1170 String value = theFilter.getValue(); 1171 if (value.endsWith("$")) { 1172 value = value.substring(0, value.length() - 1); 1173 } else if (!value.endsWith(".*")) { 1174 value = value + ".*"; 1175 } 1176 if (!value.startsWith("^") && !value.startsWith(".*")) { 1177 value = ".*" + value; 1178 } else if (value.startsWith("^")) { 1179 value = value.substring(1); 1180 } 1181 1182 if (isFullTextSetToUseElastic()) { 1183 ElasticsearchNestedQueryBuilderUtil nestedQueryBuildUtil = new ElasticsearchNestedQueryBuilderUtil( 1184 "myProperties", "myKey", theFilter.getProperty(), 1185 "myValueString", value); 1186 1187 JsonObject nestedQueryJO = nestedQueryBuildUtil.toGson(); 1188 1189 ourLog.debug("Build nested Elasticsearch query: {}", nestedQueryJO); 1190 theB.must(theF.extension(ElasticsearchExtension.get()).fromJson(nestedQueryJO)); 1191 return; 1192 1193 } 1194 1195 // native Lucene configured 1196 Query termPropKeyQuery = new TermQuery(new Term(IDX_PROP_KEY, theFilter.getProperty())); 1197 Query regexpValueQuery = new RegexpQuery(new Term(IDX_PROP_VALUE_STRING, value)); 1198 1199 theB.must(theF.nested().objectField("myProperties").nest( 1200 theF.bool() 1201 .must(theF.extension(LuceneExtension.get()).fromLuceneQuery(termPropKeyQuery)) 1202 .must(theF.extension(LuceneExtension.get()).fromLuceneQuery(regexpValueQuery)) 1203 )); 1204 } 1205 1206 1207 private void handleFilterLoincCopyright(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) { 1208 if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1209 1210 String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase(); 1211 switch (copyrightFilterValue) { 1212 case "3rdparty": 1213 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1214 addFilterLoincCopyright3rdParty(theF, theB, theFilter); 1215 break; 1216 case "loinc": 1217 logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1218 addFilterLoincCopyrightLoinc(theF, theB, theFilter); 1219 break; 1220 default: 1221 throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty()); 1222 } 1223 1224 } else { 1225 throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty()); 1226 } 1227 } 1228 1229 private void addFilterLoincCopyrightLoinc(SearchPredicateFactory theF, 1230 BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) { 1231 1232 theB.mustNot(theF.match().field(IDX_PROP_KEY).matching("EXTERNAL_COPYRIGHT_NOTICE")); 1233 } 1234 1235 1236 private void addFilterLoincCopyright3rdParty(SearchPredicateFactory thePredicateFactory, 1237 BooleanPredicateClausesStep<?> theBooleanClause, ValueSet.ConceptSetFilterComponent theFilter) { 1238 //TODO GGG HS These used to be Term term = new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE", ".*");, which was lucene-specific. 1239 //TODO GGG HS ask diederik if this is equivalent. 1240 //This old .* regex is the same as an existence check on a field, which I've implemented here. 1241// theBooleanClause.must(thePredicateFactory.exists().field("EXTERNAL_COPYRIGHT_NOTICE")); 1242 1243 theBooleanClause.must(thePredicateFactory.match().field(IDX_PROP_KEY).matching("EXTERNAL_COPYRIGHT_NOTICE")); 1244 } 1245 1246 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1247 private void handleFilterLoincAncestor(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1248 switch (theFilter.getOp()) { 1249 case EQUAL: 1250 addLoincFilterAncestorEqual(theSystem, f, b, theFilter); 1251 break; 1252 case IN: 1253 addLoincFilterAncestorIn(theSystem, f, b, theFilter); 1254 break; 1255 default: 1256 throw new InvalidRequestException(Msg.code(892) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty()); 1257 } 1258 1259 } 1260 1261 private void addLoincFilterAncestorEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1262 long parentPid = getAncestorCodePid(theSystem, theFilter.getProperty(), theFilter.getValue()); 1263 b.must(f.match().field("myParentPids").matching(String.valueOf(parentPid))); 1264 } 1265 1266 1267 private void addLoincFilterAncestorIn(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1268 String[] values = theFilter.getValue().split(","); 1269 List<Long> ancestorCodePidList = new ArrayList<>(); 1270 for (String value : values) { 1271 ancestorCodePidList.add(getAncestorCodePid(theSystem, theFilter.getProperty(), value)); 1272 } 1273 1274 b.must(f.bool(innerB -> ancestorCodePidList.forEach( 1275 ancestorPid -> innerB.should(f.match().field("myParentPids").matching(String.valueOf(ancestorPid))) 1276 ))); 1277 } 1278 1279 1280 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1281 private void handleFilterLoincParentChild(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1282 switch (theFilter.getOp()) { 1283 case EQUAL: 1284 addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue()); 1285 break; 1286 case IN: 1287 addLoincFilterParentChildIn(f, b, theFilter); 1288 break; 1289 default: 1290 throw new InvalidRequestException(Msg.code(893) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty()); 1291 } 1292 } 1293 1294 private void addLoincFilterParentChildIn(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1295 String[] values = theFilter.getValue().split(","); 1296 b.minimumShouldMatchNumber(1); 1297 for (String value : values) { 1298 logFilteringValueOnProperty(value, theFilter.getProperty()); 1299 b.should(getPropertyNameValueNestedPredicate(f, theFilter.getProperty(), value)); 1300 } 1301 } 1302 1303 private void addLoincFilterParentChildEqual(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) { 1304 logFilteringValueOnProperty(theValue, theProperty); 1305 b.must(getPropertyNameValueNestedPredicate(f, theProperty, theValue)); 1306 } 1307 1308 /** 1309 * A nested predicate is required for both predicates to be applied to same property, otherwise if properties 1310 * propAA:valueAA and propBB:valueBB are defined, a search for propAA:valueBB would be a match 1311 * @see "https://docs.jboss.org/hibernate/search/6.0/reference/en-US/html_single/#search-dsl-predicate-nested" 1312 */ 1313 private PredicateFinalStep getPropertyNameValueNestedPredicate(SearchPredicateFactory f, String theProperty, String theValue) { 1314 return f.nested().objectField(IDX_PROPERTIES).nest( 1315 f.bool() 1316 .must(f.match().field(IDX_PROP_KEY).matching(theProperty)) 1317 .must(f.match().field(IDX_PROP_VALUE_STRING).field(IDX_PROP_DISPLAY_STRING).matching(theValue)) 1318 ); 1319 } 1320 1321 1322 private void handleFilterConceptAndCode(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1323 TermConcept code = findCodeForFilterCriteria(theSystem, theFilter); 1324 1325 if (theFilter.getOp() == ValueSet.FilterOperator.ISA) { 1326 ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay()); 1327 1328 b.must(f.match().field("myParentPids").matching("" + code.getId())); 1329 } else { 1330 throwInvalidFilter(theFilter, ""); 1331 } 1332 } 1333 1334 @Nonnull 1335 private TermConcept findCodeForFilterCriteria(String theSystem, ValueSet.ConceptSetFilterComponent theFilter) { 1336 return findCode(theSystem, theFilter.getValue()) 1337 .orElseThrow(() -> new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue())); 1338 } 1339 1340 private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) { 1341 throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty() + theErrorSuffix); 1342 } 1343 1344 private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) { 1345 String systemUrl = getUrlFromIdentifier(theSystemIdentifier); 1346 if (!isCodeSystemLoinc(systemUrl)) { 1347 throw new InvalidRequestException(Msg.code(895) + "Invalid filter, property " + theProperty + " is LOINC-specific and cannot be used with system: " + systemUrl); 1348 } 1349 } 1350 1351 private boolean isCodeSystemLoinc(String theSystem) { 1352 return LOINC_URI.equals(theSystem); 1353 } 1354 1355 private void handleFilterDisplay(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1356 if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1357 addDisplayFilterExact(f, b, theFilter); 1358 } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) { 1359 if (theFilter.getValue().trim().contains(" ")) { 1360 addDisplayFilterExact(f, b, theFilter); 1361 } else { 1362 addDisplayFilterInexact(f, b, theFilter); 1363 } 1364 } 1365 } 1366 1367 private void addDisplayFilterExact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) { 1368 bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue())); 1369 } 1370 1371 private void addDisplayFilterInexact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) { 1372 bool.must(f.phrase() 1373 .field("myDisplay").boost(4.0f) 1374 .field("myDisplayWordEdgeNGram").boost(1.0f) 1375 .field("myDisplayEdgeNGram").boost(1.0f) 1376 .matching(nextFilter.getValue().toLowerCase()) 1377 .slop(2) 1378 ); 1379 } 1380 1381 private long getAncestorCodePid(String theSystem, String theProperty, String theValue) { 1382 TermConcept code = findCode(theSystem, theValue) 1383 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1384 1385 logFilteringValueOnProperty(theValue, theProperty); 1386 return code.getId(); 1387 } 1388 1389 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 1390 private void handleFilterLoincDescendant(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1391 switch (theFilter.getOp()) { 1392 case EQUAL: 1393 addLoincFilterDescendantEqual(theSystem, f, b, theFilter); 1394 break; 1395 case IN: 1396 addLoincFilterDescendantIn(theSystem, f, b, theFilter); 1397 break; 1398 default: 1399 throw new InvalidRequestException(Msg.code(896) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty()); 1400 } 1401 } 1402 1403 1404 private void addLoincFilterDescendantEqual(String theSystem, SearchPredicateFactory f, 1405 BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1406 1407 List<Long> parentPids = getCodeParentPids(theSystem, theFilter.getProperty(), theFilter.getValue()); 1408 if (parentPids.isEmpty()) { 1409 // Can't return empty must, because it wil match according to other predicates. 1410 // Some day there will be a 'matchNone' predicate (https://discourse.hibernate.org/t/fail-fast-predicate/6062) 1411 b.mustNot( f.matchAll() ); 1412 return; 1413 } 1414 1415 b.must(f.bool(innerB -> { 1416 innerB.minimumShouldMatchNumber(1); 1417 parentPids.forEach(pid -> innerB.should(f.match().field("myId").matching(pid))); 1418 })); 1419 1420 } 1421 1422 /** 1423 * We are looking for codes which have codes indicated in theFilter.getValue() as descendants. 1424 * Strategy is to find codes which have their pId(s) in the list of the parentId(s) of all the TermConcept(s) 1425 * representing the codes in theFilter.getValue() 1426 */ 1427 private void addLoincFilterDescendantIn(String theSystem, SearchPredicateFactory f, 1428 BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) { 1429 1430 String[] values = theFilter.getValue().split(","); 1431 if (values.length == 0) { 1432 throw new InvalidRequestException(Msg.code(2062) + "Invalid filter criteria - no codes specified"); 1433 } 1434 1435 List<Long> descendantCodePidList = getMultipleCodeParentPids(theSystem, theFilter.getProperty(), values); 1436 1437 b.must(f.bool(innerB -> descendantCodePidList.forEach( 1438 pId -> innerB.should(f.match().field("myId").matching(pId)) 1439 ))); 1440 } 1441 1442 1443 /** 1444 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1445 */ 1446 private List<Long> getCodeParentPids(String theSystem, String theProperty, String theValue) { 1447 TermConcept code = findCode(theSystem, theValue) 1448 .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + 1449 Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue)); 1450 1451 String[] parentPids = code.getParentPidsAsString().split(" "); 1452 List<Long> retVal = Arrays.stream(parentPids) 1453 .filter( pid -> !StringUtils.equals(pid, "NONE") ) 1454 .map(Long::parseLong) 1455 .collect(Collectors.toList()); 1456 logFilteringValueOnProperty(theValue, theProperty); 1457 return retVal; 1458 } 1459 1460 1461 /** 1462 * Returns the list of parentId(s) of the TermConcept representing theValue as a code 1463 */ 1464 private List<Long> getMultipleCodeParentPids(String theSystem, String theProperty, String[] theValues) { 1465 List<String> valuesList = Arrays.asList(theValues); 1466 List<TermConcept> termConcepts = findCodes(theSystem, valuesList); 1467 if (valuesList.size() != termConcepts.size()) { 1468 String exMsg = getTermConceptsFetchExceptionMsg(termConcepts, valuesList); 1469 throw new InvalidRequestException(Msg.code(2064) + "Invalid filter criteria - {" + 1470 Constants.codeSystemWithDefaultDescription(theSystem) + "}: " + exMsg); 1471 } 1472 1473 List<Long> retVal = termConcepts.stream() 1474 .flatMap(tc -> Arrays.stream(tc.getParentPidsAsString().split(" "))) 1475 .filter( pid -> !StringUtils.equals(pid, "NONE") ) 1476 .map(Long::parseLong) 1477 .collect(Collectors.toList()); 1478 1479 logFilteringValueOnProperties(valuesList, theProperty); 1480 1481 return retVal; 1482 } 1483 1484 /** 1485 * Generate message indicating for which of theValues a TermConcept was not found 1486 */ 1487 private String getTermConceptsFetchExceptionMsg(List<TermConcept> theTermConcepts, List<String> theValues) { 1488 // case: more TermConcept(s) retrieved than codes queried 1489 if (theTermConcepts.size() > theValues.size()) { 1490 return "Invalid filter criteria - More TermConcepts were found than indicated codes. Queried codes: [" + 1491 join(",", theValues + "]; Obtained TermConcept IDs, codes: [" + 1492 theTermConcepts.stream().map(tc -> tc.getId() + ", " + tc.getCode()) 1493 .collect(joining("; "))+ "]"); 1494 } 1495 1496 // case: less TermConcept(s) retrieved than codes queried 1497 Set<String> matchedCodes = theTermConcepts.stream().map(TermConcept::getCode).collect(toSet()); 1498 List<String> notMatchedValues = theValues.stream() 1499 .filter(v -> ! matchedCodes.contains (v)) .collect(toList()); 1500 1501 return "Invalid filter criteria - No TermConcept(s) were found for the requested codes: [" + 1502 join(",", notMatchedValues + "]"); 1503 } 1504 1505 1506 private void logFilteringValueOnProperty(String theValue, String theProperty) { 1507 ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty); 1508 } 1509 1510 private void logFilteringValueOnProperties(List<String> theValues, String theProperty) { 1511 ourLog.debug(" * Filtering with values={} on property {}", String.join(", ", theValues), theProperty); 1512 } 1513 1514 private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) { 1515 throw new InvalidRequestException(Msg.code(897) + "Don't know how to handle op=" + theOp + " on property " + theProperty); 1516 } 1517 1518 private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) { 1519 throw new InvalidRequestException(Msg.code(898) + "Don't know how to handle value=" + theValue + " on property " + theProperty); 1520 } 1521 1522 private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd) { 1523 ourLog.trace("Hibernate search is not enabled"); 1524 1525 if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) { 1526 Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server."); 1527 } 1528 1529 Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server."); 1530 1531 for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) { 1532 boolean handled = false; 1533 switch (nextFilter.getProperty()) { 1534 case "concept": 1535 case "code": 1536 if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) { 1537 theValueSetCodeAccumulator.addMessage("Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient."); 1538 TermConcept code = findCodeForFilterCriteria(theSystem, nextFilter); 1539 addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code); 1540 handled = true; 1541 } 1542 break; 1543 } 1544 1545 if (!handled) { 1546 throwInvalidFilter(nextFilter, " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion funtionality is available."); 1547 } 1548 } 1549 1550 if (theInclude.getConcept().isEmpty()) { 1551 1552 Collection<TermConcept> concepts = myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid()); 1553 for (TermConcept next : concepts) { 1554 addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), next.getId(), next.getParentPidsAsString(), next.getDesignations()); 1555 } 1556 } 1557 1558 for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) { 1559 if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) { 1560 continue; 1561 } 1562 Collection<TermConceptDesignation> designations = next 1563 .getDesignation() 1564 .stream() 1565 .map(t->new TermConceptDesignation() 1566 .setValue(t.getValue()) 1567 .setLanguage(t.getLanguage()) 1568 .setUseCode(t.getUse().getCode()) 1569 .setUseSystem(t.getUse().getSystem()) 1570 .setUseDisplay(t.getUse().getDisplay()) 1571 ) 1572 .collect(Collectors.toList()); 1573 addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), null, null, designations); 1574 } 1575 1576 1577 } 1578 1579 private void addConceptAndChildren(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, TermConcept theConcept) { 1580 for (TermConcept nextChild : theConcept.getChildCodes()) { 1581 boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), nextChild.getCode(), nextChild.getDisplay(), nextChild.getId(), nextChild.getParentPidsAsString(), nextChild.getDesignations()); 1582 if (added) { 1583 addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild); 1584 } 1585 } 1586 } 1587 1588 @Override 1589 @Transactional 1590 public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) { 1591 IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails); 1592 ValueSet canonicalValueSet = toCanonicalValueSet(valueSet); 1593 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet); 1594 if (!optionalTermValueSet.isPresent()) { 1595 return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId); 1596 } 1597 1598 ourLog.info("Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl()); 1599 1600 TermValueSet termValueSet = optionalTermValueSet.get(); 1601 if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) { 1602 return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetCantInvalidateNotYetPrecalculated", termValueSet.getUrl(), termValueSet.getExpansionStatus()); 1603 } 1604 1605 Long totalConcepts = termValueSet.getTotalConcepts(); 1606 1607 deletePreCalculatedValueSetContents(termValueSet); 1608 1609 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 1610 termValueSet.setExpansionTimestamp(null); 1611 myTermValueSetDao.save(termValueSet); 1612 return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts); 1613 } 1614 1615 @Override 1616 @Transactional 1617 public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) { 1618 Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet); 1619 1620 if (!optionalTermValueSet.isPresent()) { 1621 ourLog.warn("ValueSet is not present in terminology tables. Will perform in-memory code validation. {}", getValueSetInfo(theValueSet)); 1622 return false; 1623 } 1624 1625 TermValueSet termValueSet = optionalTermValueSet.get(); 1626 1627 if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) { 1628 ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}", 1629 getValueSetInfo(theValueSet), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription()); 1630 return false; 1631 } 1632 1633 return true; 1634 } 1635 1636 private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) { 1637 ResourcePersistentId valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 1638 Optional<TermValueSet> optionalTermValueSet = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong()); 1639 return optionalTermValueSet; 1640 } 1641 1642 private ResourcePersistentId getValueSetResourcePersistentId(ValueSet theValueSet) { 1643 ResourcePersistentId valueSetResourcePid = myIdHelperService.resolveResourcePersistentIds(RequestPartitionId.allPartitions(), theValueSet.getIdElement().getResourceType(), theValueSet.getIdElement().getIdPart()); 1644 return valueSetResourcePid; 1645 } 1646 1647 protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet( 1648 ConceptValidationOptions theValidationOptions, 1649 ValueSet theValueSet, String theSystem, String theCode, String theDisplay, Coding theCoding, CodeableConcept theCodeableConcept) { 1650 assert TransactionSynchronizationManager.isSynchronizationActive(); 1651 1652 ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required"); 1653 ResourcePersistentId valueSetResourcePid = getValueSetResourcePersistentId(theValueSet); 1654 1655 1656 List<TermValueSetConcept> concepts = new ArrayList<>(); 1657 if (isNotBlank(theCode)) { 1658 if (theValidationOptions.isInferSystem()) { 1659 concepts.addAll(myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getIdAsLong(), theCode)); 1660 } else if (isNotBlank(theSystem)) { 1661 concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode)); 1662 } 1663 } else if (theCoding != null) { 1664 if (theCoding.hasSystem() && theCoding.hasCode()) { 1665 concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theCoding.getSystem(), theCoding.getCode())); 1666 } 1667 } else if (theCodeableConcept != null) { 1668 for (Coding coding : theCodeableConcept.getCoding()) { 1669 if (coding.hasSystem() && coding.hasCode()) { 1670 concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, coding.getSystem(), coding.getCode())); 1671 if (!concepts.isEmpty()) { 1672 break; 1673 } 1674 } 1675 } 1676 } else { 1677 return null; 1678 } 1679 1680 TermValueSet valueSetEntity = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong()).orElseThrow(() -> new IllegalStateException()); 1681 String timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity); 1682 String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription); 1683 1684 if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) { 1685 String systemVersion = null; 1686 for (TermValueSetConcept concept : concepts) { 1687 systemVersion = concept.getSystemVersion(); 1688 if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) { 1689 return new IValidationSupport.CodeValidationResult() 1690 .setCode(concept.getCode()) 1691 .setDisplay(concept.getDisplay()) 1692 .setCodeSystemVersion(concept.getSystemVersion()) 1693 .setMessage(msg); 1694 } 1695 } 1696 1697 return createFailureCodeValidationResult(theSystem, theCode, systemVersion, " - Concept Display \"" + theDisplay + "\" does not match expected \"" + concepts.get(0).getDisplay() + "\". " + msg).setDisplay(concepts.get(0).getDisplay()); 1698 } 1699 1700 if (!concepts.isEmpty()) { 1701 return new IValidationSupport.CodeValidationResult() 1702 .setCode(concepts.get(0).getCode()) 1703 .setDisplay(concepts.get(0).getDisplay()) 1704 .setCodeSystemVersion(concepts.get(0).getSystemVersion()) 1705 .setMessage(msg); 1706 } 1707 1708 // Ok, we failed 1709 List<TermValueSetConcept> outcome = myValueSetConceptDao.findByTermValueSetIdSystemOnly(Pageable.ofSize(1), valueSetEntity.getId(), theSystem); 1710 String append; 1711 if (outcome.size() == 0) { 1712 append = " - No codes in ValueSet belong to CodeSystem with URL " + theSystem; 1713 } else { 1714 append = " - Unknown code " + theSystem + "#" + theCode + ". " + msg; 1715 } 1716 1717 return createFailureCodeValidationResult(theSystem, theCode, null, append); 1718 } 1719 1720 private CodeValidationResult createFailureCodeValidationResult(String theSystem, String theCode, String theCodeSystemVersion, String theAppend) { 1721 return new CodeValidationResult() 1722 .setSeverity(IssueSeverity.ERROR) 1723 .setCodeSystemVersion(theCodeSystemVersion) 1724 .setMessage("Unable to validate code " + theSystem + "#" + theCode + theAppend); 1725 } 1726 1727 private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(ResourcePersistentId theResourcePid, String theSystem, String theCode) { 1728 assert TransactionSynchronizationManager.isSynchronizationActive(); 1729 1730 List<TermValueSetConcept> retVal = new ArrayList<>(); 1731 Optional<TermValueSetConcept> optionalTermValueSetConcept; 1732 int versionIndex = theSystem.indexOf("|"); 1733 if (versionIndex >= 0) { 1734 String systemUrl = theSystem.substring(0, versionIndex); 1735 String systemVersion = theSystem.substring(versionIndex + 1); 1736 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion(theResourcePid.getIdAsLong(), systemUrl, systemVersion, theCode); 1737 } else { 1738 optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode(theResourcePid.getIdAsLong(), theSystem, theCode); 1739 } 1740 optionalTermValueSetConcept.ifPresent(retVal::add); 1741 return retVal; 1742 } 1743 1744 private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 1745 for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) { 1746 TermConcept nextChild = nextChildLink.getChild(); 1747 if (addToSet(theSetToPopulate, nextChild)) { 1748 fetchChildren(nextChild, theSetToPopulate); 1749 } 1750 } 1751 } 1752 1753 private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) { 1754 TermCodeSystemVersion codeSystem = myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid); 1755 return myConceptDao.findByCodeSystemAndCode(codeSystem, theCode); 1756 } 1757 1758 private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) { 1759 for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) { 1760 TermConcept nextChild = nextChildLink.getParent(); 1761 if (addToSet(theSetToPopulate, nextChild)) { 1762 fetchParents(nextChild, theSetToPopulate); 1763 } 1764 } 1765 } 1766 1767 private CodeSystem.ConceptDefinitionComponent findCode(List<CodeSystem.ConceptDefinitionComponent> theConcepts, String theCode) { 1768 for (CodeSystem.ConceptDefinitionComponent next : theConcepts) { 1769 if (theCode.equals(next.getCode())) { 1770 return next; 1771 } 1772 CodeSystem.ConceptDefinitionComponent val = findCode(next.getConcept(), theCode); 1773 if (val != null) { 1774 return val; 1775 } 1776 } 1777 return null; 1778 } 1779 1780 @Override 1781 public Optional<TermConcept> findCode(String theCodeSystem, String theCode) { 1782 /* 1783 * Loading concepts without a transaction causes issues later on some 1784 * platforms (e.g. PSQL) so this transactiontemplate is here to make 1785 * sure that we always call this with an open transaction 1786 */ 1787 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 1788 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_MANDATORY); 1789 1790 return txTemplate.execute(t -> { 1791 TermCodeSystemVersion csv = getCurrentCodeSystemVersion(theCodeSystem); 1792 if (csv == null) { 1793 return Optional.empty(); 1794 } 1795 return myConceptDao.findByCodeSystemAndCode(csv, theCode); 1796 }); 1797 } 1798 1799 1800 @Transactional(propagation = Propagation.MANDATORY) 1801 public List<TermConcept> findCodes(String theCodeSystem, List<String> theCodeList) { 1802 TermCodeSystemVersion csv = getCurrentCodeSystemVersion(theCodeSystem); 1803 if (csv == null) { return Collections.emptyList(); } 1804 1805 return myConceptDao.findByCodeSystemAndCodeList(csv, theCodeList); 1806 } 1807 1808 1809 @Nullable 1810 private TermCodeSystemVersion getCurrentCodeSystemVersion(String theCodeSystemIdentifier) { 1811 String version = getVersionFromIdentifier(theCodeSystemIdentifier); 1812 TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theCodeSystemIdentifier, t -> myTxTemplate.execute(tx -> { 1813 TermCodeSystemVersion csv = null; 1814 TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier)); 1815 if (cs != null) { 1816 if (version != null) { 1817 csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version); 1818 } else if (cs.getCurrentVersion() != null) { 1819 csv = cs.getCurrentVersion(); 1820 } 1821 } 1822 if (csv != null) { 1823 return csv; 1824 } else { 1825 return NO_CURRENT_VERSION; 1826 } 1827 })); 1828 if (retVal == NO_CURRENT_VERSION) { 1829 return null; 1830 } 1831 return retVal; 1832 } 1833 1834 private String getVersionFromIdentifier(String theUri) { 1835 String retVal = null; 1836 if (StringUtils.isNotEmpty((theUri))) { 1837 int versionSeparator = theUri.lastIndexOf('|'); 1838 if (versionSeparator != -1) { 1839 retVal = theUri.substring(versionSeparator + 1); 1840 } 1841 } 1842 return retVal; 1843 } 1844 1845 private String getUrlFromIdentifier(String theUri) { 1846 String retVal = theUri; 1847 if (StringUtils.isNotEmpty((theUri))) { 1848 int versionSeparator = theUri.lastIndexOf('|'); 1849 if (versionSeparator != -1) { 1850 retVal = theUri.substring(0, versionSeparator); 1851 } 1852 } 1853 return retVal; 1854 } 1855 1856 @Transactional(propagation = Propagation.REQUIRED) 1857 @Override 1858 public Set<TermConcept> findCodesAbove(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 1859 StopWatch stopwatch = new StopWatch(); 1860 1861 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 1862 if (!concept.isPresent()) { 1863 return Collections.emptySet(); 1864 } 1865 1866 Set<TermConcept> retVal = new HashSet<>(); 1867 retVal.add(concept.get()); 1868 1869 fetchParents(concept.get(), retVal); 1870 1871 ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis()); 1872 return retVal; 1873 } 1874 1875 @Transactional 1876 @Override 1877 public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) { 1878 TermCodeSystem cs = getCodeSystem(theSystem); 1879 if (cs == null) { 1880 return findCodesAboveUsingBuiltInSystems(theSystem, theCode); 1881 } 1882 TermCodeSystemVersion csv = cs.getCurrentVersion(); 1883 1884 Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode); 1885 return toVersionIndependentConcepts(theSystem, codes); 1886 } 1887 1888 @Transactional(propagation = Propagation.REQUIRED) 1889 @Override 1890 public Set<TermConcept> findCodesBelow(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) { 1891 Stopwatch stopwatch = Stopwatch.createStarted(); 1892 1893 Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode); 1894 if (!concept.isPresent()) { 1895 return Collections.emptySet(); 1896 } 1897 1898 Set<TermConcept> retVal = new HashSet<>(); 1899 retVal.add(concept.get()); 1900 1901 fetchChildren(concept.get(), retVal); 1902 1903 ourLog.debug("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS)); 1904 return retVal; 1905 } 1906 1907 @Transactional 1908 @Override 1909 public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) { 1910 TermCodeSystem cs = getCodeSystem(theSystem); 1911 if (cs == null) { 1912 return findCodesBelowUsingBuiltInSystems(theSystem, theCode); 1913 } 1914 TermCodeSystemVersion csv = cs.getCurrentVersion(); 1915 1916 Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode); 1917 return toVersionIndependentConcepts(theSystem, codes); 1918 } 1919 1920 private TermCodeSystem getCodeSystem(String theSystem) { 1921 return myCodeSystemDao.findByCodeSystemUri(theSystem); 1922 } 1923 1924 @PostConstruct 1925 public void start() { 1926 RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute(); 1927 rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class)); 1928 myTxTemplate = new TransactionTemplate(myTransactionManager, rules); 1929 scheduleJob(); 1930 } 1931 1932 public void scheduleJob() { 1933 // Register scheduled job to pre-expand ValueSets 1934 // In the future it would be great to make this a cluster-aware task somehow 1935 ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition(); 1936 vsJobDefinition.setId(getClass().getName()); 1937 vsJobDefinition.setJobClass(Job.class); 1938 mySchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition); 1939 } 1940 1941 @Override 1942 public synchronized void preExpandDeferredValueSetsToTerminologyTables() { 1943 if (!myDaoConfig.isEnableTaskPreExpandValueSets()) { 1944 return; 1945 } 1946 if (isNotSafeToPreExpandValueSets()) { 1947 ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded."); 1948 return; 1949 } 1950 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 1951 1952 while (true) { 1953 StopWatch sw = new StopWatch(); 1954 TermValueSet valueSetToExpand = txTemplate.execute(t -> { 1955 Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded(); 1956 if (!optionalTermValueSet.isPresent()) { 1957 return null; 1958 } 1959 1960 TermValueSet termValueSet = optionalTermValueSet.get(); 1961 termValueSet.setTotalConcepts(0L); 1962 termValueSet.setTotalConceptDesignations(0L); 1963 termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS); 1964 return myTermValueSetDao.saveAndFlush(termValueSet); 1965 }); 1966 if (valueSetToExpand == null) { 1967 return; 1968 } 1969 1970 // We have a ValueSet to pre-expand. 1971 setPreExpandingValueSets(true); 1972 try { 1973 ValueSet valueSet = txTemplate.execute(t -> { 1974 TermValueSet refreshedValueSetToExpand = myTermValueSetDao.findById(valueSetToExpand.getId()).orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId())); 1975 return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource()); 1976 }); 1977 assert valueSet != null; 1978 1979 ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myTermValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao); 1980 ValueSetExpansionOptions options = new ValueSetExpansionOptions(); 1981 options.setIncludeHierarchy(true); 1982 expandValueSet(options, valueSet, accumulator); 1983 1984 // We are done with this ValueSet. 1985 txTemplate.executeWithoutResult(t -> { 1986 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED); 1987 valueSetToExpand.setExpansionTimestamp(new Date()); 1988 myTermValueSetDao.saveAndFlush(valueSetToExpand); 1989 1990 }); 1991 1992 ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw); 1993 1994 } catch (Exception e) { 1995 ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e); 1996 txTemplate.executeWithoutResult(t -> { 1997 valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND); 1998 myTermValueSetDao.saveAndFlush(valueSetToExpand); 1999 2000 }); 2001 2002 } finally { 2003 setPreExpandingValueSets(false); 2004 } 2005 } 2006 } 2007 2008 private synchronized void setPreExpandingValueSets(boolean thePreExpandingValueSets) { 2009 myPreExpandingValueSets = thePreExpandingValueSets; 2010 } 2011 2012 private synchronized boolean isPreExpandingValueSets() { 2013 return myPreExpandingValueSets; 2014 } 2015 2016 @Override 2017 @Transactional 2018 public CodeValidationResult validateCode(ConceptValidationOptions theOptions, IIdType theValueSetId, String theValueSetIdentifier, String theCodeSystemIdentifierToValidate, String theCodeToValidate, String theDisplayToValidate, IBaseDatatype theCodingToValidate, IBaseDatatype theCodeableConceptToValidate) { 2019 2020 CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConceptToValidate); 2021 boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0; 2022 2023 Coding canonicalCodingToValidate = toCanonicalCoding(theCodingToValidate); 2024 boolean haveCoding = canonicalCodingToValidate != null && canonicalCodingToValidate.isEmpty() == false; 2025 2026 boolean haveCode = theCodeToValidate != null && theCodeToValidate.isEmpty() == false; 2027 2028 if (!haveCodeableConcept && !haveCoding && !haveCode) { 2029 throw new InvalidRequestException(Msg.code(899) + "No code, coding, or codeableConcept provided to validate"); 2030 } 2031 if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) { 2032 throw new InvalidRequestException(Msg.code(900) + "$validate-code can only validate (system AND code) OR (coding) OR (codeableConcept)"); 2033 } 2034 2035 boolean haveIdentifierParam = isNotBlank(theValueSetIdentifier); 2036 String valueSetIdentifier; 2037 if (theValueSetId != null) { 2038 IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId); 2039 StringBuilder valueSetIdentifierBuilder = new StringBuilder(CommonCodeSystemsTerminologyService.getValueSetUrl(valueSet)); 2040 String valueSetVersion = CommonCodeSystemsTerminologyService.getValueSetVersion(valueSet); 2041 if (valueSetVersion != null) { 2042 valueSetIdentifierBuilder.append("|").append(valueSetVersion); 2043 } 2044 valueSetIdentifier = valueSetIdentifierBuilder.toString(); 2045 2046 } else if (haveIdentifierParam) { 2047 valueSetIdentifier = theValueSetIdentifier; 2048 } else { 2049 throw new InvalidRequestException(Msg.code(901) + "Either ValueSet ID or ValueSet identifier or system and code must be provided. Unable to validate."); 2050 } 2051 2052 ValidationSupportContext validationContext = new ValidationSupportContext(provideValidationSupport()); 2053 2054 String codeValueToValidate = theCodeToValidate; 2055 String codeSystemIdentifierValueToValidate = theCodeSystemIdentifierToValidate; 2056 String codeDisplayValueToValidate = theDisplayToValidate; 2057 2058 if (haveCodeableConcept) { 2059 for (int i = 0; i < codeableConcept.getCoding().size(); i++) { 2060 Coding nextCoding = codeableConcept.getCoding().get(i); 2061 String codeSystemIdentifier; 2062 if (nextCoding.hasVersion()) { 2063 codeSystemIdentifier = nextCoding.getSystem() + "|" + nextCoding.getVersion(); 2064 } else { 2065 codeSystemIdentifier = nextCoding.getSystem(); 2066 } 2067 CodeValidationResult nextValidation = validateCode(validationContext, theOptions, codeSystemIdentifier, nextCoding.getCode(), nextCoding.getDisplay(), valueSetIdentifier); 2068 if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) { 2069 return nextValidation; 2070 } 2071 } 2072 } else if (haveCoding) { 2073 if (canonicalCodingToValidate.hasVersion()) { 2074 codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem() + "|" + canonicalCodingToValidate.getVersion(); 2075 } else { 2076 codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem(); 2077 } 2078 codeValueToValidate = canonicalCodingToValidate.getCode(); 2079 codeDisplayValueToValidate = canonicalCodingToValidate.getDisplay(); 2080 } 2081 2082 return validateCode(validationContext, theOptions, codeSystemIdentifierValueToValidate, codeValueToValidate, codeDisplayValueToValidate, valueSetIdentifier); 2083 } 2084 2085 private boolean isNotSafeToPreExpandValueSets() { 2086 return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty(); 2087 } 2088 2089 protected abstract ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable); 2090 2091 private Optional<TermValueSet> getNextTermValueSetNotExpanded() { 2092 Optional<TermValueSet> retVal = Optional.empty(); 2093 Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED); 2094 2095 if (!page.getContent().isEmpty()) { 2096 retVal = Optional.of(page.getContent().get(0)); 2097 } 2098 2099 return retVal; 2100 } 2101 2102 @Override 2103 @Transactional 2104 public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) { 2105 2106 ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied"); 2107 if (isPlaceholder(theValueSet)) { 2108 ourLog.info("Not storing TermValueSet for placeholder {}", theValueSet.getIdElement().toVersionless().getValueAsString()); 2109 return; 2110 } 2111 2112 ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theValueSet.getUrl(), "ValueSet has no value for ValueSet.url"); 2113 ourLog.info("Storing TermValueSet for {}", theValueSet.getIdElement().toVersionless().getValueAsString()); 2114 2115 /* 2116 * Get CodeSystem and validate CodeSystemVersion 2117 */ 2118 TermValueSet termValueSet = new TermValueSet(); 2119 termValueSet.setResource(theResourceTable); 2120 termValueSet.setUrl(theValueSet.getUrl()); 2121 termValueSet.setVersion(theValueSet.getVersion()); 2122 termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null); 2123 2124 // Delete version being replaced 2125 deleteValueSetForResource(theResourceTable); 2126 2127 /* 2128 * Do the upload. 2129 */ 2130 String url = termValueSet.getUrl(); 2131 String version = termValueSet.getVersion(); 2132 Optional<TermValueSet> optionalExistingTermValueSetByUrl; 2133 if (version != null) { 2134 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version); 2135 } else { 2136 optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url); 2137 } 2138 if (!optionalExistingTermValueSetByUrl.isPresent()) { 2139 2140 myTermValueSetDao.save(termValueSet); 2141 2142 } else { 2143 TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get(); 2144 String msg; 2145 if (version != null) { 2146 msg = myContext.getLocalizer().getMessage( 2147 BaseTermReadSvcImpl.class, 2148 "cannotCreateDuplicateValueSetUrlAndVersion", 2149 url, version, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue()); 2150 } else { 2151 msg = myContext.getLocalizer().getMessage( 2152 BaseTermReadSvcImpl.class, 2153 "cannotCreateDuplicateValueSetUrl", 2154 url, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue()); 2155 } 2156 throw new UnprocessableEntityException(Msg.code(902) + msg); 2157 } 2158 } 2159 2160 @Override 2161 @Transactional 2162 public IFhirResourceDaoCodeSystem.SubsumesResult subsumes(IPrimitiveType<String> theCodeA, IPrimitiveType<String> theCodeB, 2163 IPrimitiveType<String> theSystem, IBaseCoding theCodingA, IBaseCoding theCodingB) { 2164 FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA); 2165 FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB); 2166 2167 if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) { 2168 throw new InvalidRequestException(Msg.code(903) + "Unable to test subsumption across different code systems"); 2169 } 2170 2171 if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) { 2172 throw new InvalidRequestException(Msg.code(904) + "Unable to test subsumption across different code system versions"); 2173 } 2174 2175 String codeASystemIdentifier; 2176 if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) { 2177 codeASystemIdentifier = conceptA.getSystem() + "|" + conceptA.getSystemVersion(); 2178 } else { 2179 codeASystemIdentifier = conceptA.getSystem(); 2180 } 2181 TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode()) 2182 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA)); 2183 2184 String codeBSystemIdentifier; 2185 if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) { 2186 codeBSystemIdentifier = conceptB.getSystem() + "|" + conceptB.getSystemVersion(); 2187 } else { 2188 codeBSystemIdentifier = conceptB.getSystem(); 2189 } 2190 TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode()) 2191 .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB)); 2192 2193 SearchSession searchSession = Search.session(myEntityManager); 2194 2195 ConceptSubsumptionOutcome subsumes; 2196 subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES); 2197 if (subsumes == null) { 2198 subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY); 2199 } 2200 if (subsumes == null) { 2201 subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED; 2202 } 2203 2204 return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes); 2205 } 2206 2207 protected abstract ValueSet toCanonicalValueSet(IBaseResource theValueSet); 2208 2209 protected IValidationSupport.LookupCodeResult lookupCode(String theSystem, String theCode, String theDisplayLanguage) { 2210 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2211 return txTemplate.execute(t -> { 2212 Optional<TermConcept> codeOpt = findCode(theSystem, theCode); 2213 if (codeOpt.isPresent()) { 2214 TermConcept code = codeOpt.get(); 2215 2216 IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult(); 2217 result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName()); 2218 result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId()); 2219 result.setSearchedForSystem(theSystem); 2220 result.setSearchedForCode(theCode); 2221 result.setFound(true); 2222 result.setCodeDisplay(code.getDisplay()); 2223 2224 for (TermConceptDesignation next : code.getDesignations()) { 2225 // filter out the designation based on displayLanguage if any 2226 if (isDisplayLanguageMatch(theDisplayLanguage, next.getLanguage())) { 2227 IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation(); 2228 designation.setLanguage(next.getLanguage()); 2229 designation.setUseSystem(next.getUseSystem()); 2230 designation.setUseCode(next.getUseCode()); 2231 designation.setUseDisplay(next.getUseDisplay()); 2232 designation.setValue(next.getValue()); 2233 result.getDesignations().add(designation); 2234 } 2235 } 2236 2237 for (TermConceptProperty next : code.getProperties()) { 2238 if (next.getType() == TermConceptPropertyTypeEnum.CODING) { 2239 IValidationSupport.CodingConceptProperty property = new IValidationSupport.CodingConceptProperty(next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay()); 2240 result.getProperties().add(property); 2241 } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) { 2242 IValidationSupport.StringConceptProperty property = new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue()); 2243 result.getProperties().add(property); 2244 } else { 2245 throw new InternalErrorException(Msg.code(905) + "Unknown type: " + next.getType()); 2246 } 2247 } 2248 2249 return result; 2250 2251 } else { 2252 return new LookupCodeResult() 2253 .setFound(false); 2254 } 2255 }); 2256 } 2257 2258 @Nullable 2259 private ConceptSubsumptionOutcome testForSubsumption(SearchSession theSearchSession, TermConcept theLeft, TermConcept theRight, ConceptSubsumptionOutcome theOutput) { 2260 List<TermConcept> fetch = theSearchSession.search(TermConcept.class) 2261 .where(f -> f.bool() 2262 .must(f.match().field("myId").matching(theRight.getId())) 2263 .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId()))) 2264 ).fetchHits(1); 2265 2266 if (fetch.size() > 0) { 2267 return theOutput; 2268 } else { 2269 return null; 2270 } 2271 } 2272 2273 private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts(String theSystem, Set<TermConcept> codes) { 2274 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size()); 2275 for (TermConcept next : codes) { 2276 retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode())); 2277 } 2278 return retVal; 2279 } 2280 2281 @Override 2282 @Transactional 2283 public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) { 2284 invokeRunnableForUnitTest(); 2285 2286 IPrimitiveType<?> urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class); 2287 String url = urlPrimitive.getValueAsString(); 2288 if (isNotBlank(url)) { 2289 return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url); 2290 } 2291 return null; 2292 } 2293 2294 @CoverageIgnore 2295 @Override 2296 public IValidationSupport.CodeValidationResult validateCode(@Nonnull ValidationSupportContext theValidationSupportContext, @Nonnull ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl) { 2297 //TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS. 2298 invokeRunnableForUnitTest(); 2299 2300 if (isNotBlank(theValueSetUrl)) { 2301 return validateCodeInValueSet(theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystem, theCode, theDisplay); 2302 } 2303 2304 TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager); 2305 txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); 2306 Optional<FhirVersionIndependentConcept> codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new FhirVersionIndependentConcept(theCodeSystem, c.getCode()))); 2307 2308 if (codeOpt != null && codeOpt.isPresent()) { 2309 FhirVersionIndependentConcept code = codeOpt.get(); 2310 if (!theOptions.isValidateDisplay() || (isNotBlank(code.getDisplay()) && isNotBlank(theDisplay) && code.getDisplay().equals(theDisplay))) { 2311 return new CodeValidationResult() 2312 .setCode(code.getCode()) 2313 .setDisplay(code.getDisplay()); 2314 } else { 2315 return createFailureCodeValidationResult(theCodeSystem, theCode, code.getSystemVersion(), " - Concept Display \"" + code.getDisplay() + "\" does not match expected \"" + code.getDisplay() + "\"").setDisplay(code.getDisplay()); 2316 } 2317 } 2318 2319 return createFailureCodeValidationResult(theCodeSystem, theCode, null, " - Code can not be found in CodeSystem"); 2320 } 2321 2322 IValidationSupport.CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theValidationOptions, String theValueSetUrl, String theCodeSystem, String theCode, String theDisplay) { 2323 IBaseResource valueSet = theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl); 2324 CodeValidationResult retVal = null; 2325 2326 // If we don't have a PID, this came from some source other than the JPA 2327 // database, so we don't need to check if it's pre-expanded or not 2328 if (valueSet instanceof IAnyResource) { 2329 Long pid = IDao.RESOURCE_PID.get((IAnyResource) valueSet); 2330 if (pid != null) { 2331 TransactionTemplate txTemplate = new TransactionTemplate(myTxManager); 2332 retVal = txTemplate.execute(tx -> { 2333 if (isValueSetPreExpandedForCodeValidation(valueSet)) { 2334 return validateCodeIsInPreExpandedValueSet(theValidationOptions, valueSet, theCodeSystem, theCode, theDisplay, null, null); 2335 } else { 2336 return null; 2337 } 2338 }); 2339 } 2340 } 2341 2342 if (retVal == null) { 2343 if (valueSet != null) { 2344 retVal = new InMemoryTerminologyServerValidationSupport(myContext).validateCodeInValueSet(theValidationSupportContext, theValidationOptions, theCodeSystem, theCode, theDisplay, valueSet); 2345 } else { 2346 String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]"; 2347 retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append); 2348 } 2349 } 2350 2351 // Check if someone is accidentally using a VS url where it should be a CS URL 2352 if (retVal != null && retVal.getCode() == null && theCodeSystem != null) { 2353 if (isValueSetSupported(theValidationSupportContext, theCodeSystem)) { 2354 if (!isCodeSystemSupported(theValidationSupportContext, theCodeSystem)) { 2355 String newMessage = "Unable to validate code " + theCodeSystem + "#" + theCode + " - Supplied system URL is a ValueSet URL and not a CodeSystem URL, check if it is correct: " + theCodeSystem; 2356 retVal.setMessage(newMessage); 2357 } 2358 } 2359 } 2360 2361 return retVal; 2362 2363 } 2364 2365 @Override 2366 public IBaseResource fetchCodeSystem(String theSystem) { 2367 IValidationSupport jpaValidationSupport = provideJpaValidationSupport(); 2368 return jpaValidationSupport.fetchCodeSystem(theSystem); 2369 } 2370 2371 @Override 2372 public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) { 2373 IValidationSupport validationSupport = provideValidationSupport(); 2374 IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem); 2375 if (codeSystem != null) { 2376 codeSystem = toCanonicalCodeSystem(codeSystem); 2377 } 2378 return (CodeSystem) codeSystem; 2379 } 2380 2381 @Nonnull 2382 private IValidationSupport provideJpaValidationSupport() { 2383 IValidationSupport jpaValidationSupport = myJpaValidationSupport; 2384 if (jpaValidationSupport == null) { 2385 jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class); 2386 myJpaValidationSupport = jpaValidationSupport; 2387 } 2388 return jpaValidationSupport; 2389 } 2390 2391 @Nonnull 2392 protected IValidationSupport provideValidationSupport() { 2393 IValidationSupport validationSupport = myValidationSupport; 2394 if (validationSupport == null) { 2395 validationSupport = myApplicationContext.getBean(IValidationSupport.class); 2396 myValidationSupport = validationSupport; 2397 } 2398 return validationSupport; 2399 } 2400 2401 public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) { 2402 IValidationSupport validationSupport = provideValidationSupport(); 2403 IBaseResource valueSet = validationSupport.fetchValueSet(theSystem); 2404 if (valueSet != null) { 2405 valueSet = toCanonicalValueSet(valueSet); 2406 } 2407 return (ValueSet) valueSet; 2408 } 2409 2410 protected abstract CodeSystem toCanonicalCodeSystem(IBaseResource theCodeSystem); 2411 2412 @Override 2413 public IBaseResource fetchValueSet(String theValueSetUrl) { 2414 return provideJpaValidationSupport().fetchValueSet(theValueSetUrl); 2415 } 2416 2417 @Override 2418 public FhirContext getFhirContext() { 2419 return myContext; 2420 } 2421 2422 private void findCodesAbove(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) { 2423 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 2424 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 2425 addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate); 2426 } 2427 } 2428 2429 @Override 2430 public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) { 2431 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 2432 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 2433 if (system != null) { 2434 findCodesAbove(system, theSystem, theCode, retVal); 2435 } 2436 return retVal; 2437 } 2438 2439 private void findCodesBelow(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) { 2440 List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept(); 2441 findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList); 2442 } 2443 2444 private void findCodesBelow(String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate, List<CodeSystem.ConceptDefinitionComponent> conceptList) { 2445 for (CodeSystem.ConceptDefinitionComponent next : conceptList) { 2446 if (theCode.equals(next.getCode())) { 2447 addAllChildren(theSystemString, next, theListToPopulate); 2448 } else { 2449 findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept()); 2450 } 2451 } 2452 } 2453 2454 @Override 2455 public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) { 2456 ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(); 2457 CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem); 2458 if (system != null) { 2459 findCodesBelow(system, theSystem, theCode, retVal); 2460 } 2461 return retVal; 2462 } 2463 2464 private void addAllChildren(String theSystemString, CodeSystem.ConceptDefinitionComponent theCode, List<FhirVersionIndependentConcept> theListToPopulate) { 2465 if (isNotBlank(theCode.getCode())) { 2466 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode())); 2467 } 2468 for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) { 2469 addAllChildren(theSystemString, nextChild, theListToPopulate); 2470 } 2471 } 2472 2473 private boolean addTreeIfItContainsCode(String theSystemString, CodeSystem.ConceptDefinitionComponent theNext, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) { 2474 boolean foundCodeInChild = false; 2475 for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) { 2476 foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate); 2477 } 2478 2479 if (theCode.equals(theNext.getCode()) || foundCodeInChild) { 2480 theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode())); 2481 return true; 2482 } 2483 2484 return false; 2485 } 2486 2487 @Nullable 2488 protected abstract Coding toCanonicalCoding(@Nullable IBaseDatatype theCoding); 2489 2490 @Nullable 2491 protected abstract Coding toCanonicalCoding(@Nullable IBaseCoding theCoding); 2492 2493 @Nullable 2494 protected abstract CodeableConcept toCanonicalCodeableConcept(@Nullable IBaseDatatype theCodeableConcept); 2495 2496 @Nonnull 2497 private FhirVersionIndependentConcept toConcept(IPrimitiveType<String> theCodeType, IPrimitiveType<String> theCodeSystemIdentifierType, IBaseCoding theCodingType) { 2498 String code = theCodeType != null ? theCodeType.getValueAsString() : null; 2499 String system = theCodeSystemIdentifierType != null ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null; 2500 String systemVersion = theCodeSystemIdentifierType != null ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null; 2501 if (theCodingType != null) { 2502 Coding canonicalizedCoding = toCanonicalCoding(theCodingType); 2503 assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't 2504 code = canonicalizedCoding.getCode(); 2505 system = canonicalizedCoding.getSystem(); 2506 systemVersion = canonicalizedCoding.getVersion(); 2507 } 2508 return new FhirVersionIndependentConcept(system, code, null, systemVersion); 2509 } 2510 2511 @Override 2512 @Transactional 2513 public CodeValidationResult codeSystemValidateCode(IIdType theCodeSystemId, String theCodeSystemUrl, String theVersion, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) { 2514 2515 CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConcept); 2516 boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0; 2517 2518 Coding coding = toCanonicalCoding(theCoding); 2519 boolean haveCoding = coding != null && coding.isEmpty() == false; 2520 2521 boolean haveCode = theCode != null && theCode.isEmpty() == false; 2522 2523 if (!haveCodeableConcept && !haveCoding && !haveCode) { 2524 throw new InvalidRequestException(Msg.code(906) + "No code, coding, or codeableConcept provided to validate."); 2525 } 2526 if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) { 2527 throw new InvalidRequestException(Msg.code(907) + "$validate-code can only validate (code) OR (coding) OR (codeableConcept)"); 2528 } 2529 2530 boolean haveIdentifierParam = isNotBlank(theCodeSystemUrl); 2531 String codeSystemUrl; 2532 if (theCodeSystemId != null) { 2533 IBaseResource codeSystem = myDaoRegistry.getResourceDao("CodeSystem").read(theCodeSystemId); 2534 codeSystemUrl = CommonCodeSystemsTerminologyService.getCodeSystemUrl(codeSystem); 2535 } else if (haveIdentifierParam) { 2536 codeSystemUrl = theCodeSystemUrl; 2537 } else { 2538 throw new InvalidRequestException(Msg.code(908) + "Either CodeSystem ID or CodeSystem identifier must be provided. Unable to validate."); 2539 } 2540 2541 2542 String code = theCode; 2543 String display = theDisplay; 2544 2545 if (haveCodeableConcept) { 2546 for (int i = 0; i < codeableConcept.getCoding().size(); i++) { 2547 Coding nextCoding = codeableConcept.getCoding().get(i); 2548 if (nextCoding.hasSystem()) { 2549 if (!codeSystemUrl.equalsIgnoreCase(nextCoding.getSystem())) { 2550 throw new InvalidRequestException(Msg.code(909) + "Coding.system '" + nextCoding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate."); 2551 } 2552 codeSystemUrl = nextCoding.getSystem(); 2553 } 2554 code = nextCoding.getCode(); 2555 display = nextCoding.getDisplay(); 2556 CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, theVersion, code, display); 2557 if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) { 2558 return nextValidation; 2559 } 2560 } 2561 } else if (haveCoding) { 2562 if (coding.hasSystem()) { 2563 if (!codeSystemUrl.equalsIgnoreCase(coding.getSystem())) { 2564 throw new InvalidRequestException(Msg.code(910) + "Coding.system '" + coding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate."); 2565 } 2566 codeSystemUrl = coding.getSystem(); 2567 } 2568 code = coding.getCode(); 2569 display = coding.getDisplay(); 2570 } 2571 2572 return codeSystemValidateCode(codeSystemUrl, theVersion, code, display); 2573 } 2574 2575 /** 2576 * When the search is for unversioned loinc system it uses the forcedId to obtain the current 2577 * version, as it is not necessarily the last one anymore. 2578 * For other cases it keeps on considering the last uploaded as the current 2579 */ 2580 @Override 2581 public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) { 2582 if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) { 2583 Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl); 2584 if (!vsIdOpt.isPresent()) { 2585 return Optional.empty(); 2586 } 2587 2588 return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get()); 2589 } 2590 2591 List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl); 2592 if (termValueSetList.isEmpty()) { 2593 return Optional.empty(); 2594 } 2595 2596 return Optional.of(termValueSetList.get(0)); 2597 } 2598 2599 @SuppressWarnings("unchecked") 2600 private CodeValidationResult codeSystemValidateCode(String theCodeSystemUrl, String theCodeSystemVersion, String theCode, String theDisplay) { 2601 2602 CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder(); 2603 CriteriaQuery<TermConcept> query = criteriaBuilder.createQuery(TermConcept.class); 2604 Root<TermConcept> root = query.from(TermConcept.class); 2605 2606 Fetch<TermCodeSystemVersion, TermConcept> systemVersionFetch = root.fetch("myCodeSystem", JoinType.INNER); 2607 Join<TermCodeSystemVersion, TermConcept> systemVersionJoin = (Join<TermCodeSystemVersion, TermConcept>) systemVersionFetch; 2608 Fetch<TermCodeSystem, TermCodeSystemVersion> systemFetch = systemVersionFetch.fetch("myCodeSystem", JoinType.INNER); 2609 Join<TermCodeSystem, TermCodeSystemVersion> systemJoin = (Join<TermCodeSystem, TermCodeSystemVersion>) systemFetch; 2610 2611 ArrayList<Predicate> predicates = new ArrayList<>(); 2612 2613 if (isNotBlank(theCode)) { 2614 predicates.add(criteriaBuilder.equal(root.get("myCode"), theCode)); 2615 } 2616 2617 if (isNoneBlank(theCodeSystemUrl)) { 2618 predicates.add(criteriaBuilder.equal(systemJoin.get("myCodeSystemUri"), theCodeSystemUrl)); 2619 } 2620 2621 // for loinc CodeSystem last version is not necessarily the current anymore, so if no version is present 2622 // we need to query for the current, which is that which version is null 2623 if (isNoneBlank(theCodeSystemVersion)) { 2624 predicates.add(criteriaBuilder.equal(systemVersionJoin.get("myCodeSystemVersionId"), theCodeSystemVersion)); 2625 } else { 2626 if (theCodeSystemUrl.toLowerCase(Locale.ROOT).contains(LOINC_LOW)) { 2627 predicates.add(criteriaBuilder.isNull(systemVersionJoin.get("myCodeSystemVersionId"))); 2628 } else { 2629 query.orderBy(criteriaBuilder.desc(root.get("myUpdated"))); 2630 } 2631 } 2632 2633 Predicate outerPredicate = criteriaBuilder.and(predicates.toArray(new Predicate[0])); 2634 query.where(outerPredicate); 2635 2636 final TypedQuery<TermConcept> typedQuery = myEntityManager.createQuery(query.select(root)); 2637 org.hibernate.query.Query<TermConcept> hibernateQuery = (org.hibernate.query.Query<TermConcept>) typedQuery; 2638 hibernateQuery.setFetchSize(SINGLE_FETCH_SIZE); 2639 List<TermConcept> resultsList = hibernateQuery.getResultList(); 2640 2641 if (!resultsList.isEmpty()) { 2642 TermConcept concept = resultsList.get(0); 2643 2644 if (isNotBlank(theDisplay) && !theDisplay.equals(concept.getDisplay())) { 2645 String message = "Concept Display \"" + theDisplay + "\" does not match expected \"" + concept.getDisplay() + "\" for CodeSystem: " + theCodeSystemUrl; 2646 return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, message); 2647 } 2648 2649 return new CodeValidationResult().setCode(concept.getCode()).setDisplay(concept.getDisplay()); 2650 } 2651 2652 return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, " - Code is not found in CodeSystem: " + theCodeSystemUrl); 2653 } 2654 2655 @Override 2656 public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) { 2657 @SuppressWarnings("unchecked") 2658 List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager.createQuery( 2659 "select f.myResource from ForcedId f " + 2660 "where f.myResourceType = 'CodeSystem' and f.myForcedId = '" + theForcedId + "'").getResultList(); 2661 if (resultList.isEmpty()) return Optional.empty(); 2662 2663 if (resultList.size() > 1) 2664 throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " + theForcedId + ". Was constraint " 2665 + ForcedId.IDX_FORCEDID_TYPE_FID + " removed?"); 2666 2667 IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem"); 2668 IBaseResource cs = csDao.toResource(resultList.get(0), false); 2669 return Optional.of(cs); 2670 } 2671 2672 2673 private static final int SECONDS_IN_MINUTE = 60; 2674 private static final int INDEXED_ROOTS_LOGGING_COUNT = 50_000; 2675 2676 2677 @Transactional 2678 @Override 2679 public ReindexTerminologyResult reindexTerminology() throws InterruptedException { 2680 if (myFulltextSearchSvc == null) { 2681 return ReindexTerminologyResult.SEARCH_SVC_DISABLED; 2682 } 2683 2684 if (isBatchTerminologyTasksRunning()) { 2685 return ReindexTerminologyResult.OTHER_BATCH_TERMINOLOGY_TASKS_RUNNING; 2686 } 2687 2688 // disallow pre-expanding ValueSets while reindexing 2689 myDeferredStorageSvc.setProcessDeferred(false); 2690 2691 int objectLoadingThreadNumber = calculateObjectLoadingThreadNumber(); 2692 ourLog.info("Using {} threads to load objects", objectLoadingThreadNumber); 2693 2694 try { 2695 SearchSession searchSession = getSearchSession(); 2696 searchSession 2697 .massIndexer( TermConcept.class ) 2698 .dropAndCreateSchemaOnStart( true ) 2699 .purgeAllOnStart( false ) 2700 .batchSizeToLoadObjects( 100 ) 2701 .cacheMode( CacheMode.IGNORE ) 2702 .threadsToLoadObjects( 6 ) 2703 .transactionTimeout( 60 * SECONDS_IN_MINUTE ) 2704 .monitor( new PojoMassIndexingLoggingMonitor(INDEXED_ROOTS_LOGGING_COUNT) ) 2705 .startAndWait(); 2706 } finally { 2707 myDeferredStorageSvc.setProcessDeferred(true); 2708 } 2709 2710 return ReindexTerminologyResult.SUCCESS; 2711 } 2712 2713 2714 @VisibleForTesting 2715 boolean isBatchTerminologyTasksRunning() { 2716 return isNotSafeToPreExpandValueSets() || isPreExpandingValueSets(); 2717 } 2718 2719 2720 @VisibleForTesting 2721 int calculateObjectLoadingThreadNumber() { 2722 IConnectionPoolInfoProvider connectionPoolInfoProvider = 2723 new ConnectionPoolInfoProvider(myHibernatePropertiesProvider.getDataSource()); 2724 Optional<Integer> maxConnectionsOpt = connectionPoolInfoProvider.getTotalConnectionSize(); 2725 if ( ! maxConnectionsOpt.isPresent() ) { 2726 return DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS; 2727 } 2728 2729 int maxConnections = maxConnectionsOpt.get(); 2730 int usableThreads = maxConnections < 6 ? 1 : maxConnections - 5; 2731 int objectThreads = Math.min(usableThreads, MAX_MASS_INDEXER_OBJECT_LOADING_THREADS); 2732 ourLog.debug("Data source connection pool has {} connections allocated, so reindexing will use {} object " + 2733 "loading threads (each using a connection)", maxConnections, objectThreads); 2734 return objectThreads; 2735 } 2736 2737 2738 @VisibleForTesting 2739 SearchSession getSearchSession() { 2740 return Search.session( myEntityManager ); 2741 } 2742 2743 2744 @VisibleForTesting 2745 public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) { 2746 ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest; 2747 } 2748 2749 static boolean isPlaceholder(DomainResource theResource) { 2750 boolean retVal = false; 2751 Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER); 2752 if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) { 2753 retVal = ((BooleanType) extension.getValue()).booleanValue(); 2754 } 2755 return retVal; 2756 } 2757 2758 /** 2759 * This is only used for unit tests to test failure conditions 2760 */ 2761 static void invokeRunnableForUnitTest() { 2762 if (myInvokeOnNextCallForUnitTest != null) { 2763 Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest; 2764 myInvokeOnNextCallForUnitTest = null; 2765 invokeOnNextCallForUnitTest.run(); 2766 } 2767 } 2768 2769 @VisibleForTesting 2770 public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) { 2771 myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest; 2772 } 2773 2774 static List<TermConcept> toPersistedConcepts(List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) { 2775 ArrayList<TermConcept> retVal = new ArrayList<>(); 2776 2777 for (CodeSystem.ConceptDefinitionComponent next : theConcept) { 2778 if (isNotBlank(next.getCode())) { 2779 TermConcept termConcept = toTermConcept(next, theCodeSystemVersion); 2780 retVal.add(termConcept); 2781 } 2782 } 2783 2784 return retVal; 2785 } 2786 2787 @Nonnull 2788 static TermConcept toTermConcept(CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) { 2789 TermConcept termConcept = new TermConcept(); 2790 termConcept.setCode(theConceptDefinition.getCode()); 2791 termConcept.setCodeSystemVersion(theCodeSystemVersion); 2792 termConcept.setDisplay(theConceptDefinition.getDisplay()); 2793 termConcept.addChildren(toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA); 2794 2795 for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent : theConceptDefinition.getDesignation()) { 2796 if (isNotBlank(designationComponent.getValue())) { 2797 TermConceptDesignation designation = termConcept.addDesignation(); 2798 designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null); 2799 if (designationComponent.hasUse()) { 2800 designation.setUseSystem(designationComponent.getUse().hasSystem() ? designationComponent.getUse().getSystem() : null); 2801 designation.setUseCode(designationComponent.getUse().hasCode() ? designationComponent.getUse().getCode() : null); 2802 designation.setUseDisplay(designationComponent.getUse().hasDisplay() ? designationComponent.getUse().getDisplay() : null); 2803 } 2804 designation.setValue(designationComponent.getValue()); 2805 } 2806 } 2807 2808 for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) { 2809 TermConceptProperty property = new TermConceptProperty(); 2810 2811 property.setKey(next.getCode()); 2812 property.setConcept(termConcept); 2813 property.setCodeSystemVersion(theCodeSystemVersion); 2814 2815 if (next.getValue() instanceof StringType) { 2816 property.setType(TermConceptPropertyTypeEnum.STRING); 2817 property.setValue(next.getValueStringType().getValue()); 2818 } else if (next.getValue() instanceof Coding) { 2819 Coding nextCoding = next.getValueCoding(); 2820 property.setType(TermConceptPropertyTypeEnum.CODING); 2821 property.setCodeSystem(nextCoding.getSystem()); 2822 property.setValue(nextCoding.getCode()); 2823 property.setDisplay(nextCoding.getDisplay()); 2824 } else if (next.getValue() != null) { 2825 // TODO: LOINC has properties of type BOOLEAN that we should handle 2826 ourLog.warn("Don't know how to handle properties of type: " + next.getValue().getClass()); 2827 continue; 2828 } 2829 2830 termConcept.getProperties().add(property); 2831 } 2832 return termConcept; 2833 } 2834 2835 static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) { 2836 // NOTE: return the designation when one of then is not specified. 2837 if (theReqLang == null || theStoredLang == null) 2838 return true; 2839 2840 return theReqLang.equalsIgnoreCase(theStoredLang); 2841 } 2842 2843 public static class Job implements HapiJob { 2844 @Autowired 2845 private ITermReadSvc myTerminologySvc; 2846 2847 @Override 2848 public void execute(JobExecutionContext theContext) { 2849 myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables(); 2850 } 2851 } 2852 2853}