001package ca.uhn.fhir.jpa.term;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.FhirContext;
024import ca.uhn.fhir.context.support.ConceptValidationOptions;
025import ca.uhn.fhir.context.support.IValidationSupport;
026import ca.uhn.fhir.context.support.ValidationSupportContext;
027import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
028import ca.uhn.fhir.i18n.Msg;
029import ca.uhn.fhir.interceptor.model.RequestPartitionId;
030import ca.uhn.fhir.jpa.api.config.DaoConfig;
031import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
032import ca.uhn.fhir.jpa.api.dao.IDao;
033import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
034import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem;
035import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
036import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
037import ca.uhn.fhir.jpa.config.util.ConnectionPoolInfoProvider;
038import ca.uhn.fhir.jpa.config.util.IConnectionPoolInfoProvider;
039import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
040import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
041import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
042import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
043import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
044import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
045import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
046import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao;
047import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao;
048import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao;
049import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
050import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView;
051import ca.uhn.fhir.jpa.entity.TermCodeSystem;
052import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
053import ca.uhn.fhir.jpa.entity.TermConcept;
054import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
055import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
056import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
057import ca.uhn.fhir.jpa.entity.TermConceptProperty;
058import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum;
059import ca.uhn.fhir.jpa.entity.TermValueSet;
060import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
061import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
062import ca.uhn.fhir.jpa.model.entity.ForcedId;
063import ca.uhn.fhir.jpa.model.entity.ResourceTable;
064import ca.uhn.fhir.jpa.model.sched.HapiJob;
065import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
066import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
067import ca.uhn.fhir.jpa.model.util.JpaConstants;
068import ca.uhn.fhir.jpa.search.ElasticsearchNestedQueryBuilderUtil;
069import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
070import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
071import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
072import ca.uhn.fhir.jpa.term.api.ReindexTerminologyResult;
073import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
074import ca.uhn.fhir.jpa.util.LogicUtil;
075import ca.uhn.fhir.rest.api.Constants;
076import ca.uhn.fhir.rest.api.server.RequestDetails;
077import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
078import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
079import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
080import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
081import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
082import ca.uhn.fhir.util.CoverageIgnore;
083import ca.uhn.fhir.util.FhirVersionIndependentConcept;
084import ca.uhn.fhir.util.HapiExtensions;
085import ca.uhn.fhir.util.StopWatch;
086import ca.uhn.fhir.util.UrlUtil;
087import ca.uhn.fhir.util.ValidateUtil;
088import com.github.benmanes.caffeine.cache.Cache;
089import com.github.benmanes.caffeine.cache.Caffeine;
090import com.google.common.annotations.VisibleForTesting;
091import com.google.common.base.Stopwatch;
092import com.google.common.collect.ArrayListMultimap;
093import com.google.gson.JsonObject;
094import org.apache.commons.collections4.ListUtils;
095import org.apache.commons.lang3.ObjectUtils;
096import org.apache.commons.lang3.StringUtils;
097import org.apache.commons.lang3.Validate;
098import org.apache.commons.lang3.time.DateUtils;
099import org.apache.lucene.index.Term;
100import org.apache.lucene.search.BooleanQuery;
101import org.apache.lucene.search.Query;
102import org.apache.lucene.search.RegexpQuery;
103import org.apache.lucene.search.TermQuery;
104import org.hibernate.CacheMode;
105import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
106import org.hibernate.search.backend.lucene.LuceneExtension;
107import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep;
108import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
109import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
110import org.hibernate.search.engine.search.query.SearchQuery;
111import org.hibernate.search.mapper.orm.Search;
112import org.hibernate.search.mapper.orm.common.EntityReference;
113import org.hibernate.search.mapper.orm.session.SearchSession;
114import org.hibernate.search.mapper.pojo.massindexing.impl.PojoMassIndexingLoggingMonitor;
115import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService;
116import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
117import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
118import org.hl7.fhir.convertors.context.ConversionContext40_50;
119import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50;
120import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50;
121import org.hl7.fhir.instance.model.api.IAnyResource;
122import org.hl7.fhir.instance.model.api.IBaseCoding;
123import org.hl7.fhir.instance.model.api.IBaseDatatype;
124import org.hl7.fhir.instance.model.api.IBaseResource;
125import org.hl7.fhir.instance.model.api.IIdType;
126import org.hl7.fhir.instance.model.api.IPrimitiveType;
127import org.hl7.fhir.r4.model.BooleanType;
128import org.hl7.fhir.r4.model.CanonicalType;
129import org.hl7.fhir.r4.model.CodeSystem;
130import org.hl7.fhir.r4.model.CodeableConcept;
131import org.hl7.fhir.r4.model.Coding;
132import org.hl7.fhir.r4.model.DomainResource;
133import org.hl7.fhir.r4.model.Enumerations;
134import org.hl7.fhir.r4.model.Extension;
135import org.hl7.fhir.r4.model.InstantType;
136import org.hl7.fhir.r4.model.IntegerType;
137import org.hl7.fhir.r4.model.StringType;
138import org.hl7.fhir.r4.model.ValueSet;
139import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome;
140import org.quartz.JobExecutionContext;
141import org.springframework.beans.factory.annotation.Autowired;
142import org.springframework.context.ApplicationContext;
143import org.springframework.context.annotation.Lazy;
144import org.springframework.data.domain.PageRequest;
145import org.springframework.data.domain.Pageable;
146import org.springframework.data.domain.Slice;
147import org.springframework.transaction.PlatformTransactionManager;
148import org.springframework.transaction.TransactionDefinition;
149import org.springframework.transaction.annotation.Propagation;
150import org.springframework.transaction.annotation.Transactional;
151import org.springframework.transaction.interceptor.NoRollbackRuleAttribute;
152import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute;
153import org.springframework.transaction.support.TransactionSynchronizationManager;
154import org.springframework.transaction.support.TransactionTemplate;
155import org.springframework.util.CollectionUtils;
156import org.springframework.util.comparator.Comparators;
157
158import javax.annotation.Nonnull;
159import javax.annotation.Nullable;
160import javax.annotation.PostConstruct;
161import javax.persistence.EntityManager;
162import javax.persistence.NonUniqueResultException;
163import javax.persistence.PersistenceContext;
164import javax.persistence.PersistenceContextType;
165import javax.persistence.TypedQuery;
166import javax.persistence.criteria.CriteriaBuilder;
167import javax.persistence.criteria.CriteriaQuery;
168import javax.persistence.criteria.Fetch;
169import javax.persistence.criteria.Join;
170import javax.persistence.criteria.JoinType;
171import javax.persistence.criteria.Predicate;
172import javax.persistence.criteria.Root;
173import java.util.ArrayList;
174import java.util.Arrays;
175import java.util.Collection;
176import java.util.Collections;
177import java.util.Date;
178import java.util.HashMap;
179import java.util.HashSet;
180import java.util.LinkedHashMap;
181import java.util.List;
182import java.util.Locale;
183import java.util.Map;
184import java.util.Objects;
185import java.util.Optional;
186import java.util.Set;
187import java.util.StringTokenizer;
188import java.util.UUID;
189import java.util.concurrent.TimeUnit;
190import java.util.concurrent.atomic.AtomicInteger;
191import java.util.function.Consumer;
192import java.util.function.Supplier;
193import java.util.stream.Collectors;
194
195import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI;
196import static java.lang.String.join;
197import static java.util.stream.Collectors.joining;
198import static java.util.stream.Collectors.toList;
199import static java.util.stream.Collectors.toSet;
200import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
201import static org.apache.commons.lang3.StringUtils.defaultString;
202import static org.apache.commons.lang3.StringUtils.isBlank;
203import static org.apache.commons.lang3.StringUtils.isEmpty;
204import static org.apache.commons.lang3.StringUtils.isNoneBlank;
205import static org.apache.commons.lang3.StringUtils.isNotBlank;
206import static org.apache.commons.lang3.StringUtils.lowerCase;
207import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase;
208import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
209
210public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
211        public static final int DEFAULT_FETCH_SIZE = 250;
212        private static final int SINGLE_FETCH_SIZE = 1;
213        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseTermReadSvcImpl.class);
214        private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions();
215        private static final TermCodeSystemVersion NO_CURRENT_VERSION = new TermCodeSystemVersion().setId(-1L);
216        private static Runnable myInvokeOnNextCallForUnitTest;
217        private static boolean ourForceDisableHibernateSearchForUnitTest;
218
219        private static final String IDX_PROPERTIES = "myProperties";
220        private static final String IDX_PROP_KEY = IDX_PROPERTIES + ".myKey";
221        private static final String IDX_PROP_VALUE_STRING = IDX_PROPERTIES + ".myValueString";
222        private static final String IDX_PROP_DISPLAY_STRING = IDX_PROPERTIES + ".myDisplayString";
223
224        public static final int DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS = 2;
225        // doesn't seem to be much gain by using more threads than this value
226        public static final int MAX_MASS_INDEXER_OBJECT_LOADING_THREADS = 6;
227
228        private boolean myPreExpandingValueSets = false;
229
230        private final Cache<String, TermCodeSystemVersion> myCodeSystemCurrentVersionCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build();
231        @Autowired
232        protected DaoRegistry myDaoRegistry;
233        @Autowired
234        protected ITermCodeSystemDao myCodeSystemDao;
235        @Autowired
236        protected ITermConceptDao myConceptDao;
237        @Autowired
238        protected ITermConceptPropertyDao myConceptPropertyDao;
239        @Autowired
240        protected ITermConceptDesignationDao myConceptDesignationDao;
241        @Autowired
242        protected ITermValueSetDao myTermValueSetDao;
243        @Autowired
244        protected ITermValueSetConceptDao myValueSetConceptDao;
245        @Autowired
246        protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao;
247        @Autowired
248        protected FhirContext myContext;
249        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
250        protected EntityManager myEntityManager;
251        @Autowired
252        private ITermCodeSystemVersionDao myCodeSystemVersionDao;
253        @Autowired
254        private DaoConfig myDaoConfig;
255        private TransactionTemplate myTxTemplate;
256        @Autowired
257        private PlatformTransactionManager myTransactionManager;
258        @Autowired(required = false)
259        private IFulltextSearchSvc myFulltextSearchSvc;
260        @Autowired
261        private PlatformTransactionManager myTxManager;
262        @Autowired
263        private ITermConceptDao myTermConceptDao;
264        @Autowired
265        private ITermValueSetConceptViewDao myTermValueSetConceptViewDao;
266        @Autowired
267        private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao;
268        @Autowired
269        private ISchedulerService mySchedulerService;
270        @Autowired(required = false)
271        private ITermDeferredStorageSvc myDeferredStorageSvc;
272        @Autowired
273        private IIdHelperService myIdHelperService;
274
275        @Autowired
276        private ApplicationContext myApplicationContext;
277
278        private volatile IValidationSupport myJpaValidationSupport;
279        private volatile IValidationSupport myValidationSupport;
280
281        //We need this bean so we can tell which mode hibernate search is running in.
282        @Autowired
283        private HibernatePropertiesProvider myHibernatePropertiesProvider;
284
285
286        private boolean isFullTextSetToUseElastic() {
287                return "elasticsearch".equalsIgnoreCase(myHibernatePropertiesProvider.getHibernateSearchBackend());
288        }
289
290        @Override
291        public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) {
292                TermCodeSystemVersion cs = getCurrentCodeSystemVersion(theSystem);
293                return cs != null;
294        }
295
296        private boolean addCodeIfNotAlreadyAdded(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
297                String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
298                String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId();
299                String code = theConcept.getCode();
300                String display = theConcept.getDisplay();
301                Long sourceConceptPid = theConcept.getId();
302                String directParentPids = "";
303
304                if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) {
305                        directParentPids = theConcept
306                                .getParents()
307                                .stream()
308                                .map(t -> t.getParent().getId().toString())
309                                .collect(joining(" "));
310                }
311
312                Collection<TermConceptDesignation> designations = theConcept.getDesignations();
313                if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
314                        return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem + "|" + theValueSetIncludeVersion, code, display, sourceConceptPid, directParentPids, codeSystemVersion);
315                } else {
316                        return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem, code, display, sourceConceptPid, directParentPids, codeSystemVersion);
317                }
318        }
319
320        private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, Collection<TermConceptDesignation> theDesignations) {
321                if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
322                        if (isNoneBlank(theCodeSystem, theCode)) {
323                                if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
324                                        theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
325                                        return true;
326                                }
327
328                                if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
329                                        theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
330                                        return true;
331                                }
332                        }
333                } else {
334                        if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
335                                theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
336                                return true;
337                        }
338
339                        if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
340                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
341                                return true;
342                        }
343                }
344
345                return false;
346        }
347
348        private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, String theSystemVersion) {
349                if (isNoneBlank(theCodeSystem, theCode)) {
350                        if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
351                                theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theSystemVersion);
352                                return true;
353                        }
354
355                        if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
356                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
357                                return true;
358                        }
359                }
360
361                return false;
362        }
363
364        private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) {
365                boolean retVal = theSetToPopulate.add(theConcept);
366                if (retVal) {
367                        if (theSetToPopulate.size() >= myDaoConfig.getMaximumExpansionSize()) {
368                                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize());
369                                throw new ExpansionTooCostlyException(Msg.code(885) + msg);
370                        }
371                }
372                return retVal;
373        }
374
375        /**
376         * This method is present only for unit tests, do not call from client code
377         */
378        @VisibleForTesting
379        public void clearCaches() {
380                myCodeSystemCurrentVersionCache.invalidateAll();
381        }
382
383
384        public void deleteValueSetForResource(ResourceTable theResourceTable) {
385                // Get existing entity so it can be deleted.
386                Optional<TermValueSet> optionalExistingTermValueSetById = myTermValueSetDao.findByResourcePid(theResourceTable.getId());
387
388                if (optionalExistingTermValueSetById.isPresent()) {
389                        TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get();
390
391                        ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId());
392                        deletePreCalculatedValueSetContents(existingTermValueSet);
393                        myTermValueSetDao.deleteById(existingTermValueSet.getId());
394                        ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId());
395                }
396        }
397
398        private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) {
399                myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId());
400                myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId());
401        }
402
403        @Override
404        @Transactional
405        public void deleteValueSetAndChildren(ResourceTable theResourceTable) {
406                deleteValueSetForResource(theResourceTable);
407        }
408
409
410        @Override
411        @Transactional
412        public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
413                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
414
415                ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl);
416
417                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
418                for (ValueSet.ValueSetExpansionContainsComponent nextContains : expanded.getExpansion().getContains()) {
419                        retVal.add(new FhirVersionIndependentConcept(nextContains.getSystem(), nextContains.getCode(), nextContains.getDisplay(), nextContains.getVersion()));
420                }
421                return retVal;
422        }
423
424        @Override
425        @Transactional
426        public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
427                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl);
428                if (valueSet == null) {
429                        throw new ResourceNotFoundException(Msg.code(886) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl));
430                }
431
432                return expandValueSet(theExpansionOptions, valueSet);
433        }
434
435        @Override
436        @Transactional(propagation = Propagation.REQUIRED)
437        public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) {
438                String filter = null;
439                if (theExpansionOptions != null) {
440                        filter = theExpansionOptions.getFilter();
441                }
442                return expandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter));
443        }
444
445        private ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, ExpansionFilter theFilter) {
446                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
447
448                ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
449                int offset = expansionOptions.getOffset();
450                int count = expansionOptions.getCount();
451
452                ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, count, expansionOptions.isIncludeHierarchy());
453                accumulator.setHardExpansionMaximumSize(myDaoConfig.getMaximumExpansionSize());
454                accumulator.setSkipCountRemaining(offset);
455                accumulator.setIdentifier(UUID.randomUUID().toString());
456                accumulator.setTimestamp(new Date());
457                accumulator.setOffset(offset);
458
459                if (theExpansionOptions != null && isHibernateSearchEnabled()) {
460                        accumulator.addParameter().setName("offset").setValue(new IntegerType(offset));
461                        accumulator.addParameter().setName("count").setValue(new IntegerType(count));
462                }
463
464                expandValueSetIntoAccumulator(theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true);
465
466                if (accumulator.getTotalConcepts() != null) {
467                        accumulator.setTotal(accumulator.getTotalConcepts());
468                }
469
470                ValueSet valueSet = new ValueSet();
471                valueSet.setUrl(theValueSetToExpand.getUrl());
472                valueSet.setId(theValueSetToExpand.getId());
473                valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
474                valueSet.setCompose(theValueSetToExpand.getCompose());
475                valueSet.setExpansion(accumulator);
476
477                for (String next : accumulator.getMessages()) {
478                        valueSet.getMeta().addExtension()
479                                .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
480                                .setValue(new StringType(next));
481                }
482
483                if (expansionOptions.isIncludeHierarchy()) {
484                        accumulator.applyHierarchy();
485                }
486
487                return valueSet;
488        }
489
490        private void expandValueSetIntoAccumulator(ValueSet theValueSetToExpand, ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theAccumulator, ExpansionFilter theFilter, boolean theAdd) {
491                Optional<TermValueSet> optionalTermValueSet;
492                if (theValueSetToExpand.hasUrl()) {
493                        if (theValueSetToExpand.hasVersion()) {
494                                optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion(theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion());
495                        } else {
496                                optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl());
497                        }
498                } else {
499                        optionalTermValueSet = Optional.empty();
500                }
501
502                /*
503                 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion
504                 */
505                if (!optionalTermValueSet.isPresent()) {
506                        ourLog.debug("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", getValueSetInfo(theValueSetToExpand));
507                        String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingInMemoryExpansion", getValueSetInfo(theValueSetToExpand));
508                        theAccumulator.addMessage(msg);
509                        expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
510                        return;
511                }
512
513                /*
514                 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion
515                 */
516                TermValueSet termValueSet = optionalTermValueSet.get();
517                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
518                        String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded", getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
519                        theAccumulator.addMessage(msg);
520                        expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
521                        return;
522                }
523
524                /*
525                 * ValueSet is pre-expanded in database so let's use that
526                 */
527                String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet);
528                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp);
529                theAccumulator.addMessage(msg);
530                expandConcepts(theAccumulator, termValueSet, theFilter, theAdd, isOracleDialect());
531        }
532
533        @Nonnull
534        private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) {
535                String expansionTimestamp = "(unknown)";
536                if (termValueSet.getExpansionTimestamp() != null) {
537                        String timeElapsed = StopWatch.formatMillis(System.currentTimeMillis() - termValueSet.getExpansionTimestamp().getTime());
538                        expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString() + " (" + timeElapsed + " ago)";
539                }
540                return expansionTimestamp;
541        }
542
543        private boolean isOracleDialect() {
544                return myHibernatePropertiesProvider.getDialect() instanceof org.hibernate.dialect.Oracle12cDialect;
545        }
546
547        private void expandConcepts(IValueSetConceptAccumulator theAccumulator, TermValueSet theTermValueSet, ExpansionFilter theFilter, boolean theAdd, boolean theOracle) {
548                // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its copy pasted sibling
549                Integer offset = theAccumulator.getSkipCountRemaining();
550                offset = ObjectUtils.defaultIfNull(offset, 0);
551                offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue());
552
553                Integer count = theAccumulator.getCapacityRemaining();
554                count = defaultIfNull(count, myDaoConfig.getMaximumExpansionSize());
555
556                int conceptsExpanded = 0;
557                int designationsExpanded = 0;
558                int toIndex = offset + count;
559
560                Collection<? extends ITermValueSetConceptView> conceptViews;
561                boolean wasFilteredResult = false;
562                String filterDisplayValue = null;
563                if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
564                        filterDisplayValue = lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]"));
565                        String displayValue = "%" + lowerCase(filterDisplayValue) + "%";
566                        if (theOracle) {
567                                conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
568                        } else {
569                                conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
570                        }
571                        wasFilteredResult = true;
572                } else {
573                        // TODO JA HS: I'm pretty sure we are overfetching here.  test says offset 3, count 4, but we are fetching index 3 -> 10 here, grabbing 7 concepts.
574                        //Specifically this test testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectRange
575                        if (theOracle) {
576                                conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
577                        } else {
578                                conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
579                        }
580                        theAccumulator.consumeSkipCount(offset);
581                        if (theAdd) {
582                                theAccumulator.incrementOrDecrementTotalConcepts(true, theTermValueSet.getTotalConcepts().intValue());
583                        }
584                }
585
586                if (conceptViews.isEmpty()) {
587                        logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded);
588                        return;
589                }
590
591                Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>();
592                ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create();
593                Map<Long, Long> pidToSourcePid = new HashMap<>();
594                Map<Long, String> pidToSourceDirectParentPids = new HashMap<>();
595
596                for (ITermValueSetConceptView conceptView : conceptViews) {
597
598                        String system = conceptView.getConceptSystemUrl();
599                        String code = conceptView.getConceptCode();
600                        String display = conceptView.getConceptDisplay();
601                        String systemVersion = conceptView.getConceptSystemVersion();
602
603                        //-- this is quick solution, may need to revisit
604                        if (!applyFilter(display, filterDisplayValue)) {
605                                continue;}
606
607                        Long conceptPid = conceptView.getConceptPid();
608                        if (!pidToConcept.containsKey(conceptPid)) {
609                                FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display, systemVersion);
610                                pidToConcept.put(conceptPid, concept);
611                        }
612
613                        // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations optional.
614                        if (conceptView.getDesignationPid() != null) {
615                                TermConceptDesignation designation = new TermConceptDesignation();
616                                designation.setUseSystem(conceptView.getDesignationUseSystem());
617                                designation.setUseCode(conceptView.getDesignationUseCode());
618                                designation.setUseDisplay(conceptView.getDesignationUseDisplay());
619                                designation.setValue(conceptView.getDesignationVal());
620                                designation.setLanguage(conceptView.getDesignationLang());
621                                pidToDesignations.put(conceptPid, designation);
622
623                                if (++designationsExpanded % 250 == 0) {
624                                        logDesignationsExpanded("Expansion of designations in progress. ", theTermValueSet, designationsExpanded);
625                                }
626                        }
627
628                        if (theAccumulator.isTrackingHierarchy()) {
629                                pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid());
630                                pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids());
631                        }
632
633                        if (++conceptsExpanded % 250 == 0) {
634                                logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded);
635                        }
636                }
637
638                for (Long nextPid : pidToConcept.keySet()) {
639                        FhirVersionIndependentConcept concept = pidToConcept.get(nextPid);
640                        List<TermConceptDesignation> designations = pidToDesignations.get(nextPid);
641                        String system = concept.getSystem();
642                        String code = concept.getCode();
643                        String display = concept.getDisplay();
644                        String systemVersion = concept.getSystemVersion();
645
646                        if (theAdd) {
647                                if (theAccumulator.getCapacityRemaining() != null) {
648                                        if (theAccumulator.getCapacityRemaining() == 0) {
649                                                break;
650                                        }
651                                }
652
653                                Long sourceConceptPid = pidToSourcePid.get(nextPid);
654                                String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid);
655                                theAccumulator.includeConceptWithDesignations(system, code, display, designations, sourceConceptPid, sourceConceptDirectParentPids, systemVersion);
656                        } else {
657                                boolean removed = theAccumulator.excludeConcept(system, code);
658                                if (removed) {
659                                        theAccumulator.incrementOrDecrementTotalConcepts(false, 1);
660                                }
661                        }
662                }
663
664                if (wasFilteredResult && theAdd) {
665                        theAccumulator.incrementOrDecrementTotalConcepts(true, pidToConcept.size());
666                }
667
668                logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded);
669                logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded);
670        }
671
672        private void logConceptsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) {
673                if (theConceptsExpanded > 0) {
674                        ourLog.debug("{}Have expanded {} concepts in ValueSet[{}]", theLogDescriptionPrefix, theConceptsExpanded, theTermValueSet.getUrl());
675                }
676        }
677
678        private void logDesignationsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) {
679                if (theDesignationsExpanded > 0) {
680                        ourLog.debug("{}Have expanded {} designations in ValueSet[{}]", theLogDescriptionPrefix, theDesignationsExpanded, theTermValueSet.getUrl());
681                }
682        }
683
684        public boolean applyFilter(final String theDisplay, final String theFilterDisplay) {
685
686                //-- safety check only, no need to apply filter
687                if (theDisplay == null || theFilterDisplay == null)
688                        return true;
689
690                // -- sentence case
691                if (startsWithIgnoreCase(theDisplay, theFilterDisplay))
692                        return true;
693
694                //-- token case
695                return startsWithByWordBoundaries(theDisplay, theFilterDisplay);
696        }
697
698        private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) {
699                // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo'
700                StringTokenizer tok = new StringTokenizer(theDisplay);
701                List<String> tokens = new ArrayList<>();
702                while (tok.hasMoreTokens()) {
703                        String token = tok.nextToken();
704                        if (startsWithIgnoreCase(token, theFilterDisplay))
705                                return true;
706                        tokens.add(token);
707                }
708
709                // Allow to search by the end of the phrase.  E.g.  "working proficiency" will match "Limited working proficiency"
710                for (int start = 0; start <= tokens.size() - 1; ++start) {
711                        for (int end = start + 1; end <= tokens.size(); ++end) {
712                                String sublist = String.join(" ", tokens.subList(start, end));
713                                if (startsWithIgnoreCase(sublist, theFilterDisplay))
714                                        return true;
715                        }
716                }
717                return false;
718        }
719
720        @Override
721        @Transactional(propagation = Propagation.REQUIRED)
722        public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
723                expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, ExpansionFilter.NO_FILTER);
724        }
725
726        @SuppressWarnings("ConstantConditions")
727        private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, @Nonnull ExpansionFilter theExpansionFilter) {
728                Set<String> addedCodes = new HashSet<>();
729
730                StopWatch sw = new StopWatch();
731                String valueSetInfo = getValueSetInfo(theValueSetToExpand);
732                ourLog.debug("Working with {}", valueSetInfo);
733
734                // Offset can't be combined with excludes
735                Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining();
736                if (skipCountRemaining != null && skipCountRemaining > 0) {
737                        if (theValueSetToExpand.getCompose().getExclude().size() > 0) {
738                                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo);
739                                throw new InvalidRequestException(Msg.code(887) + msg);
740                        }
741                }
742
743                // Handle includes
744                ourLog.debug("Handling includes");
745                for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
746                        for (int i = 0; ; i++) {
747                                int queryIndex = i;
748                                Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
749                                        boolean add = true;
750                                        return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, queryIndex, theExpansionFilter);
751                                });
752                                if (!shouldContinue) {
753                                        break;
754                                }
755                        }
756                }
757
758                // Handle excludes
759                ourLog.debug("Handling excludes");
760                for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) {
761                        for (int i = 0; ; i++) {
762                                int queryIndex = i;
763                                Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
764                                        boolean add = false;
765                                        ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
766                                        return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, queryIndex, expansionFilter);
767                                });
768                                if (!shouldContinue) {
769                                        break;
770                                }
771                        }
772                }
773
774                if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) {
775                        myTxTemplate.execute(t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder());
776                }
777
778                ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
779        }
780
781        /**
782         * Execute in a new transaction only if we aren't already in one. We do this because in some cases
783         * when performing a VS expansion we throw an {@link ExpansionTooCostlyException} and we don't want
784         * this to cause the TX to be marked a rollback prematurely.
785         */
786        private <T> T executeInNewTransactionIfNeeded(Supplier<T> theAction) {
787                if (TransactionSynchronizationManager.isSynchronizationActive()) {
788                        return theAction.get();
789                }
790                return myTxTemplate.execute(t -> theAction.get());
791        }
792
793        private String getValueSetInfo(ValueSet theValueSet) {
794                StringBuilder sb = new StringBuilder();
795                boolean isIdentified = false;
796                if (theValueSet.hasUrl()) {
797                        isIdentified = true;
798                        sb
799                                .append("ValueSet.url[")
800                                .append(theValueSet.getUrl())
801                                .append("]");
802                } else if (theValueSet.hasId()) {
803                        isIdentified = true;
804                        sb
805                                .append("ValueSet.id[")
806                                .append(theValueSet.getId())
807                                .append("]");
808                }
809
810                if (!isIdentified) {
811                        sb.append("Unidentified ValueSet");
812                }
813
814                return sb.toString();
815        }
816
817        /**
818         * @return Returns true if there are potentially more results to process.
819         */
820        private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter) {
821
822                String system = theIncludeOrExclude.getSystem();
823                boolean hasSystem = isNotBlank(system);
824                boolean hasValueSet = theIncludeOrExclude.getValueSet().size() > 0;
825
826                if (hasSystem) {
827
828                        if (theExpansionFilter.hasCode() && theExpansionFilter.getSystem() != null && !system.equals(theExpansionFilter.getSystem())) {
829                                return false;
830                        }
831
832                        ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
833
834                        TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
835                        if (cs != null) {
836
837                                return expandValueSetHandleIncludeOrExcludeUsingDatabase(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
838
839                        } else {
840
841                                if (theIncludeOrExclude.getConcept().size() > 0 && theExpansionFilter.hasCode()) {
842                                        if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) {
843                                                if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) {
844                                                        return false;
845                                                }
846                                        }
847                                }
848
849                                Consumer<FhirVersionIndependentConcept> consumer = c -> {
850                                        addOrRemoveCode(theValueSetCodeAccumulator, theAddedCodes, theAdd, system, c.getCode(), c.getDisplay(), c.getSystemVersion());
851                                };
852
853                                try {
854                                        ConversionContext40_50.INSTANCE.init(new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet");
855                                        org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude = ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude);
856                                        new InMemoryTerminologyServerValidationSupport(myContext).expandValueSetIncludeOrExclude(new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude);
857                                } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) {
858                                        if (!theExpansionOptions.isFailOnMissingCodeSystem() && e.getFailureType() == InMemoryTerminologyServerValidationSupport.FailureType.UNKNOWN_CODE_SYSTEM) {
859                                                return false;
860                                        }
861                                        throw new InternalErrorException(Msg.code(888) + e);
862                                } finally {
863                                        ConversionContext40_50.INSTANCE.close("ValueSet");
864                                }
865
866                                return false;
867                        }
868
869                } else if (hasValueSet) {
870
871                        for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
872                                String valueSetUrl = nextValueSet.getValueAsString();
873                                ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl);
874
875                                ExpansionFilter subExpansionFilter = new ExpansionFilter(theExpansionFilter, theIncludeOrExclude.getFilter(), theValueSetCodeAccumulator.getCapacityRemaining());
876
877                                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
878
879                                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl);
880                                if (valueSet == null) {
881                                        throw new ResourceNotFoundException(Msg.code(889) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl));
882                                }
883
884                                expandValueSetIntoAccumulator(valueSet, theExpansionOptions, theValueSetCodeAccumulator, subExpansionFilter, theAdd);
885
886                        }
887
888                        return false;
889
890                } else {
891                        throw new InvalidRequestException(Msg.code(890) + "ValueSet contains " + (theAdd ? "include" : "exclude") + " criteria with no system defined");
892                }
893
894
895        }
896
897        private boolean isHibernateSearchEnabled() {
898                return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest;
899        }
900
901        @Nonnull
902        private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
903                String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
904                TermCodeSystemVersion csv;
905                if (isEmpty(includeOrExcludeVersion)) {
906                        csv = theCs.getCurrentVersion();
907                } else {
908                        csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCs.getPid(), includeOrExcludeVersion);
909                }
910
911                SearchSession searchSession = Search.session(myEntityManager);
912                /*
913                 * If FullText searching is not enabled, we can handle only basic expansions
914                 * since we're going to do it without the database.
915                 */
916                if (!isHibernateSearchEnabled()) {
917                        expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd);
918                        return false;
919                }
920
921                /*
922                 * Ok, let's use hibernate search to build the expansion
923                 */
924                //Manually building a predicate since we need to throw it around.
925                SearchPredicateFactory predicate = searchSession.scope(TermConcept.class).predicate();
926
927                //Build the top-level expansion on filters.
928                PredicateFinalStep step = predicate.bool(b -> {
929                        b.must(predicate.match().field("myCodeSystemVersionPid").matching(csv.getPid()));
930
931                        if (theExpansionFilter.hasCode()) {
932                                b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode()));
933                        }
934
935                        String codeSystemUrlAndVersion = buildCodeSystemUrlAndVersion(theSystem, includeOrExcludeVersion);
936                        for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
937                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
938                        }
939                        for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) {
940                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
941                        }
942                });
943
944                List<String> codes = theIncludeOrExclude
945                        .getConcept()
946                        .stream()
947                        .filter(Objects::nonNull)
948                        .map(ValueSet.ConceptReferenceComponent::getCode)
949                        .filter(StringUtils::isNotBlank)
950                        .collect(Collectors.toList());
951
952                Optional<PredicateFinalStep> expansionStepOpt = buildExpansionPredicate(codes, predicate);
953                final PredicateFinalStep finishedQuery = expansionStepOpt.isPresent()
954                        ? predicate.bool().must(step).must(expansionStepOpt.get()) : step;
955
956                /*
957                 * DM 2019-08-21 - Processing slows after any ValueSets with many codes explicitly identified. This might
958                 * be due to the dark arts that is memory management. Will monitor but not do anything about this right now.
959                 */
960
961                //BooleanQuery.setMaxClauseCount(SearchBuilder.getMaximumPageSize());
962                //TODO GGG HS looks like we can't set max clause count, but it can be set server side.
963                //BooleanQuery.setMaxClauseCount(10000);
964                // JM 2-22-02-15 - Hopefully increasing maxClauseCount should be not needed anymore
965
966                StopWatch sw = new StopWatch();
967                AtomicInteger count = new AtomicInteger(0);
968
969                int maxResultsPerBatch = SearchBuilder.getMaximumPageSize();
970
971                /*
972                 * If the accumulator is bounded, we may reduce the size of the query to
973                 * Lucene in order to be more efficient.
974                 */
975                if (theAdd) {
976                        Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining();
977                        if (accumulatorCapacityRemaining != null) {
978                                maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1);
979                        }
980                        if (maxResultsPerBatch <= 0) {
981                                return false;
982                        }
983                }
984
985                ourLog.debug("Beginning batch expansion for {} with max results per batch: {}", (theAdd ? "inclusion" : "exclusion"), maxResultsPerBatch);
986
987                StopWatch swForBatch = new StopWatch();
988                AtomicInteger countForBatch = new AtomicInteger(0);
989
990                SearchQuery<EntityReference> termConceptsQuery = searchSession
991                        .search(TermConcept.class)
992                        .selectEntityReference()
993                        .where(f -> finishedQuery)
994                        .toQuery();
995
996                ourLog.trace("About to query: {}", termConceptsQuery.queryString());
997                List<EntityReference> termConceptRefs = termConceptsQuery.fetchHits(theQueryIndex * maxResultsPerBatch, maxResultsPerBatch);
998                List<Long> pids = termConceptRefs
999                        .stream()
1000                        .map(t -> (Long) t.id())
1001                        .collect(Collectors.toList());
1002
1003                List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids);
1004
1005                // If the include section had multiple codes, return the codes in the same order
1006                if (codes.size() > 1) {
1007                        termConcepts = new ArrayList<>(termConcepts);
1008                        Map<String, Integer> codeToIndex = new HashMap<>(codes.size());
1009                        for (int i = 0; i < codes.size(); i++) {
1010                                codeToIndex.put(codes.get(i), i);
1011                        }
1012                        termConcepts.sort(((o1, o2) -> {
1013                                Integer idx1 = codeToIndex.get(o1.getCode());
1014                                Integer idx2 = codeToIndex.get(o2.getCode());
1015                                return Comparators.nullsHigh().compare(idx1, idx2);
1016                        }));
1017                }
1018
1019                int resultsInBatch = termConcepts.size();
1020                int firstResult = theQueryIndex * maxResultsPerBatch;// TODO GGG HS we lose the ability to check the index of the first result, so just best-guessing it here.
1021                int delta = 0;
1022                for (TermConcept concept : termConcepts) {
1023                        count.incrementAndGet();
1024                        countForBatch.incrementAndGet();
1025                        if (theAdd && expansionStepOpt.isPresent()) {
1026                                ValueSet.ConceptReferenceComponent theIncludeConcept = getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept);
1027                                if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) {
1028                                        concept.setDisplay(theIncludeConcept.getDisplay());
1029                                }
1030                        }
1031                        boolean added = addCodeIfNotAlreadyAdded(theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
1032                        if (added) {
1033                                delta++;
1034                        }
1035                }
1036
1037                ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
1038                theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta);
1039
1040                if (resultsInBatch < maxResultsPerBatch) {
1041                        ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
1042                        return false;
1043                } else {
1044                        return true;
1045                }
1046        }
1047
1048        private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet(ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) {
1049                return theIncludeOrExclude
1050                        .getConcept()
1051                        .stream().filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode()))
1052                        .findFirst()
1053                        .orElse(null);
1054        }
1055
1056        /**
1057         * Helper method which builds a predicate for the expansion
1058         */
1059        private Optional<PredicateFinalStep> buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) {
1060                if (CollectionUtils.isEmpty(theCodes)) { return  Optional.empty(); }
1061
1062                if (theCodes.size() < BooleanQuery.getMaxClauseCount()) {
1063                        return Optional.of(thePredicate.simpleQueryString()
1064                                .field( "myCode" ).matching( String.join(" | ", theCodes)) );
1065                }
1066                
1067                // Number of codes is larger than maxClauseCount, so we split the query in several clauses
1068
1069                // partition codes in lists of BooleanQuery.getMaxClauseCount() size
1070                List<List<String>> listOfLists = ListUtils.partition(theCodes, BooleanQuery.getMaxClauseCount());
1071
1072                PredicateFinalStep step = thePredicate.bool(b -> {
1073                        b.minimumShouldMatchNumber(1);
1074                        for (List<String> codeList : listOfLists) {
1075                                b.should(p -> p.simpleQueryString().field("myCode").matching(String.join(" | ", codeList)));
1076                        }
1077                });
1078
1079                return Optional.of(step);
1080        }
1081
1082
1083        private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) {
1084                String codeSystemUrlAndVersion;
1085                if (theIncludeOrExcludeVersion != null) {
1086                        codeSystemUrlAndVersion = theSystem + "|" + theIncludeOrExcludeVersion;
1087                } else {
1088                        codeSystemUrlAndVersion = theSystem;
1089                }
1090                return codeSystemUrlAndVersion;
1091        }
1092
1093        private @Nonnull
1094        ValueSetExpansionOptions provideExpansionOptions(@Nullable ValueSetExpansionOptions theExpansionOptions) {
1095                if (theExpansionOptions != null) {
1096                        return theExpansionOptions;
1097                } else {
1098                        return DEFAULT_EXPANSION_OPTIONS;
1099                }
1100        }
1101
1102        private void addOrRemoveCode(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theSystem, String theCode, String theDisplay, String theSystemVersion) {
1103                if (theAdd && theAddedCodes.add(theSystem + "|" + theCode)) {
1104                        theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion);
1105                }
1106                if (!theAdd && theAddedCodes.remove(theSystem + "|" + theCode)) {
1107                        theValueSetCodeAccumulator.excludeConcept(theSystem, theCode);
1108                }
1109        }
1110
1111        private void handleFilter(String theCodeSystemIdentifier, SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1112                if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) {
1113                        return;
1114                }
1115
1116                if (isBlank(theFilter.getValue()) || theFilter.getOp() == null || isBlank(theFilter.getProperty())) {
1117                        throw new InvalidRequestException(Msg.code(891) + "Invalid filter, must have fields populated: property op value");
1118                }
1119
1120                switch (theFilter.getProperty()) {
1121                        case "display:exact":
1122                        case "display":
1123                                handleFilterDisplay(theF, theB, theFilter);
1124                                break;
1125                        case "concept":
1126                        case "code":
1127                                handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter);
1128                                break;
1129                        case "parent":
1130                        case "child":
1131                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1132                                handleFilterLoincParentChild(theF, theB, theFilter);
1133                                break;
1134                        case "ancestor":
1135                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1136                                handleFilterLoincAncestor(theCodeSystemIdentifier, theF, theB, theFilter);
1137                                break;
1138                        case "descendant":
1139                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1140                                handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter);
1141                                break;
1142                        case "copyright":
1143                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1144                                handleFilterLoincCopyright(theF, theB, theFilter);
1145                                break;
1146                        default:
1147                                if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) {
1148                                        handleFilterRegex(theF, theB, theFilter);
1149                                } else {
1150                                        handleFilterPropertyDefault(theF, theB, theFilter);
1151                                }
1152                                break;
1153                }
1154        }
1155
1156        private void handleFilterPropertyDefault(SearchPredicateFactory theF,
1157                        BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1158
1159                theB.must(getPropertyNameValueNestedPredicate(theF, theFilter.getProperty(), theFilter.getValue()));
1160        }
1161
1162
1163        private void handleFilterRegex(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1164                /*
1165                 * We treat the regex filter as a match on the regex
1166                 * anywhere in the property string. The spec does not
1167                 * say whether or not this is the right behaviour, but
1168                 * there are examples that seem to suggest that it is.
1169                 */
1170                String value = theFilter.getValue();
1171                if (value.endsWith("$")) {
1172                        value = value.substring(0, value.length() - 1);
1173                } else if (!value.endsWith(".*")) {
1174                        value = value + ".*";
1175                }
1176                if (!value.startsWith("^") && !value.startsWith(".*")) {
1177                        value = ".*" + value;
1178                } else if (value.startsWith("^")) {
1179                        value = value.substring(1);
1180                }
1181
1182                if (isFullTextSetToUseElastic()) {
1183                        ElasticsearchNestedQueryBuilderUtil nestedQueryBuildUtil = new ElasticsearchNestedQueryBuilderUtil(
1184                                "myProperties", "myKey", theFilter.getProperty(),
1185                                "myValueString", value);
1186
1187                        JsonObject nestedQueryJO =  nestedQueryBuildUtil.toGson();
1188
1189                        ourLog.debug("Build nested Elasticsearch query: {}", nestedQueryJO);
1190                        theB.must(theF.extension(ElasticsearchExtension.get()).fromJson(nestedQueryJO));
1191                        return;
1192
1193                }
1194
1195                // native Lucene configured
1196                Query termPropKeyQuery = new TermQuery(new Term(IDX_PROP_KEY, theFilter.getProperty()));
1197                Query regexpValueQuery = new RegexpQuery(new Term(IDX_PROP_VALUE_STRING, value));
1198
1199                theB.must(theF.nested().objectField("myProperties").nest(
1200                        theF.bool()
1201                                .must(theF.extension(LuceneExtension.get()).fromLuceneQuery(termPropKeyQuery))
1202                                .must(theF.extension(LuceneExtension.get()).fromLuceneQuery(regexpValueQuery))
1203                ));
1204        }
1205
1206
1207        private void handleFilterLoincCopyright(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1208                if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1209
1210                        String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase();
1211                        switch (copyrightFilterValue) {
1212                                case "3rdparty":
1213                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1214                                        addFilterLoincCopyright3rdParty(theF, theB, theFilter);
1215                                        break;
1216                                case "loinc":
1217                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1218                                        addFilterLoincCopyrightLoinc(theF, theB, theFilter);
1219                                        break;
1220                                default:
1221                                        throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1222                        }
1223
1224                } else {
1225                        throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty());
1226                }
1227        }
1228
1229        private void addFilterLoincCopyrightLoinc(SearchPredicateFactory theF,
1230                                BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1231
1232                theB.mustNot(theF.match().field(IDX_PROP_KEY).matching("EXTERNAL_COPYRIGHT_NOTICE"));
1233        }
1234
1235
1236        private void addFilterLoincCopyright3rdParty(SearchPredicateFactory thePredicateFactory,
1237                                BooleanPredicateClausesStep<?> theBooleanClause, ValueSet.ConceptSetFilterComponent theFilter) {
1238                //TODO GGG HS These used to be Term term = new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE", ".*");, which was lucene-specific.
1239                //TODO GGG HS ask diederik if this is equivalent.
1240                //This old .* regex is the same as an existence check on a field, which I've implemented here.
1241//              theBooleanClause.must(thePredicateFactory.exists().field("EXTERNAL_COPYRIGHT_NOTICE"));
1242
1243                theBooleanClause.must(thePredicateFactory.match().field(IDX_PROP_KEY).matching("EXTERNAL_COPYRIGHT_NOTICE"));
1244        }
1245
1246        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1247        private void handleFilterLoincAncestor(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1248                switch (theFilter.getOp()) {
1249                        case EQUAL:
1250                                addLoincFilterAncestorEqual(theSystem, f, b, theFilter);
1251                                break;
1252                        case IN:
1253                                addLoincFilterAncestorIn(theSystem, f, b, theFilter);
1254                                break;
1255                        default:
1256                                throw new InvalidRequestException(Msg.code(892) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1257                }
1258
1259        }
1260
1261        private void addLoincFilterAncestorEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1262                long parentPid = getAncestorCodePid(theSystem, theFilter.getProperty(), theFilter.getValue());
1263                b.must(f.match().field("myParentPids").matching(String.valueOf(parentPid)));
1264        }
1265
1266
1267        private void addLoincFilterAncestorIn(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1268                String[] values = theFilter.getValue().split(",");
1269                List<Long> ancestorCodePidList = new ArrayList<>();
1270                for (String value : values) {
1271                        ancestorCodePidList.add(getAncestorCodePid(theSystem, theFilter.getProperty(), value));
1272                }
1273
1274                b.must(f.bool(innerB -> ancestorCodePidList.forEach(
1275                        ancestorPid -> innerB.should(f.match().field("myParentPids").matching(String.valueOf(ancestorPid)))
1276                )));
1277        }
1278
1279
1280        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1281        private void handleFilterLoincParentChild(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1282                switch (theFilter.getOp()) {
1283                        case EQUAL:
1284                                addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue());
1285                                break;
1286                        case IN:
1287                                addLoincFilterParentChildIn(f, b, theFilter);
1288                                break;
1289                        default:
1290                                throw new InvalidRequestException(Msg.code(893) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1291                }
1292        }
1293
1294        private void addLoincFilterParentChildIn(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1295                String[] values = theFilter.getValue().split(",");
1296                b.minimumShouldMatchNumber(1);
1297                for (String value : values) {
1298                        logFilteringValueOnProperty(value, theFilter.getProperty());
1299                        b.should(getPropertyNameValueNestedPredicate(f, theFilter.getProperty(), value));
1300                }
1301        }
1302
1303        private void addLoincFilterParentChildEqual(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) {
1304                logFilteringValueOnProperty(theValue, theProperty);
1305                b.must(getPropertyNameValueNestedPredicate(f, theProperty, theValue));
1306        }
1307
1308        /**
1309         * A nested predicate is required for both predicates to be applied to same property, otherwise if properties
1310         * propAA:valueAA and propBB:valueBB are defined, a search for propAA:valueBB would be a match
1311         * @see "https://docs.jboss.org/hibernate/search/6.0/reference/en-US/html_single/#search-dsl-predicate-nested"
1312         */
1313        private PredicateFinalStep getPropertyNameValueNestedPredicate(SearchPredicateFactory f, String theProperty, String theValue) {
1314                return f.nested().objectField(IDX_PROPERTIES).nest(
1315                        f.bool()
1316                                .must(f.match().field(IDX_PROP_KEY).matching(theProperty))
1317                                .must(f.match().field(IDX_PROP_VALUE_STRING).field(IDX_PROP_DISPLAY_STRING).matching(theValue))
1318                );
1319        }
1320
1321
1322        private void handleFilterConceptAndCode(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1323                TermConcept code = findCodeForFilterCriteria(theSystem, theFilter);
1324
1325                if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
1326                        ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
1327
1328                        b.must(f.match().field("myParentPids").matching("" + code.getId()));
1329                } else {
1330                        throwInvalidFilter(theFilter, "");
1331                }
1332        }
1333
1334        @Nonnull
1335        private TermConcept findCodeForFilterCriteria(String theSystem, ValueSet.ConceptSetFilterComponent theFilter) {
1336                return findCode(theSystem, theFilter.getValue())
1337                        .orElseThrow(() -> new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
1338        }
1339
1340        private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) {
1341                throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty() + theErrorSuffix);
1342        }
1343
1344        private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
1345                String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
1346                if (!isCodeSystemLoinc(systemUrl)) {
1347                        throw new InvalidRequestException(Msg.code(895) + "Invalid filter, property " + theProperty + " is LOINC-specific and cannot be used with system: " + systemUrl);
1348                }
1349        }
1350
1351        private boolean isCodeSystemLoinc(String theSystem) {
1352                return LOINC_URI.equals(theSystem);
1353        }
1354
1355        private void handleFilterDisplay(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1356                if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1357                        addDisplayFilterExact(f, b, theFilter);
1358                } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1359                        if (theFilter.getValue().trim().contains(" ")) {
1360                                addDisplayFilterExact(f, b, theFilter);
1361                        } else {
1362                                addDisplayFilterInexact(f, b, theFilter);
1363                        }
1364                }
1365        }
1366
1367        private void addDisplayFilterExact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
1368                bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue()));
1369        }
1370
1371        private void addDisplayFilterInexact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
1372                bool.must(f.phrase()
1373                        .field("myDisplay").boost(4.0f)
1374                        .field("myDisplayWordEdgeNGram").boost(1.0f)
1375                        .field("myDisplayEdgeNGram").boost(1.0f)
1376                        .matching(nextFilter.getValue().toLowerCase())
1377                        .slop(2)
1378                );
1379        }
1380
1381        private long getAncestorCodePid(String theSystem, String theProperty, String theValue) {
1382                TermConcept code = findCode(theSystem, theValue)
1383                        .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1384
1385                logFilteringValueOnProperty(theValue, theProperty);
1386                return code.getId();
1387        }
1388
1389        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1390        private void handleFilterLoincDescendant(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1391                switch (theFilter.getOp()) {
1392                        case EQUAL:
1393                                addLoincFilterDescendantEqual(theSystem, f, b, theFilter);
1394                                break;
1395                        case IN:
1396                                addLoincFilterDescendantIn(theSystem, f, b, theFilter);
1397                                break;
1398                        default:
1399                                throw new InvalidRequestException(Msg.code(896) + "Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1400                }
1401        }
1402
1403
1404        private void addLoincFilterDescendantEqual(String theSystem, SearchPredicateFactory f,
1405                        BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1406
1407                List<Long> parentPids = getCodeParentPids(theSystem, theFilter.getProperty(), theFilter.getValue());
1408                if (parentPids.isEmpty()) {
1409                        // Can't return empty must, because it wil match according to other predicates.
1410                        // Some day there will be a 'matchNone' predicate (https://discourse.hibernate.org/t/fail-fast-predicate/6062)
1411                        b.mustNot( f.matchAll() );
1412                        return;
1413                }
1414
1415                b.must(f.bool(innerB -> {
1416                        innerB.minimumShouldMatchNumber(1);
1417                        parentPids.forEach(pid -> innerB.should(f.match().field("myId").matching(pid)));
1418                }));
1419
1420        }
1421
1422        /**
1423         * We are looking for codes which have codes indicated in theFilter.getValue() as descendants.
1424         * Strategy is to find codes which have their pId(s) in the list of the parentId(s) of all the TermConcept(s)
1425         * representing the codes in theFilter.getValue()
1426         */
1427        private void addLoincFilterDescendantIn(String theSystem, SearchPredicateFactory f,
1428                        BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1429
1430                String[] values = theFilter.getValue().split(",");
1431                if (values.length == 0) {
1432                        throw new InvalidRequestException(Msg.code(2062) + "Invalid filter criteria - no codes specified");
1433                }
1434
1435                List<Long> descendantCodePidList = getMultipleCodeParentPids(theSystem, theFilter.getProperty(), values);
1436
1437                b.must(f.bool(innerB -> descendantCodePidList.forEach(
1438                        pId -> innerB.should(f.match().field("myId").matching(pId))
1439                )));
1440        }
1441
1442
1443        /**
1444         * Returns the list of parentId(s) of the TermConcept representing theValue as a code
1445         */
1446        private List<Long> getCodeParentPids(String theSystem, String theProperty, String theValue) {
1447                TermConcept code = findCode(theSystem, theValue)
1448                        .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" +
1449                                Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1450
1451                String[] parentPids = code.getParentPidsAsString().split(" ");
1452                List<Long> retVal = Arrays.stream(parentPids)
1453                        .filter( pid -> !StringUtils.equals(pid, "NONE") )
1454                        .map(Long::parseLong)
1455                        .collect(Collectors.toList());
1456                logFilteringValueOnProperty(theValue, theProperty);
1457                return retVal;
1458        }
1459
1460
1461        /**
1462         * Returns the list of parentId(s) of the TermConcept representing theValue as a code
1463         */
1464        private List<Long> getMultipleCodeParentPids(String theSystem, String theProperty, String[] theValues) {
1465                List<String> valuesList = Arrays.asList(theValues);
1466                List<TermConcept> termConcepts = findCodes(theSystem, valuesList);
1467                if (valuesList.size() != termConcepts.size()) {
1468                        String exMsg = getTermConceptsFetchExceptionMsg(termConcepts, valuesList);
1469                        throw new InvalidRequestException(Msg.code(2064) + "Invalid filter criteria - {" +
1470                                Constants.codeSystemWithDefaultDescription(theSystem) + "}: " + exMsg);
1471                }
1472
1473                List<Long> retVal = termConcepts.stream()
1474                        .flatMap(tc -> Arrays.stream(tc.getParentPidsAsString().split(" ")))
1475                        .filter( pid -> !StringUtils.equals(pid, "NONE") )
1476                        .map(Long::parseLong)
1477                        .collect(Collectors.toList());
1478
1479                logFilteringValueOnProperties(valuesList, theProperty);
1480
1481                return retVal;
1482        }
1483
1484        /**
1485         * Generate message indicating for which of theValues a TermConcept was not found
1486         */
1487        private String getTermConceptsFetchExceptionMsg(List<TermConcept> theTermConcepts, List<String> theValues) {
1488                // case: more TermConcept(s) retrieved than codes queried
1489                if (theTermConcepts.size() > theValues.size()) {
1490                        return "Invalid filter criteria - More TermConcepts were found than indicated codes. Queried codes: [" +
1491                                join(",", theValues + "]; Obtained TermConcept IDs, codes: [" +
1492                                        theTermConcepts.stream().map(tc -> tc.getId() + ", " + tc.getCode())
1493                                                .collect(joining("; "))+ "]");
1494                }
1495
1496                // case: less TermConcept(s) retrieved than codes queried
1497                Set<String> matchedCodes = theTermConcepts.stream().map(TermConcept::getCode).collect(toSet());
1498                List<String> notMatchedValues = theValues.stream()
1499                        .filter(v -> ! matchedCodes.contains (v)) .collect(toList());
1500
1501                return "Invalid filter criteria - No TermConcept(s) were found for the requested codes: [" +
1502                        join(",", notMatchedValues + "]");
1503        }
1504
1505
1506        private void logFilteringValueOnProperty(String theValue, String theProperty) {
1507                ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty);
1508        }
1509
1510        private void logFilteringValueOnProperties(List<String> theValues, String theProperty) {
1511                ourLog.debug(" * Filtering with values={} on property {}", String.join(", ", theValues), theProperty);
1512        }
1513
1514        private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) {
1515                throw new InvalidRequestException(Msg.code(897) + "Don't know how to handle op=" + theOp + " on property " + theProperty);
1516        }
1517
1518        private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) {
1519                throw new InvalidRequestException(Msg.code(898) + "Don't know how to handle value=" + theValue + " on property " + theProperty);
1520        }
1521
1522        private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd) {
1523                ourLog.trace("Hibernate search is not enabled");
1524
1525                if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
1526                        Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
1527                }
1528
1529                Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
1530
1531                for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
1532                        boolean handled = false;
1533                        switch (nextFilter.getProperty()) {
1534                                case "concept":
1535                                case "code":
1536                                        if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
1537                                                theValueSetCodeAccumulator.addMessage("Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient.");
1538                                                TermConcept code = findCodeForFilterCriteria(theSystem, nextFilter);
1539                                                addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code);
1540                                                handled = true;
1541                                        }
1542                                        break;
1543                        }
1544
1545                        if (!handled) {
1546                                throwInvalidFilter(nextFilter, " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion funtionality is available.");
1547                        }
1548                }
1549
1550                if (theInclude.getConcept().isEmpty()) {
1551
1552                        Collection<TermConcept> concepts = myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid());
1553                        for (TermConcept next : concepts) {
1554                                addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), next.getId(), next.getParentPidsAsString(), next.getDesignations());
1555                        }
1556                }
1557
1558                for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) {
1559                        if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
1560                                continue;
1561                        }
1562                        Collection<TermConceptDesignation> designations = next
1563                                .getDesignation()
1564                                .stream()
1565                                .map(t->new TermConceptDesignation()
1566                                        .setValue(t.getValue())
1567                                        .setLanguage(t.getLanguage())
1568                                        .setUseCode(t.getUse().getCode())
1569                                        .setUseSystem(t.getUse().getSystem())
1570                                        .setUseDisplay(t.getUse().getDisplay())
1571                                )
1572                                .collect(Collectors.toList());
1573                        addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), null, null, designations);
1574                }
1575
1576
1577        }
1578
1579        private void addConceptAndChildren(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd, TermConcept theConcept) {
1580                for (TermConcept nextChild : theConcept.getChildCodes()) {
1581                        boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), nextChild.getCode(), nextChild.getDisplay(), nextChild.getId(), nextChild.getParentPidsAsString(), nextChild.getDesignations());
1582                        if (added) {
1583                                addConceptAndChildren(theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild);
1584                        }
1585                }
1586        }
1587
1588        @Override
1589        @Transactional
1590        public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) {
1591                IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails);
1592                ValueSet canonicalValueSet = toCanonicalValueSet(valueSet);
1593                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet);
1594                if (!optionalTermValueSet.isPresent()) {
1595                        return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId);
1596                }
1597
1598                ourLog.info("Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl());
1599
1600                TermValueSet termValueSet = optionalTermValueSet.get();
1601                if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) {
1602                        return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetCantInvalidateNotYetPrecalculated", termValueSet.getUrl(), termValueSet.getExpansionStatus());
1603                }
1604
1605                Long totalConcepts = termValueSet.getTotalConcepts();
1606
1607                deletePreCalculatedValueSetContents(termValueSet);
1608
1609                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
1610                termValueSet.setExpansionTimestamp(null);
1611                myTermValueSetDao.save(termValueSet);
1612                return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts);
1613        }
1614
1615        @Override
1616        @Transactional
1617        public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) {
1618                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet);
1619
1620                if (!optionalTermValueSet.isPresent()) {
1621                        ourLog.warn("ValueSet is not present in terminology tables. Will perform in-memory code validation. {}", getValueSetInfo(theValueSet));
1622                        return false;
1623                }
1624
1625                TermValueSet termValueSet = optionalTermValueSet.get();
1626
1627                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
1628                        ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}",
1629                                getValueSetInfo(theValueSet), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
1630                        return false;
1631                }
1632
1633                return true;
1634        }
1635
1636        private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) {
1637                ResourcePersistentId valueSetResourcePid = getValueSetResourcePersistentId(theValueSet);
1638                Optional<TermValueSet> optionalTermValueSet = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong());
1639                return optionalTermValueSet;
1640        }
1641
1642        private ResourcePersistentId getValueSetResourcePersistentId(ValueSet theValueSet) {
1643                ResourcePersistentId valueSetResourcePid = myIdHelperService.resolveResourcePersistentIds(RequestPartitionId.allPartitions(), theValueSet.getIdElement().getResourceType(), theValueSet.getIdElement().getIdPart());
1644                return valueSetResourcePid;
1645        }
1646
1647        protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet(
1648                ConceptValidationOptions theValidationOptions,
1649                ValueSet theValueSet, String theSystem, String theCode, String theDisplay, Coding theCoding, CodeableConcept theCodeableConcept) {
1650                assert TransactionSynchronizationManager.isSynchronizationActive();
1651
1652                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required");
1653                ResourcePersistentId valueSetResourcePid = getValueSetResourcePersistentId(theValueSet);
1654
1655
1656                List<TermValueSetConcept> concepts = new ArrayList<>();
1657                if (isNotBlank(theCode)) {
1658                        if (theValidationOptions.isInferSystem()) {
1659                                concepts.addAll(myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getIdAsLong(), theCode));
1660                        } else if (isNotBlank(theSystem)) {
1661                                concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode));
1662                        }
1663                } else if (theCoding != null) {
1664                        if (theCoding.hasSystem() && theCoding.hasCode()) {
1665                                concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theCoding.getSystem(), theCoding.getCode()));
1666                        }
1667                } else if (theCodeableConcept != null) {
1668                        for (Coding coding : theCodeableConcept.getCoding()) {
1669                                if (coding.hasSystem() && coding.hasCode()) {
1670                                        concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, coding.getSystem(), coding.getCode()));
1671                                        if (!concepts.isEmpty()) {
1672                                                break;
1673                                        }
1674                                }
1675                        }
1676                } else {
1677                        return null;
1678                }
1679
1680                TermValueSet valueSetEntity = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong()).orElseThrow(() -> new IllegalStateException());
1681                String timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity);
1682                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription);
1683
1684                if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) {
1685                        String systemVersion = null;
1686                        for (TermValueSetConcept concept : concepts) {
1687                                systemVersion = concept.getSystemVersion();
1688                                if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) {
1689                                        return new IValidationSupport.CodeValidationResult()
1690                                                .setCode(concept.getCode())
1691                                                .setDisplay(concept.getDisplay())
1692                                                .setCodeSystemVersion(concept.getSystemVersion())
1693                                                .setMessage(msg);
1694                                }
1695                        }
1696
1697                        return createFailureCodeValidationResult(theSystem, theCode, systemVersion, " - Concept Display \"" + theDisplay + "\" does not match expected \"" + concepts.get(0).getDisplay() + "\". " + msg).setDisplay(concepts.get(0).getDisplay());
1698                }
1699
1700                if (!concepts.isEmpty()) {
1701                        return new IValidationSupport.CodeValidationResult()
1702                                .setCode(concepts.get(0).getCode())
1703                                .setDisplay(concepts.get(0).getDisplay())
1704                                .setCodeSystemVersion(concepts.get(0).getSystemVersion())
1705                                .setMessage(msg);
1706                }
1707                
1708                // Ok, we failed
1709                List<TermValueSetConcept> outcome = myValueSetConceptDao.findByTermValueSetIdSystemOnly(Pageable.ofSize(1), valueSetEntity.getId(), theSystem);
1710                String append;
1711                if (outcome.size() == 0) {
1712                        append = " - No codes in ValueSet belong to CodeSystem with URL " + theSystem;
1713                } else {
1714                        append = " - Unknown code " + theSystem + "#" + theCode + ". " + msg;
1715                }
1716
1717                return createFailureCodeValidationResult(theSystem, theCode, null, append);
1718        }
1719
1720        private CodeValidationResult createFailureCodeValidationResult(String theSystem, String theCode, String theCodeSystemVersion, String theAppend) {
1721                return new CodeValidationResult()
1722                        .setSeverity(IssueSeverity.ERROR)
1723                        .setCodeSystemVersion(theCodeSystemVersion)
1724                        .setMessage("Unable to validate code " + theSystem + "#" + theCode + theAppend);
1725        }
1726
1727        private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(ResourcePersistentId theResourcePid, String theSystem, String theCode) {
1728                assert TransactionSynchronizationManager.isSynchronizationActive();
1729
1730                List<TermValueSetConcept> retVal = new ArrayList<>();
1731                Optional<TermValueSetConcept> optionalTermValueSetConcept;
1732                int versionIndex = theSystem.indexOf("|");
1733                if (versionIndex >= 0) {
1734                        String systemUrl = theSystem.substring(0, versionIndex);
1735                        String systemVersion = theSystem.substring(versionIndex + 1);
1736                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion(theResourcePid.getIdAsLong(), systemUrl, systemVersion, theCode);
1737                } else {
1738                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode(theResourcePid.getIdAsLong(), theSystem, theCode);
1739                }
1740                optionalTermValueSetConcept.ifPresent(retVal::add);
1741                return retVal;
1742        }
1743
1744        private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
1745                for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
1746                        TermConcept nextChild = nextChildLink.getChild();
1747                        if (addToSet(theSetToPopulate, nextChild)) {
1748                                fetchChildren(nextChild, theSetToPopulate);
1749                        }
1750                }
1751        }
1752
1753        private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) {
1754                TermCodeSystemVersion codeSystem = myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid);
1755                return myConceptDao.findByCodeSystemAndCode(codeSystem, theCode);
1756        }
1757
1758        private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
1759                for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) {
1760                        TermConcept nextChild = nextChildLink.getParent();
1761                        if (addToSet(theSetToPopulate, nextChild)) {
1762                                fetchParents(nextChild, theSetToPopulate);
1763                        }
1764                }
1765        }
1766
1767        private CodeSystem.ConceptDefinitionComponent findCode(List<CodeSystem.ConceptDefinitionComponent> theConcepts, String theCode) {
1768                for (CodeSystem.ConceptDefinitionComponent next : theConcepts) {
1769                        if (theCode.equals(next.getCode())) {
1770                                return next;
1771                        }
1772                        CodeSystem.ConceptDefinitionComponent val = findCode(next.getConcept(), theCode);
1773                        if (val != null) {
1774                                return val;
1775                        }
1776                }
1777                return null;
1778        }
1779
1780        @Override
1781        public Optional<TermConcept> findCode(String theCodeSystem, String theCode) {
1782                /*
1783                 * Loading concepts without a transaction causes issues later on some
1784                 * platforms (e.g. PSQL) so this transactiontemplate is here to make
1785                 * sure that we always call this with an open transaction
1786                 */
1787                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
1788                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_MANDATORY);
1789
1790                return txTemplate.execute(t -> {
1791                        TermCodeSystemVersion csv = getCurrentCodeSystemVersion(theCodeSystem);
1792                        if (csv == null) {
1793                                return Optional.empty();
1794                        }
1795                        return myConceptDao.findByCodeSystemAndCode(csv, theCode);
1796                });
1797        }
1798
1799
1800        @Transactional(propagation = Propagation.MANDATORY)
1801        public List<TermConcept> findCodes(String theCodeSystem, List<String> theCodeList) {
1802                TermCodeSystemVersion csv = getCurrentCodeSystemVersion(theCodeSystem);
1803                if (csv == null) { return Collections.emptyList(); }
1804
1805                return myConceptDao.findByCodeSystemAndCodeList(csv, theCodeList);
1806        }
1807
1808
1809        @Nullable
1810        private TermCodeSystemVersion getCurrentCodeSystemVersion(String theCodeSystemIdentifier) {
1811                String version = getVersionFromIdentifier(theCodeSystemIdentifier);
1812                TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theCodeSystemIdentifier, t -> myTxTemplate.execute(tx -> {
1813                        TermCodeSystemVersion csv = null;
1814                        TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier));
1815                        if (cs != null) {
1816                                if (version != null) {
1817                                        csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version);
1818                                } else if (cs.getCurrentVersion() != null) {
1819                                        csv = cs.getCurrentVersion();
1820                                }
1821                        }
1822                        if (csv != null) {
1823                                return csv;
1824                        } else {
1825                                return NO_CURRENT_VERSION;
1826                        }
1827                }));
1828                if (retVal == NO_CURRENT_VERSION) {
1829                        return null;
1830                }
1831                return retVal;
1832        }
1833
1834        private String getVersionFromIdentifier(String theUri) {
1835                String retVal = null;
1836                if (StringUtils.isNotEmpty((theUri))) {
1837                        int versionSeparator = theUri.lastIndexOf('|');
1838                        if (versionSeparator != -1) {
1839                                retVal = theUri.substring(versionSeparator + 1);
1840                        }
1841                }
1842                return retVal;
1843        }
1844
1845        private String getUrlFromIdentifier(String theUri) {
1846                String retVal = theUri;
1847                if (StringUtils.isNotEmpty((theUri))) {
1848                        int versionSeparator = theUri.lastIndexOf('|');
1849                        if (versionSeparator != -1) {
1850                                retVal = theUri.substring(0, versionSeparator);
1851                        }
1852                }
1853                return retVal;
1854        }
1855
1856        @Transactional(propagation = Propagation.REQUIRED)
1857        @Override
1858        public Set<TermConcept> findCodesAbove(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
1859                StopWatch stopwatch = new StopWatch();
1860
1861                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
1862                if (!concept.isPresent()) {
1863                        return Collections.emptySet();
1864                }
1865
1866                Set<TermConcept> retVal = new HashSet<>();
1867                retVal.add(concept.get());
1868
1869                fetchParents(concept.get(), retVal);
1870
1871                ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
1872                return retVal;
1873        }
1874
1875        @Transactional
1876        @Override
1877        public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) {
1878                TermCodeSystem cs = getCodeSystem(theSystem);
1879                if (cs == null) {
1880                        return findCodesAboveUsingBuiltInSystems(theSystem, theCode);
1881                }
1882                TermCodeSystemVersion csv = cs.getCurrentVersion();
1883
1884                Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode);
1885                return toVersionIndependentConcepts(theSystem, codes);
1886        }
1887
1888        @Transactional(propagation = Propagation.REQUIRED)
1889        @Override
1890        public Set<TermConcept> findCodesBelow(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
1891                Stopwatch stopwatch = Stopwatch.createStarted();
1892
1893                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
1894                if (!concept.isPresent()) {
1895                        return Collections.emptySet();
1896                }
1897
1898                Set<TermConcept> retVal = new HashSet<>();
1899                retVal.add(concept.get());
1900
1901                fetchChildren(concept.get(), retVal);
1902
1903                ourLog.debug("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
1904                return retVal;
1905        }
1906
1907        @Transactional
1908        @Override
1909        public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) {
1910                TermCodeSystem cs = getCodeSystem(theSystem);
1911                if (cs == null) {
1912                        return findCodesBelowUsingBuiltInSystems(theSystem, theCode);
1913                }
1914                TermCodeSystemVersion csv = cs.getCurrentVersion();
1915
1916                Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode);
1917                return toVersionIndependentConcepts(theSystem, codes);
1918        }
1919
1920        private TermCodeSystem getCodeSystem(String theSystem) {
1921                return myCodeSystemDao.findByCodeSystemUri(theSystem);
1922        }
1923
1924        @PostConstruct
1925        public void start() {
1926                RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute();
1927                rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class));
1928                myTxTemplate = new TransactionTemplate(myTransactionManager, rules);
1929                scheduleJob();
1930        }
1931
1932        public void scheduleJob() {
1933                // Register scheduled job to pre-expand ValueSets
1934                // In the future it would be great to make this a cluster-aware task somehow
1935                ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition();
1936                vsJobDefinition.setId(getClass().getName());
1937                vsJobDefinition.setJobClass(Job.class);
1938                mySchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition);
1939        }
1940
1941        @Override
1942        public synchronized void preExpandDeferredValueSetsToTerminologyTables() {
1943                if (!myDaoConfig.isEnableTaskPreExpandValueSets()) {
1944                        return;
1945                }
1946                if (isNotSafeToPreExpandValueSets()) {
1947                        ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded.");
1948                        return;
1949                }
1950                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
1951
1952                while (true) {
1953                        StopWatch sw = new StopWatch();
1954                        TermValueSet valueSetToExpand = txTemplate.execute(t -> {
1955                                Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
1956                                if (!optionalTermValueSet.isPresent()) {
1957                                        return null;
1958                                }
1959
1960                                TermValueSet termValueSet = optionalTermValueSet.get();
1961                                termValueSet.setTotalConcepts(0L);
1962                                termValueSet.setTotalConceptDesignations(0L);
1963                                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS);
1964                                return myTermValueSetDao.saveAndFlush(termValueSet);
1965                        });
1966                        if (valueSetToExpand == null) {
1967                                return;
1968                        }
1969
1970                        // We have a ValueSet to pre-expand.
1971                        setPreExpandingValueSets(true);
1972                        try {
1973                                ValueSet valueSet = txTemplate.execute(t -> {
1974                                        TermValueSet refreshedValueSetToExpand = myTermValueSetDao.findById(valueSetToExpand.getId()).orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId()));
1975                                        return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
1976                                });
1977                                assert valueSet != null;
1978
1979                                ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myTermValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
1980                                ValueSetExpansionOptions options = new ValueSetExpansionOptions();
1981                                options.setIncludeHierarchy(true);
1982                                expandValueSet(options, valueSet, accumulator);
1983
1984                                // We are done with this ValueSet.
1985                                txTemplate.executeWithoutResult(t -> {
1986                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
1987                                        valueSetToExpand.setExpansionTimestamp(new Date());
1988                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
1989
1990                                });
1991
1992                                ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw);
1993
1994                        } catch (Exception e) {
1995                                ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
1996                                txTemplate.executeWithoutResult(t -> {
1997                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
1998                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
1999
2000                                });
2001
2002                        } finally {
2003                                setPreExpandingValueSets(false);
2004                        }
2005                }
2006        }
2007
2008        private synchronized void setPreExpandingValueSets(boolean thePreExpandingValueSets) {
2009                myPreExpandingValueSets = thePreExpandingValueSets;
2010        }
2011
2012        private synchronized boolean isPreExpandingValueSets() {
2013                return myPreExpandingValueSets;
2014        }
2015
2016        @Override
2017        @Transactional
2018        public CodeValidationResult validateCode(ConceptValidationOptions theOptions, IIdType theValueSetId, String theValueSetIdentifier, String theCodeSystemIdentifierToValidate, String theCodeToValidate, String theDisplayToValidate, IBaseDatatype theCodingToValidate, IBaseDatatype theCodeableConceptToValidate) {
2019
2020                CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConceptToValidate);
2021                boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0;
2022
2023                Coding canonicalCodingToValidate = toCanonicalCoding(theCodingToValidate);
2024                boolean haveCoding = canonicalCodingToValidate != null && canonicalCodingToValidate.isEmpty() == false;
2025
2026                boolean haveCode = theCodeToValidate != null && theCodeToValidate.isEmpty() == false;
2027
2028                if (!haveCodeableConcept && !haveCoding && !haveCode) {
2029                        throw new InvalidRequestException(Msg.code(899) + "No code, coding, or codeableConcept provided to validate");
2030                }
2031                if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) {
2032                        throw new InvalidRequestException(Msg.code(900) + "$validate-code can only validate (system AND code) OR (coding) OR (codeableConcept)");
2033                }
2034
2035                boolean haveIdentifierParam = isNotBlank(theValueSetIdentifier);
2036                String valueSetIdentifier;
2037                if (theValueSetId != null) {
2038                        IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId);
2039                        StringBuilder valueSetIdentifierBuilder = new StringBuilder(CommonCodeSystemsTerminologyService.getValueSetUrl(valueSet));
2040                        String valueSetVersion = CommonCodeSystemsTerminologyService.getValueSetVersion(valueSet);
2041                        if (valueSetVersion != null) {
2042                                valueSetIdentifierBuilder.append("|").append(valueSetVersion);
2043                        }
2044                        valueSetIdentifier = valueSetIdentifierBuilder.toString();
2045
2046                } else if (haveIdentifierParam) {
2047                        valueSetIdentifier = theValueSetIdentifier;
2048                } else {
2049                        throw new InvalidRequestException(Msg.code(901) + "Either ValueSet ID or ValueSet identifier or system and code must be provided. Unable to validate.");
2050                }
2051
2052                ValidationSupportContext validationContext = new ValidationSupportContext(provideValidationSupport());
2053
2054                String codeValueToValidate = theCodeToValidate;
2055                String codeSystemIdentifierValueToValidate = theCodeSystemIdentifierToValidate;
2056                String codeDisplayValueToValidate = theDisplayToValidate;
2057
2058                if (haveCodeableConcept) {
2059                        for (int i = 0; i < codeableConcept.getCoding().size(); i++) {
2060                                Coding nextCoding = codeableConcept.getCoding().get(i);
2061                                String codeSystemIdentifier;
2062                                if (nextCoding.hasVersion()) {
2063                                        codeSystemIdentifier = nextCoding.getSystem() + "|" + nextCoding.getVersion();
2064                                } else {
2065                                        codeSystemIdentifier = nextCoding.getSystem();
2066                                }
2067                                CodeValidationResult nextValidation = validateCode(validationContext, theOptions, codeSystemIdentifier, nextCoding.getCode(), nextCoding.getDisplay(), valueSetIdentifier);
2068                                if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
2069                                        return nextValidation;
2070                                }
2071                        }
2072                } else if (haveCoding) {
2073                        if (canonicalCodingToValidate.hasVersion()) {
2074                                codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem() + "|" + canonicalCodingToValidate.getVersion();
2075                        } else {
2076                                codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem();
2077                        }
2078                        codeValueToValidate = canonicalCodingToValidate.getCode();
2079                        codeDisplayValueToValidate = canonicalCodingToValidate.getDisplay();
2080                }
2081
2082                return validateCode(validationContext, theOptions, codeSystemIdentifierValueToValidate, codeValueToValidate, codeDisplayValueToValidate, valueSetIdentifier);
2083        }
2084
2085        private boolean isNotSafeToPreExpandValueSets() {
2086                return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty();
2087        }
2088
2089        protected abstract ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable);
2090
2091        private Optional<TermValueSet> getNextTermValueSetNotExpanded() {
2092                Optional<TermValueSet> retVal = Optional.empty();
2093                Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
2094
2095                if (!page.getContent().isEmpty()) {
2096                        retVal = Optional.of(page.getContent().get(0));
2097                }
2098
2099                return retVal;
2100        }
2101
2102        @Override
2103        @Transactional
2104        public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) {
2105
2106                ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied");
2107                if (isPlaceholder(theValueSet)) {
2108                        ourLog.info("Not storing TermValueSet for placeholder {}", theValueSet.getIdElement().toVersionless().getValueAsString());
2109                        return;
2110                }
2111
2112                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theValueSet.getUrl(), "ValueSet has no value for ValueSet.url");
2113                ourLog.info("Storing TermValueSet for {}", theValueSet.getIdElement().toVersionless().getValueAsString());
2114
2115                /*
2116                 * Get CodeSystem and validate CodeSystemVersion
2117                 */
2118                TermValueSet termValueSet = new TermValueSet();
2119                termValueSet.setResource(theResourceTable);
2120                termValueSet.setUrl(theValueSet.getUrl());
2121                termValueSet.setVersion(theValueSet.getVersion());
2122                termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null);
2123
2124                // Delete version being replaced
2125                deleteValueSetForResource(theResourceTable);
2126
2127                /*
2128                 * Do the upload.
2129                 */
2130                String url = termValueSet.getUrl();
2131                String version = termValueSet.getVersion();
2132                Optional<TermValueSet> optionalExistingTermValueSetByUrl;
2133                if (version != null) {
2134                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version);
2135                } else {
2136                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url);
2137                }
2138                if (!optionalExistingTermValueSetByUrl.isPresent()) {
2139
2140                        myTermValueSetDao.save(termValueSet);
2141
2142                } else {
2143                        TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get();
2144                        String msg;
2145                        if (version != null) {
2146                                msg = myContext.getLocalizer().getMessage(
2147                                        BaseTermReadSvcImpl.class,
2148                                        "cannotCreateDuplicateValueSetUrlAndVersion",
2149                                        url, version, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue());
2150                        } else {
2151                                msg = myContext.getLocalizer().getMessage(
2152                                        BaseTermReadSvcImpl.class,
2153                                        "cannotCreateDuplicateValueSetUrl",
2154                                        url, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue());
2155                        }
2156                        throw new UnprocessableEntityException(Msg.code(902) + msg);
2157                }
2158        }
2159
2160        @Override
2161        @Transactional
2162        public IFhirResourceDaoCodeSystem.SubsumesResult subsumes(IPrimitiveType<String> theCodeA, IPrimitiveType<String> theCodeB,
2163                                                                                                                                                                 IPrimitiveType<String> theSystem, IBaseCoding theCodingA, IBaseCoding theCodingB) {
2164                FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA);
2165                FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB);
2166
2167                if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) {
2168                        throw new InvalidRequestException(Msg.code(903) + "Unable to test subsumption across different code systems");
2169                }
2170
2171                if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) {
2172                        throw new InvalidRequestException(Msg.code(904) + "Unable to test subsumption across different code system versions");
2173                }
2174
2175                String codeASystemIdentifier;
2176                if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) {
2177                        codeASystemIdentifier = conceptA.getSystem() + "|" + conceptA.getSystemVersion();
2178                } else {
2179                        codeASystemIdentifier = conceptA.getSystem();
2180                }
2181                TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode())
2182                        .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA));
2183
2184                String codeBSystemIdentifier;
2185                if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) {
2186                        codeBSystemIdentifier = conceptB.getSystem() + "|" + conceptB.getSystemVersion();
2187                } else {
2188                        codeBSystemIdentifier = conceptB.getSystem();
2189                }
2190                TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode())
2191                        .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB));
2192
2193                SearchSession searchSession = Search.session(myEntityManager);
2194
2195                ConceptSubsumptionOutcome subsumes;
2196                subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES);
2197                if (subsumes == null) {
2198                        subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY);
2199                }
2200                if (subsumes == null) {
2201                        subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED;
2202                }
2203
2204                return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes);
2205        }
2206
2207        protected abstract ValueSet toCanonicalValueSet(IBaseResource theValueSet);
2208
2209        protected IValidationSupport.LookupCodeResult lookupCode(String theSystem, String theCode, String theDisplayLanguage) {
2210                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2211                return txTemplate.execute(t -> {
2212                        Optional<TermConcept> codeOpt = findCode(theSystem, theCode);
2213                        if (codeOpt.isPresent()) {
2214                                TermConcept code = codeOpt.get();
2215
2216                                IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult();
2217                                result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName());
2218                                result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId());
2219                                result.setSearchedForSystem(theSystem);
2220                                result.setSearchedForCode(theCode);
2221                                result.setFound(true);
2222                                result.setCodeDisplay(code.getDisplay());
2223
2224                                for (TermConceptDesignation next : code.getDesignations()) {
2225                                        // filter out the designation based on displayLanguage if any
2226                                        if (isDisplayLanguageMatch(theDisplayLanguage, next.getLanguage())) {
2227                                                IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation();
2228                                                designation.setLanguage(next.getLanguage());
2229                                                designation.setUseSystem(next.getUseSystem());
2230                                                designation.setUseCode(next.getUseCode());
2231                                                designation.setUseDisplay(next.getUseDisplay());
2232                                                designation.setValue(next.getValue());
2233                                                result.getDesignations().add(designation);
2234                                        }
2235                                }
2236
2237                                for (TermConceptProperty next : code.getProperties()) {
2238                                        if (next.getType() == TermConceptPropertyTypeEnum.CODING) {
2239                                                IValidationSupport.CodingConceptProperty property = new IValidationSupport.CodingConceptProperty(next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay());
2240                                                result.getProperties().add(property);
2241                                        } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) {
2242                                                IValidationSupport.StringConceptProperty property = new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue());
2243                                                result.getProperties().add(property);
2244                                        } else {
2245                                                throw new InternalErrorException(Msg.code(905) + "Unknown type: " + next.getType());
2246                                        }
2247                                }
2248
2249                                return result;
2250
2251                        } else {
2252                                return new LookupCodeResult()
2253                                        .setFound(false);
2254                        }
2255                });
2256        }
2257
2258        @Nullable
2259        private ConceptSubsumptionOutcome testForSubsumption(SearchSession theSearchSession, TermConcept theLeft, TermConcept theRight, ConceptSubsumptionOutcome theOutput) {
2260                List<TermConcept> fetch = theSearchSession.search(TermConcept.class)
2261                        .where(f -> f.bool()
2262                                .must(f.match().field("myId").matching(theRight.getId()))
2263                                .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId())))
2264                        ).fetchHits(1);
2265
2266                if (fetch.size() > 0) {
2267                        return theOutput;
2268                } else {
2269                        return null;
2270                }
2271        }
2272
2273        private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts(String theSystem, Set<TermConcept> codes) {
2274                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size());
2275                for (TermConcept next : codes) {
2276                        retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode()));
2277                }
2278                return retVal;
2279        }
2280
2281        @Override
2282        @Transactional
2283        public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) {
2284                invokeRunnableForUnitTest();
2285
2286                IPrimitiveType<?> urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class);
2287                String url = urlPrimitive.getValueAsString();
2288                if (isNotBlank(url)) {
2289                        return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url);
2290                }
2291                return null;
2292        }
2293
2294        @CoverageIgnore
2295        @Override
2296        public IValidationSupport.CodeValidationResult validateCode(@Nonnull ValidationSupportContext theValidationSupportContext, @Nonnull ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl) {
2297                //TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS.
2298                invokeRunnableForUnitTest();
2299
2300                if (isNotBlank(theValueSetUrl)) {
2301                        return validateCodeInValueSet(theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystem, theCode, theDisplay);
2302                }
2303
2304                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2305                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
2306                Optional<FhirVersionIndependentConcept> codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new FhirVersionIndependentConcept(theCodeSystem, c.getCode())));
2307
2308                if (codeOpt != null && codeOpt.isPresent()) {
2309                        FhirVersionIndependentConcept code = codeOpt.get();
2310                        if (!theOptions.isValidateDisplay() || (isNotBlank(code.getDisplay()) && isNotBlank(theDisplay) && code.getDisplay().equals(theDisplay))) {
2311                                return new CodeValidationResult()
2312                                        .setCode(code.getCode())
2313                                        .setDisplay(code.getDisplay());
2314                        } else {
2315                                return createFailureCodeValidationResult(theCodeSystem, theCode, code.getSystemVersion(), " - Concept Display \"" + code.getDisplay() + "\" does not match expected \"" + code.getDisplay() + "\"").setDisplay(code.getDisplay());
2316                        }
2317                }
2318
2319                return createFailureCodeValidationResult(theCodeSystem, theCode, null, " - Code can not be found in CodeSystem");
2320        }
2321
2322        IValidationSupport.CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theValidationOptions, String theValueSetUrl, String theCodeSystem, String theCode, String theDisplay) {
2323                IBaseResource valueSet = theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl);
2324                CodeValidationResult retVal = null;
2325
2326                // If we don't have a PID, this came from some source other than the JPA
2327                // database, so we don't need to check if it's pre-expanded or not
2328                if (valueSet instanceof IAnyResource) {
2329                        Long pid = IDao.RESOURCE_PID.get((IAnyResource) valueSet);
2330                        if (pid != null) {
2331                                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
2332                                retVal = txTemplate.execute(tx -> {
2333                                        if (isValueSetPreExpandedForCodeValidation(valueSet)) {
2334                                                return validateCodeIsInPreExpandedValueSet(theValidationOptions, valueSet, theCodeSystem, theCode, theDisplay, null, null);
2335                                        } else {
2336                                                return null;
2337                                        }
2338                                });
2339                        }
2340                }
2341
2342                if (retVal == null) {
2343                        if (valueSet != null) {
2344                                retVal = new InMemoryTerminologyServerValidationSupport(myContext).validateCodeInValueSet(theValidationSupportContext, theValidationOptions, theCodeSystem, theCode, theDisplay, valueSet);
2345                        } else {
2346                                String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]";
2347                                retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append);
2348                        }
2349                }
2350
2351                // Check if someone is accidentally using a VS url where it should be a CS URL
2352                if (retVal != null && retVal.getCode() == null && theCodeSystem != null) {
2353                        if (isValueSetSupported(theValidationSupportContext, theCodeSystem)) {
2354                                if (!isCodeSystemSupported(theValidationSupportContext, theCodeSystem)) {
2355                                        String newMessage = "Unable to validate code " + theCodeSystem + "#" + theCode + " - Supplied system URL is a ValueSet URL and not a CodeSystem URL, check if it is correct: " + theCodeSystem;
2356                                        retVal.setMessage(newMessage);
2357                                }
2358                        }
2359                }
2360
2361                return retVal;
2362
2363        }
2364
2365        @Override
2366        public IBaseResource fetchCodeSystem(String theSystem) {
2367                IValidationSupport jpaValidationSupport = provideJpaValidationSupport();
2368                return jpaValidationSupport.fetchCodeSystem(theSystem);
2369        }
2370
2371        @Override
2372        public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) {
2373                IValidationSupport validationSupport = provideValidationSupport();
2374                IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem);
2375                if (codeSystem != null) {
2376                        codeSystem = toCanonicalCodeSystem(codeSystem);
2377                }
2378                return (CodeSystem) codeSystem;
2379        }
2380
2381        @Nonnull
2382        private IValidationSupport provideJpaValidationSupport() {
2383                IValidationSupport jpaValidationSupport = myJpaValidationSupport;
2384                if (jpaValidationSupport == null) {
2385                        jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class);
2386                        myJpaValidationSupport = jpaValidationSupport;
2387                }
2388                return jpaValidationSupport;
2389        }
2390
2391        @Nonnull
2392        protected IValidationSupport provideValidationSupport() {
2393                IValidationSupport validationSupport = myValidationSupport;
2394                if (validationSupport == null) {
2395                        validationSupport = myApplicationContext.getBean(IValidationSupport.class);
2396                        myValidationSupport = validationSupport;
2397                }
2398                return validationSupport;
2399        }
2400
2401        public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) {
2402                IValidationSupport validationSupport = provideValidationSupport();
2403                IBaseResource valueSet = validationSupport.fetchValueSet(theSystem);
2404                if (valueSet != null) {
2405                        valueSet = toCanonicalValueSet(valueSet);
2406                }
2407                return (ValueSet) valueSet;
2408        }
2409
2410        protected abstract CodeSystem toCanonicalCodeSystem(IBaseResource theCodeSystem);
2411
2412        @Override
2413        public IBaseResource fetchValueSet(String theValueSetUrl) {
2414                return provideJpaValidationSupport().fetchValueSet(theValueSetUrl);
2415        }
2416
2417        @Override
2418        public FhirContext getFhirContext() {
2419                return myContext;
2420        }
2421
2422        private void findCodesAbove(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2423                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
2424                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
2425                        addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate);
2426                }
2427        }
2428
2429        @Override
2430        public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) {
2431                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
2432                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
2433                if (system != null) {
2434                        findCodesAbove(system, theSystem, theCode, retVal);
2435                }
2436                return retVal;
2437        }
2438
2439        private void findCodesBelow(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2440                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
2441                findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList);
2442        }
2443
2444        private void findCodesBelow(String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate, List<CodeSystem.ConceptDefinitionComponent> conceptList) {
2445                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
2446                        if (theCode.equals(next.getCode())) {
2447                                addAllChildren(theSystemString, next, theListToPopulate);
2448                        } else {
2449                                findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept());
2450                        }
2451                }
2452        }
2453
2454        @Override
2455        public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) {
2456                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
2457                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
2458                if (system != null) {
2459                        findCodesBelow(system, theSystem, theCode, retVal);
2460                }
2461                return retVal;
2462        }
2463
2464        private void addAllChildren(String theSystemString, CodeSystem.ConceptDefinitionComponent theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2465                if (isNotBlank(theCode.getCode())) {
2466                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode()));
2467                }
2468                for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) {
2469                        addAllChildren(theSystemString, nextChild, theListToPopulate);
2470                }
2471        }
2472
2473        private boolean addTreeIfItContainsCode(String theSystemString, CodeSystem.ConceptDefinitionComponent theNext, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2474                boolean foundCodeInChild = false;
2475                for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) {
2476                        foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate);
2477                }
2478
2479                if (theCode.equals(theNext.getCode()) || foundCodeInChild) {
2480                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode()));
2481                        return true;
2482                }
2483
2484                return false;
2485        }
2486
2487        @Nullable
2488        protected abstract Coding toCanonicalCoding(@Nullable IBaseDatatype theCoding);
2489
2490        @Nullable
2491        protected abstract Coding toCanonicalCoding(@Nullable IBaseCoding theCoding);
2492
2493        @Nullable
2494        protected abstract CodeableConcept toCanonicalCodeableConcept(@Nullable IBaseDatatype theCodeableConcept);
2495
2496        @Nonnull
2497        private FhirVersionIndependentConcept toConcept(IPrimitiveType<String> theCodeType, IPrimitiveType<String> theCodeSystemIdentifierType, IBaseCoding theCodingType) {
2498                String code = theCodeType != null ? theCodeType.getValueAsString() : null;
2499                String system = theCodeSystemIdentifierType != null ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null;
2500                String systemVersion = theCodeSystemIdentifierType != null ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null;
2501                if (theCodingType != null) {
2502                        Coding canonicalizedCoding = toCanonicalCoding(theCodingType);
2503                        assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't
2504                        code = canonicalizedCoding.getCode();
2505                        system = canonicalizedCoding.getSystem();
2506                        systemVersion = canonicalizedCoding.getVersion();
2507                }
2508                return new FhirVersionIndependentConcept(system, code, null, systemVersion);
2509        }
2510
2511        @Override
2512        @Transactional
2513        public CodeValidationResult codeSystemValidateCode(IIdType theCodeSystemId, String theCodeSystemUrl, String theVersion, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) {
2514
2515                CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConcept);
2516                boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0;
2517
2518                Coding coding = toCanonicalCoding(theCoding);
2519                boolean haveCoding = coding != null && coding.isEmpty() == false;
2520
2521                boolean haveCode = theCode != null && theCode.isEmpty() == false;
2522
2523                if (!haveCodeableConcept && !haveCoding && !haveCode) {
2524                        throw new InvalidRequestException(Msg.code(906) + "No code, coding, or codeableConcept provided to validate.");
2525                }
2526                if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) {
2527                        throw new InvalidRequestException(Msg.code(907) + "$validate-code can only validate (code) OR (coding) OR (codeableConcept)");
2528                }
2529
2530                boolean haveIdentifierParam = isNotBlank(theCodeSystemUrl);
2531                String codeSystemUrl;
2532                if (theCodeSystemId != null) {
2533                        IBaseResource codeSystem = myDaoRegistry.getResourceDao("CodeSystem").read(theCodeSystemId);
2534                        codeSystemUrl = CommonCodeSystemsTerminologyService.getCodeSystemUrl(codeSystem);
2535                } else if (haveIdentifierParam) {
2536                        codeSystemUrl = theCodeSystemUrl;
2537                } else {
2538                        throw new InvalidRequestException(Msg.code(908) + "Either CodeSystem ID or CodeSystem identifier must be provided. Unable to validate.");
2539                }
2540
2541
2542                String code = theCode;
2543                String display = theDisplay;
2544
2545                if (haveCodeableConcept) {
2546                        for (int i = 0; i < codeableConcept.getCoding().size(); i++) {
2547                                Coding nextCoding = codeableConcept.getCoding().get(i);
2548                                if (nextCoding.hasSystem()) {
2549                                        if (!codeSystemUrl.equalsIgnoreCase(nextCoding.getSystem())) {
2550                                                throw new InvalidRequestException(Msg.code(909) + "Coding.system '" + nextCoding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate.");
2551                                        }
2552                                        codeSystemUrl = nextCoding.getSystem();
2553                                }
2554                                code = nextCoding.getCode();
2555                                display = nextCoding.getDisplay();
2556                                CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
2557                                if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
2558                                        return nextValidation;
2559                                }
2560                        }
2561                } else if (haveCoding) {
2562                        if (coding.hasSystem()) {
2563                                if (!codeSystemUrl.equalsIgnoreCase(coding.getSystem())) {
2564                                        throw new InvalidRequestException(Msg.code(910) + "Coding.system '" + coding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate.");
2565                                }
2566                                codeSystemUrl = coding.getSystem();
2567                        }
2568                        code = coding.getCode();
2569                        display = coding.getDisplay();
2570                }
2571
2572                return codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
2573        }
2574
2575        /**
2576         * When the search is for unversioned loinc system it uses the forcedId to obtain the current
2577         * version, as it is not necessarily the last  one anymore.
2578         * For other cases it keeps on considering the last uploaded as the current
2579         */
2580        @Override
2581        public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) {
2582                if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) {
2583                        Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl);
2584                        if (!vsIdOpt.isPresent()) {
2585                                return Optional.empty();
2586                        }
2587
2588                        return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get());
2589                }
2590
2591                List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl);
2592                if (termValueSetList.isEmpty()) {
2593                        return Optional.empty();
2594                }
2595
2596                return Optional.of(termValueSetList.get(0));
2597        }
2598
2599        @SuppressWarnings("unchecked")
2600        private CodeValidationResult codeSystemValidateCode(String theCodeSystemUrl, String theCodeSystemVersion, String theCode, String theDisplay) {
2601
2602                CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
2603                CriteriaQuery<TermConcept> query = criteriaBuilder.createQuery(TermConcept.class);
2604                Root<TermConcept> root = query.from(TermConcept.class);
2605
2606                Fetch<TermCodeSystemVersion, TermConcept> systemVersionFetch = root.fetch("myCodeSystem", JoinType.INNER);
2607                Join<TermCodeSystemVersion, TermConcept> systemVersionJoin = (Join<TermCodeSystemVersion, TermConcept>) systemVersionFetch;
2608                Fetch<TermCodeSystem, TermCodeSystemVersion> systemFetch = systemVersionFetch.fetch("myCodeSystem", JoinType.INNER);
2609                Join<TermCodeSystem, TermCodeSystemVersion> systemJoin = (Join<TermCodeSystem, TermCodeSystemVersion>) systemFetch;
2610
2611                ArrayList<Predicate> predicates = new ArrayList<>();
2612
2613                if (isNotBlank(theCode)) {
2614                        predicates.add(criteriaBuilder.equal(root.get("myCode"), theCode));
2615                }
2616
2617                if (isNoneBlank(theCodeSystemUrl)) {
2618                        predicates.add(criteriaBuilder.equal(systemJoin.get("myCodeSystemUri"), theCodeSystemUrl));
2619                }
2620
2621                // for loinc CodeSystem last version is not necessarily the current anymore, so if no version is present
2622                // we need to query for the current, which is that which version is null
2623                if (isNoneBlank(theCodeSystemVersion)) {
2624                        predicates.add(criteriaBuilder.equal(systemVersionJoin.get("myCodeSystemVersionId"), theCodeSystemVersion));
2625                } else {
2626                        if (theCodeSystemUrl.toLowerCase(Locale.ROOT).contains(LOINC_LOW)) {
2627                                predicates.add(criteriaBuilder.isNull(systemVersionJoin.get("myCodeSystemVersionId")));
2628                        } else {
2629                                query.orderBy(criteriaBuilder.desc(root.get("myUpdated")));
2630                        }
2631                }
2632
2633                Predicate outerPredicate = criteriaBuilder.and(predicates.toArray(new Predicate[0]));
2634                query.where(outerPredicate);
2635
2636                final TypedQuery<TermConcept> typedQuery = myEntityManager.createQuery(query.select(root));
2637                org.hibernate.query.Query<TermConcept> hibernateQuery = (org.hibernate.query.Query<TermConcept>) typedQuery;
2638                hibernateQuery.setFetchSize(SINGLE_FETCH_SIZE);
2639                List<TermConcept> resultsList = hibernateQuery.getResultList();
2640
2641                if (!resultsList.isEmpty()) {
2642                        TermConcept concept = resultsList.get(0);
2643
2644                        if (isNotBlank(theDisplay) && !theDisplay.equals(concept.getDisplay())) {
2645                                String message = "Concept Display \"" + theDisplay + "\" does not match expected \"" + concept.getDisplay() + "\" for CodeSystem: " + theCodeSystemUrl;
2646                                return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, message);
2647                        }
2648
2649                        return new CodeValidationResult().setCode(concept.getCode()).setDisplay(concept.getDisplay());
2650                }
2651
2652                return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, " - Code is not found in CodeSystem: " + theCodeSystemUrl);
2653        }
2654
2655        @Override
2656        public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) {
2657                @SuppressWarnings("unchecked")
2658                List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager.createQuery(
2659                        "select f.myResource from ForcedId f " +
2660                                "where f.myResourceType = 'CodeSystem' and f.myForcedId = '" + theForcedId + "'").getResultList();
2661                if (resultList.isEmpty()) return Optional.empty();
2662
2663                if (resultList.size() > 1)
2664                        throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: " + theForcedId + ". Was constraint "
2665                                + ForcedId.IDX_FORCEDID_TYPE_FID + " removed?");
2666
2667                IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem");
2668                IBaseResource cs = csDao.toResource(resultList.get(0), false);
2669                return Optional.of(cs);
2670        }
2671
2672
2673        private static final int SECONDS_IN_MINUTE = 60;
2674        private static final int INDEXED_ROOTS_LOGGING_COUNT = 50_000;
2675
2676
2677        @Transactional
2678        @Override
2679        public ReindexTerminologyResult reindexTerminology() throws InterruptedException {
2680                if (myFulltextSearchSvc == null) {
2681                        return ReindexTerminologyResult.SEARCH_SVC_DISABLED;
2682                }
2683
2684                if (isBatchTerminologyTasksRunning()) {
2685                        return ReindexTerminologyResult.OTHER_BATCH_TERMINOLOGY_TASKS_RUNNING;
2686                }
2687
2688                // disallow pre-expanding ValueSets while reindexing
2689                myDeferredStorageSvc.setProcessDeferred(false);
2690
2691                int objectLoadingThreadNumber = calculateObjectLoadingThreadNumber();
2692                ourLog.info("Using {} threads to load objects", objectLoadingThreadNumber);
2693
2694                try {
2695                        SearchSession searchSession = getSearchSession();
2696                        searchSession
2697                                .massIndexer( TermConcept.class )
2698                                .dropAndCreateSchemaOnStart( true )
2699                                .purgeAllOnStart( false )
2700                                .batchSizeToLoadObjects( 100 )
2701                                .cacheMode( CacheMode.IGNORE )
2702                                .threadsToLoadObjects( 6 )
2703                                .transactionTimeout( 60 * SECONDS_IN_MINUTE )
2704                                .monitor( new PojoMassIndexingLoggingMonitor(INDEXED_ROOTS_LOGGING_COUNT) )
2705                                .startAndWait();
2706                } finally {
2707                        myDeferredStorageSvc.setProcessDeferred(true);
2708                }
2709
2710                return ReindexTerminologyResult.SUCCESS;
2711        }
2712
2713
2714        @VisibleForTesting
2715        boolean isBatchTerminologyTasksRunning() {
2716                return isNotSafeToPreExpandValueSets() || isPreExpandingValueSets();
2717        }
2718
2719
2720        @VisibleForTesting
2721        int calculateObjectLoadingThreadNumber() {
2722                IConnectionPoolInfoProvider connectionPoolInfoProvider =
2723                        new ConnectionPoolInfoProvider(myHibernatePropertiesProvider.getDataSource());
2724                Optional<Integer> maxConnectionsOpt = connectionPoolInfoProvider.getTotalConnectionSize();
2725                if ( ! maxConnectionsOpt.isPresent() ) {
2726                        return DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS;
2727                }
2728
2729                int maxConnections = maxConnectionsOpt.get();
2730                int usableThreads = maxConnections < 6 ? 1 : maxConnections - 5;
2731                int objectThreads = Math.min(usableThreads, MAX_MASS_INDEXER_OBJECT_LOADING_THREADS);
2732                ourLog.debug("Data source connection pool has {} connections allocated, so reindexing will use {} object " +
2733                        "loading threads (each using a connection)", maxConnections, objectThreads);
2734                return objectThreads;
2735        }
2736
2737
2738        @VisibleForTesting
2739        SearchSession getSearchSession() {
2740                return Search.session( myEntityManager );
2741        }
2742
2743
2744        @VisibleForTesting
2745        public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) {
2746                ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest;
2747        }
2748
2749        static boolean isPlaceholder(DomainResource theResource) {
2750                boolean retVal = false;
2751                Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER);
2752                if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) {
2753                        retVal = ((BooleanType) extension.getValue()).booleanValue();
2754                }
2755                return retVal;
2756        }
2757
2758        /**
2759         * This is only used for unit tests to test failure conditions
2760         */
2761        static void invokeRunnableForUnitTest() {
2762                if (myInvokeOnNextCallForUnitTest != null) {
2763                        Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest;
2764                        myInvokeOnNextCallForUnitTest = null;
2765                        invokeOnNextCallForUnitTest.run();
2766                }
2767        }
2768
2769        @VisibleForTesting
2770        public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) {
2771                myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest;
2772        }
2773
2774        static List<TermConcept> toPersistedConcepts(List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) {
2775                ArrayList<TermConcept> retVal = new ArrayList<>();
2776
2777                for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
2778                        if (isNotBlank(next.getCode())) {
2779                                TermConcept termConcept = toTermConcept(next, theCodeSystemVersion);
2780                                retVal.add(termConcept);
2781                        }
2782                }
2783
2784                return retVal;
2785        }
2786
2787        @Nonnull
2788        static TermConcept toTermConcept(CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) {
2789                TermConcept termConcept = new TermConcept();
2790                termConcept.setCode(theConceptDefinition.getCode());
2791                termConcept.setCodeSystemVersion(theCodeSystemVersion);
2792                termConcept.setDisplay(theConceptDefinition.getDisplay());
2793                termConcept.addChildren(toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA);
2794
2795                for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent : theConceptDefinition.getDesignation()) {
2796                        if (isNotBlank(designationComponent.getValue())) {
2797                                TermConceptDesignation designation = termConcept.addDesignation();
2798                                designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null);
2799                                if (designationComponent.hasUse()) {
2800                                        designation.setUseSystem(designationComponent.getUse().hasSystem() ? designationComponent.getUse().getSystem() : null);
2801                                        designation.setUseCode(designationComponent.getUse().hasCode() ? designationComponent.getUse().getCode() : null);
2802                                        designation.setUseDisplay(designationComponent.getUse().hasDisplay() ? designationComponent.getUse().getDisplay() : null);
2803                                }
2804                                designation.setValue(designationComponent.getValue());
2805                        }
2806                }
2807
2808                for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) {
2809                        TermConceptProperty property = new TermConceptProperty();
2810
2811                        property.setKey(next.getCode());
2812                        property.setConcept(termConcept);
2813                        property.setCodeSystemVersion(theCodeSystemVersion);
2814
2815                        if (next.getValue() instanceof StringType) {
2816                                property.setType(TermConceptPropertyTypeEnum.STRING);
2817                                property.setValue(next.getValueStringType().getValue());
2818                        } else if (next.getValue() instanceof Coding) {
2819                                Coding nextCoding = next.getValueCoding();
2820                                property.setType(TermConceptPropertyTypeEnum.CODING);
2821                                property.setCodeSystem(nextCoding.getSystem());
2822                                property.setValue(nextCoding.getCode());
2823                                property.setDisplay(nextCoding.getDisplay());
2824                        } else if (next.getValue() != null) {
2825                                // TODO: LOINC has properties of type BOOLEAN that we should handle
2826                                ourLog.warn("Don't know how to handle properties of type: " + next.getValue().getClass());
2827                                continue;
2828                        }
2829
2830                        termConcept.getProperties().add(property);
2831                }
2832                return termConcept;
2833        }
2834
2835        static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) {
2836                // NOTE: return the designation when one of then is not specified.
2837                if (theReqLang == null || theStoredLang == null)
2838                        return true;
2839
2840                return theReqLang.equalsIgnoreCase(theStoredLang);
2841        }
2842
2843        public static class Job implements HapiJob {
2844                @Autowired
2845                private ITermReadSvc myTerminologySvc;
2846
2847                @Override
2848                public void execute(JobExecutionContext theContext) {
2849                        myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables();
2850                }
2851        }
2852
2853}