Source code

001package org.hl7.fhir.r5.terminologies.expansion;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import static org.apache.commons.lang3.StringUtils.isNotBlank;
035
036import java.io.FileNotFoundException;
037import java.io.IOException;
038import java.text.MessageFormat;
039
040/*
041 * Copyright (c) 2011+, HL7, Inc
042 * All rights reserved.
043 * 
044 * Redistribution and use in source and binary forms, with or without modification,
045 * are permitted provided that the following conditions are met:
046 * 
047 * Redistributions of source code must retain the above copyright notice, this
048 * list of conditions and the following disclaimer.
049 * Redistributions in binary form must reproduce the above copyright notice,
050 * this list of conditions and the following disclaimer in the documentation
051 * and/or other materials provided with the distribution.
052 * Neither the name of HL7 nor the names of its contributors may be used to
053 * endorse or promote products derived from this software without specific
054 * prior written permission.
055 * 
056 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
057 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
058 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
059 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
060 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
061 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
062 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
063 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
064 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
065 * POSSIBILITY OF SUCH DAMAGE.
066 * 
067 */
068
069import java.util.ArrayList;
070import java.util.Calendar;
071import java.util.Collection;
072import java.util.GregorianCalendar;
073import java.util.List;
074
075import org.hl7.fhir.exceptions.FHIRException;
076import org.hl7.fhir.exceptions.FHIRFormatError;
077import org.hl7.fhir.exceptions.NoTerminologyServiceException;
078import org.hl7.fhir.exceptions.TerminologyServiceException;
079import org.hl7.fhir.r5.context.IWorkerContext;
080import org.hl7.fhir.r5.elementmodel.LanguageUtils;
081import org.hl7.fhir.r5.extensions.ExtensionConstants;
082import org.hl7.fhir.r5.extensions.Extensions;
083import org.hl7.fhir.r5.extensions.ExtensionsUtils;
084import org.hl7.fhir.r5.model.BooleanType;
085import org.hl7.fhir.r5.model.CodeSystem;
086import org.hl7.fhir.r5.model.Enumerations.CodeSystemContentMode;
087import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
088import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
089import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
090import org.hl7.fhir.r5.model.CodeSystem.PropertyComponent;
091import org.hl7.fhir.r5.model.CodeType;
092import org.hl7.fhir.r5.model.Coding;
093import org.hl7.fhir.r5.model.DataType;
094import org.hl7.fhir.r5.model.DateTimeType;
095import org.hl7.fhir.r5.model.DecimalType;
096import org.hl7.fhir.r5.model.Enumerations.FilterOperator;
097import org.hl7.fhir.r5.model.Extension;
098import org.hl7.fhir.r5.model.Factory;
099import org.hl7.fhir.r5.model.IntegerType;
100import org.hl7.fhir.r5.model.PackageInformation;
101import org.hl7.fhir.r5.model.Parameters;
102import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
103import org.hl7.fhir.r5.model.PrimitiveType;
104import org.hl7.fhir.r5.model.Resource;
105import org.hl7.fhir.r5.model.StringType;
106import org.hl7.fhir.r5.model.CanonicalType;
107import org.hl7.fhir.r5.model.UriType;
108import org.hl7.fhir.r5.model.ValueSet;
109import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceComponent;
110import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceDesignationComponent;
111import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
112import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
113import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
114import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionComponent;
115import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
116import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionParameterComponent;
117import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionPropertyComponent;
118import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
119import org.hl7.fhir.r5.terminologies.ValueSetUtilities;
120import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpander.Token;
121import org.hl7.fhir.r5.terminologies.providers.CodeSystemProvider;
122import org.hl7.fhir.r5.terminologies.providers.CodeSystemProviderExtension;
123import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext;
124import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext.TerminologyServiceProtectionException;
125import org.hl7.fhir.r5.terminologies.utilities.TerminologyServiceErrorClass;
126import org.hl7.fhir.r5.terminologies.utilities.ValueSetProcessBase;
127import org.hl7.fhir.r5.utils.ToolingExtensions;
128import org.hl7.fhir.r5.utils.client.EFhirClientException;
129import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
130import org.hl7.fhir.utilities.Utilities;
131import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader;
132import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader.LanguagePreference;
133import org.hl7.fhir.utilities.i18n.I18nConstants;
134import org.hl7.fhir.utilities.validation.ValidationOptions;
135
136public class ValueSetExpander extends ValueSetProcessBase {
137
138
139  public class Token {
140    private String system;
141    private String code;
142    public Token(String system, String code) {
143      super();
144      this.system = system;
145      this.code = code;
146    }
147    public String getSystem() {
148      return system;
149    }
150    public String getCode() {
151      return code;
152    }
153    public boolean matches(Coding use) {
154      return system.equals(use.getSystem()) && code.equals(use.getCode());
155    }
156    public boolean matchesLang(String language) {
157      return system.equals("urn:ietf:bcp:47") && code.equals(language);
158    }
159  }
160
161  private static final boolean REPORT_VERSION_ANYWAY = true;
162  
163  private ValueSet focus;
164  private List<String> allErrors = new ArrayList<>();
165  private int maxExpansionSize = 1000;
166  private WorkingContext dwc = new WorkingContext();
167  
168  private boolean checkCodesWhenExpanding;
169  private boolean includeAbstract = true;
170  private boolean debug;
171
172  private AcceptLanguageHeader langs;
173  private List<Token> designations = new ArrayList<>();
174
175  public ValueSetExpander(IWorkerContext context, TerminologyOperationContext opContext) {
176    super(context, opContext);
177  }
178
179  public ValueSetExpander(IWorkerContext context, TerminologyOperationContext opContext, List<String> allErrors) {
180    super(context, opContext);
181    this.allErrors = allErrors;
182  }
183
184  public void setMaxExpansionSize(int theMaxExpansionSize) {
185    maxExpansionSize = theMaxExpansionSize;
186  }
187  
188  private ValueSetExpansionContainsComponent addCode(WorkingContext wc, String system, String code, String display, String dispLang, ValueSetExpansionContainsComponent parent, List<ConceptDefinitionDesignationComponent> designations, Parameters expParams, 
189      boolean isAbstract, boolean inactive, List<ValueSet> filters, boolean noInactive, boolean deprecated, List<ValueSetExpansionPropertyComponent> vsProp, 
190      List<ConceptPropertyComponent> csProps, CodeSystem cs, List<org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent> expProps, List<Extension> csExtList, List<Extension> vsExtList, ValueSetExpansionComponent exp) throws ETooCostly {
191    opContext.deadCheck("addCode"+code);
192    
193    if (filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code, exp))
194      return null;
195    if (noInactive && inactive) {
196      return null;
197    }
198    
199    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
200    n.setSystem(system);
201    n.setCode(code);
202    if (isAbstract) {
203      n.setAbstract(true);
204    }
205    if (inactive) {
206      n.setInactive(true);
207    }
208    if (deprecated) {
209      ValueSetUtilities.setDeprecated(vsProp, n);
210    }
211    if (ExtensionsUtils.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-label")) {
212      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#label", "label", ExtensionsUtils.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-label"));
213    }
214    if (ExtensionsUtils.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-label")) {
215      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#label", "label", ExtensionsUtils.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-label"));
216    }
217    if (ExtensionsUtils.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder")) {
218      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#order", "order", convertToDecimal(ExtensionsUtils.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder")));
219    }
220    if (ExtensionsUtils.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder")) {
221      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#order", "order", convertToDecimal(ExtensionsUtils.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder")));
222    }
223    if (ExtensionsUtils.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight")) {
224      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#itemWeight", "weight", ExtensionsUtils.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight"));
225    }
226    if (ExtensionsUtils.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight")) {
227      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#itemWeight", "weight", ExtensionsUtils.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight"));
228    }
229    ExtensionsUtils.copyExtensions(csExtList, n.getExtension(), 
230        "http://hl7.org/fhir/StructureDefinition/coding-sctdescid", 
231        "http://hl7.org/fhir/StructureDefinition/rendering-style", 
232        "http://hl7.org/fhir/StructureDefinition/rendering-xhtml",
233        "http://hl7.org/fhir/StructureDefinition/codesystem-alternate");
234    
235    ExtensionsUtils.copyExtensions(vsExtList, n.getExtension(), 
236        "http://hl7.org/fhir/StructureDefinition/valueset-supplement", 
237        "http://hl7.org/fhir/StructureDefinition/valueset-deprecated",
238        "http://hl7.org/fhir/StructureDefinition/valueset-concept-definition",
239        "http://hl7.org/fhir/StructureDefinition/coding-sctdescid", 
240        "http://hl7.org/fhir/StructureDefinition/rendering-style", 
241        "http://hl7.org/fhir/StructureDefinition/rendering-xhtml");
242    
243    // display and designations
244    ConceptDefinitionDesignationComponent pref = null;
245    if (langs == null) {
246      n.setDisplay(display);
247    } else {
248      if (designations == null) {
249        designations = new ArrayList<>();
250      }
251      designations.add(new ConceptDefinitionDesignationComponent().setLanguage(dispLang).setValue(display).setUse(new Coding().setSystem("http://terminology.hl7.org/CodeSystem/designation-usage").setCode("display")));
252      pref = findMatchingDesignation(designations);
253      if (pref != null) {
254        n.setDisplay(pref.getValue());
255      }
256    }
257
258    if (expParams.getParameterBool("includeDesignations") && designations != null) {
259      
260      for (ConceptDefinitionDesignationComponent t : designations) {
261        if (t != pref && (t.hasLanguage() || t.hasUse()) && t.getValue() != null && passesDesignationFilter(t)) {
262          ConceptReferenceDesignationComponent d = n.addDesignation();
263          if (t.getLanguage() != null) {
264            d.setLanguage(t.getLanguage().trim());
265          }
266          if (t.getValue() != null) {
267            d.setValue(t.getValue().trim());
268          }
269          if (t.getUse() != null) {
270            d.setUse(t.getUse());
271          }
272          for (Extension ext : t.getExtension()) {
273            if (Utilities.existsInList(ext.getUrl(), "http://hl7.org/fhir/StructureDefinition/coding-sctdescid")) {
274              d.addExtension(ext);
275            }
276          }
277        }
278      }
279    }
280    for (ParametersParameterComponent p : expParams.getParameter()) {
281      if ("property".equals(p.getName())) {
282        if (csProps != null && p.hasValue()) {
283          for (ConceptPropertyComponent cp : csProps) {
284            if (p.getValue().primitiveValue().equals(cp.getCode())) {
285              PropertyComponent pd = cs.getProperty(cp.getCode());
286              String url = pd == null ? null : pd.getUri();
287              if (url == null) {
288                if ("definition".equals(cp.getCode())) {
289                  url = "http://hl7.org/fhir/concept-properties#definition";
290                } else {
291                  // ??
292                }
293              }
294              ValueSetUtilities.addProperty(focus, n, url, cp.getCode(), cp.getValue()).copyExtensions(cp, "http://hl7.org/fhir/StructureDefinition/alternate-code-use", "http://hl7.org/fhir/StructureDefinition/alternate-code-status");
295            }
296          }
297        }
298        if (expProps != null && p.hasValue()) {
299          for (org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent cp : expProps) {
300            if (p.getValue().primitiveValue().equals(cp.getCode())) {
301              String url = null;
302              for (ValueSetExpansionPropertyComponent t : vsProp) {
303                if (t.hasCode() && t.getCode().equals(cp.getCode())) {
304                  url = t.getUri();
305                }
306              }
307              if (url == null) {
308                if ("definition".equals(cp.getCode())) {
309                  url = "http://hl7.org/fhir/concept-properties#definition";
310                } else {
311                  // TODO: try looking it up from the code system
312                }
313              }
314              ValueSetUtilities.addProperty(focus, n, url, cp.getCode(), cp.getValue()).copyExtensions(cp, "http://hl7.org/fhir/StructureDefinition/alternate-code-use", "http://hl7.org/fhir/StructureDefinition/alternate-code-status");
315            }
316          }
317        }        
318      }
319    }
320
321    String s = key(n);
322    if (wc.getExcludeKeys().contains(s)) {
323      return null;
324    } else if (wc.getMap().containsKey(s)) {
325      wc.setCanBeHierarchy(false);
326    } else {
327      wc.getCodes().add(n);
328      wc.getMap().put(s, n);
329//      if (wc == dwc && wc.getTotal() > maxExpansionSize) {
330//        if (wc.getOffset()+wc.getCount() > 0 && wc.getTotal() > wc.getOffset()+wc.getCount()) {
331//          wc.setTotal(-1);
332//          throw new EFinished();
333//        }
334//        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, focus.getUrl(), ">" + Integer.toString(maxExpansionSize)));
335//      }
336    }
337    if (wc.isCanBeHierarchy() && parent != null) {
338      parent.getContains().add(n);
339    } else if (!wc.getRootMap().containsKey(s)) {
340      wc.getRootMap().put(s, n);
341      wc.getRoots().add(n);
342    }
343    return n;
344  }
345
346  private DataType convertToDecimal(DataType v) {
347    if (v == null) {
348      return null;
349    } 
350    if (v instanceof DecimalType) {
351      return v;
352    } 
353    if (v instanceof IntegerType) {
354      return new DecimalType(((IntegerType) v).asStringValue());
355    }
356    return null;
357  }
358
359  private boolean passesDesignationFilter(ConceptDefinitionDesignationComponent d) {
360    if (designations.isEmpty()) {
361      return true;
362    }
363    for (Token t : designations) {
364      if (t.matches(d.getUse()) || t.matchesLang(d.getLanguage())) {
365        return true;
366      }
367      for (Coding c : d.getAdditionalUse()) {
368        if (t.matches(c)) {
369          return true;
370        }
371      }
372    }
373    return false;
374  }
375
376  private ConceptDefinitionDesignationComponent findMatchingDesignation(List<ConceptDefinitionDesignationComponent> designations) {
377    if (langs == null) {
378      return null;
379    }
380    // we have a list of languages in priority order 
381    // we have a list of designations in no order 
382    // language exact match is preferred
383    // display is always preferred
384    
385    for (LanguagePreference lang : langs.getLangs()) {
386      if (lang.getValue() > 0) {
387        for (ConceptDefinitionDesignationComponent cd : designations) {
388          if (isDisplay(cd) && LanguageUtils.langsMatchExact(cd.getLanguage(), lang.getLang())) {
389            return cd;
390          }
391        }
392        for (ConceptDefinitionDesignationComponent cd : designations) {
393          if (isDisplay(cd) && LanguageUtils.langsMatch(cd.getLanguage(), lang.getLang())) {
394            return cd;
395          }
396        }
397        for (ConceptDefinitionDesignationComponent cd : designations) {
398          if (LanguageUtils.langsMatchExact(cd.getLanguage(), lang.getLang())) {
399            return cd;
400          }
401        }
402        for (ConceptDefinitionDesignationComponent cd : designations) {
403          if (LanguageUtils.langsMatch(cd.getLanguage(), lang.getLang())) {
404            return cd;
405          }
406        }
407      }
408    }
409    return null;
410  }
411
412  private boolean isDisplay(ConceptDefinitionDesignationComponent cd) {
413    return cd.getUse().is("http://terminology.hl7.org/CodeSystem/designation-usage", "display");
414  }
415
416  private boolean filterContainsCode(List<ValueSet> filters, String system, String code, ValueSetExpansionComponent exp) {
417    for (ValueSet vse : filters) {
418      checkCanonical(exp, vse, focus);
419      if (expansionContainsCode(vse.getExpansion().getContains(), system, code))
420        return true;
421    }
422    return false;
423  }
424
425  private boolean expansionContainsCode(List<ValueSetExpansionContainsComponent> contains, String system, String code) {
426    for (ValueSetExpansionContainsComponent cc : contains) {
427      if (system.equals(cc.getSystem()) && code.equals(cc.getCode()))
428        return true;
429      if (expansionContainsCode(cc.getContains(), system, code))
430        return true;
431    }
432    return false;
433  }
434
435  private ConceptDefinitionDesignationComponent getMatchingLang(List<ConceptDefinitionDesignationComponent> list, AcceptLanguageHeader langs) {
436    for (ConceptDefinitionDesignationComponent t : list) {
437      if (LanguageUtils.langsMatchExact(langs, t.getLanguage())) {
438        return t;
439      }
440    }
441    for (ConceptDefinitionDesignationComponent t : list) {
442      if (LanguageUtils.langsMatch(langs, t.getLanguage())) {
443        return t;
444      }
445    }
446    return null;
447  }
448
449  private void addCodeAndDescendents(WorkingContext wc, ValueSetExpansionContainsComponent focus, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp)  throws FHIRException, ETooCostly {
450    opContext.deadCheck("addCodeAndDescendents");
451    focus.checkNoModifiers("Expansion.contains", "expanding");
452    ValueSetExpansionContainsComponent np = null;
453    for (String code : getCodesForConcept(focus, expParams)) {
454      ValueSetExpansionContainsComponent t = addCode(wc, focus.getSystem(), code, focus.getDisplay(), vsSrc.getLanguage(), parent, 
455           convert(focus.getDesignation()), expParams, focus.getAbstract(), focus.getInactive(), filters, noInactive, false, vsProps, makeCSProps(focus.getExtensionString(ToolingExtensions.EXT_DEFINITION), null), null, focus.getProperty(), null, focus.getExtension(), exp);
456      if (np == null) {
457        np = t;
458      }
459    }
460    for (ValueSetExpansionContainsComponent c : focus.getContains())
461      addCodeAndDescendents(wc, c, np, expParams, filters, noInactive, vsProps, vsSrc, exp);
462  }
463  
464  private List<ConceptPropertyComponent> makeCSProps(String definition, List<ConceptPropertyComponent> list) {
465    List<ConceptPropertyComponent> res = new ArrayList<>();
466    if (!Utilities.noString(definition)) {
467      res.add(new ConceptPropertyComponent("definition", new StringType(definition)));
468    }
469    if (list != null) {
470      res.addAll(list);
471    }
472    return res;
473  }
474
475  private List<String> getCodesForConcept(ValueSetExpansionContainsComponent focus, Parameters expParams) {
476    List<String> codes = new ArrayList<>();
477    codes.add(focus.getCode());
478    for (org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent p : focus.getProperty()) {
479      if ("alternateCode".equals(p.getCode()) && (altCodeParams.passes(p.getExtension())) && p.getValue().isPrimitive()) {
480        codes.add(p.getValue().primitiveValue());        
481      }
482    }
483    return codes;
484  }
485
486  private List<ConceptDefinitionDesignationComponent> convert(List<ConceptReferenceDesignationComponent> designations) {
487    List<ConceptDefinitionDesignationComponent> list = new ArrayList<ConceptDefinitionDesignationComponent>();
488    for (ConceptReferenceDesignationComponent d : designations) {
489      ConceptDefinitionDesignationComponent n = new ConceptDefinitionDesignationComponent();
490      n.setLanguage(d.getLanguage());
491      n.setUse(d.getUse());
492      n.setValue(d.getValue());
493      list.add(n);
494    }
495    return list;
496  }
497
498  private void addCodeAndDescendents(WorkingContext wc, CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, 
499        ConceptDefinitionComponent exclusion, ConceptFilter filterFunc, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, List<WorkingContext> otherFilters, ValueSetExpansionComponent exp)  throws FHIRException, ETooCostly {
500    opContext.deadCheck("addCodeAndDescendents");
501    def.checkNoModifiers("Code in Code System", "expanding");
502    if (exclusion != null) {
503      if (exclusion.getCode().equals(def.getCode()))
504        return; // excluded.
505    }
506    ValueSetExpansionContainsComponent np = null;
507    boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
508    boolean inc = CodeSystemUtilities.isInactive(cs, def);
509    boolean dep = CodeSystemUtilities.isDeprecated(cs, def, false);
510    if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def) && passesOtherFilters(otherFilters, cs, def.getCode())) {
511      for (String code : getCodesForConcept(def, expParams)) {
512        ValueSetExpansionContainsComponent t = addCode(wc, system, code, def.getDisplay(), cs.getLanguage(), parent, def.getDesignation(), expParams, abs, inc, filters, noInactive, dep, vsProps, makeCSProps(def.getDefinition(), def.getProperty()), cs, null, def.getExtension(), null, exp);
513        if (np == null) {
514          np = t;
515        }
516      }
517    }
518    for (ConceptDefinitionComponent c : def.getConcept()) {
519      addCodeAndDescendents(wc, cs, system, c, np, expParams, filters, exclusion, filterFunc, noInactive, vsProps, otherFilters, exp);
520    }
521    if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
522      List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
523      for (ConceptDefinitionComponent c : children)
524        addCodeAndDescendents(wc, cs, system, c, np, expParams, filters, exclusion, filterFunc, noInactive, vsProps, otherFilters, exp);
525    }
526  }
527
528
529  private void excludeCodeAndDescendents(WorkingContext wc, CodeSystem cs, String system, ConceptDefinitionComponent def, Parameters expParams, List<ValueSet> filters, 
530      ConceptDefinitionComponent exclusion, ConceptFilter filterFunc, List<WorkingContext> otherFilters, ValueSetExpansionComponent exp)  throws FHIRException, ETooCostly {
531    opContext.deadCheck("excludeCodeAndDescendents");
532    def.checkNoModifiers("Code in Code System", "expanding");
533    if (exclusion != null) {
534      if (exclusion.getCode().equals(def.getCode()))
535        return; // excluded.
536    }
537    boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
538    if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def) && passesOtherFilters(otherFilters, cs, def.getCode())) {
539      for (String code : getCodesForConcept(def, expParams)) {
540        if (!(filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code, exp)))
541          excludeCode(wc, system, code);
542      }
543    }
544    for (ConceptDefinitionComponent c : def.getConcept()) {
545      excludeCodeAndDescendents(wc, cs, system, c, expParams, filters, exclusion, filterFunc, otherFilters, exp);
546    }
547    if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
548      List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
549      for (ConceptDefinitionComponent c : children)
550        excludeCodeAndDescendents(wc, cs, system, c, expParams, filters, exclusion, filterFunc, otherFilters, exp);
551    }
552  }
553
554
555  private List<String> getCodesForConcept(ConceptDefinitionComponent focus, Parameters expParams) {
556    List<String> codes = new ArrayList<>();
557    codes.add(focus.getCode());
558    for (ConceptPropertyComponent p : focus.getProperty()) {
559      if ("alternateCode".equals(p.getCode()) && (altCodeParams.passes(p.getExtension())) && p.getValue().isPrimitive()) {
560        codes.add(p.getValue().primitiveValue());        
561      }
562    }
563    return codes;
564  }
565
566  private static boolean hasUse(ConceptPropertyComponent p, List<String> uses) {
567    for (Extension ext : p.getExtensionsByUrl(ToolingExtensions.EXT_CS_ALTERNATE_USE)) {
568      if (ext.hasValueCoding() && Utilities.existsInList(ext.getValueCoding().getCode(), uses)) {
569        return true;
570      }
571    }
572    return false;
573  }
574
575  
576
577  private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params, Parameters expParams, List<ValueSet> filters, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp) throws ETooCostly, FHIRException {
578    if (expand != null) {
579      if (expand.getContains().size() > maxExpansionSize)
580        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, vsSrc.getUrl(), ">" + Integer.toString(expand.getContains().size())));
581      for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
582        if (!existsInParams(params, p.getName(), p.getValue()))
583          params.add(p);
584      }
585      
586      copyImportContains(expand.getContains(), null, expParams, filters, noInactive, vsProps, vsSrc, exp);
587    }
588  }
589
590  private void excludeCode(WorkingContext wc, String theSystem, String theCode) {
591    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
592    n.setSystem(theSystem);
593    n.setCode(theCode);
594    String s = key(n);
595    wc.getExcludeKeys().add(s);
596  }
597
598  private void excludeCodes(WorkingContext wc, ConceptSetComponent exc, Parameters expParams, ValueSetExpansionComponent exp, ValueSet vs) throws FHIRException, FileNotFoundException, ETooCostly, IOException {
599    opContext.deadCheck("excludeCodes");
600    exc.checkNoModifiers("Compose.exclude", "expanding");
601    if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
602      wc.getExcludeSystems().add(exc.getSystem());
603    }
604
605    for (UriType imp : exc.getValueSet()) {
606      excludeCodes(wc, importValueSetForExclude(wc, imp.getValue(), exp, expParams, false, vs).getExpansion());
607    }
608    
609    if (exc.hasSystem()) {
610      CodeSystem cs = context.fetchSupplementedCodeSystem(exc.getSystem());
611      if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem(), opContext.getOptions().getFhirVersion())) {
612        ValueSetExpansionOutcome vse = context.expandVS(exc, false, false);
613        ValueSet valueset = vse.getValueset();
614        if (valueset == null)
615          throw failTSE("Error Expanding ValueSet: "+vse.getError());
616        excludeCodes(wc, valueset.getExpansion());
617        return;
618      }
619
620      for (ConceptReferenceComponent c : exc.getConcept()) {
621        excludeCode(wc, exc.getSystem(), c.getCode());
622      }
623
624      if (exc.getFilter().size() > 0) {
625        if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
626          addFragmentWarning(exp, cs);
627        }
628        List<WorkingContext> filters = new ArrayList<>();
629        for (int i = 1; i < exc.getFilter().size(); i++) {
630          WorkingContext wc1 = new WorkingContext();
631          filters.add(wc1);
632          processFilter(exc, exp, expParams, null, cs, false, exc.getFilter().get(i), wc1, null, true);
633        }
634        ConceptSetFilterComponent fc = exc.getFilter().get(0);
635        WorkingContext wc1 = dwc;
636        processFilter(exc, exp, expParams, null, cs, false, fc, wc1, filters, true);
637      }
638    }
639  }
640
641  private void excludeCodes(WorkingContext wc, ValueSetExpansionComponent expand) {
642    opContext.deadCheck("excludeCodes");
643    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
644      excludeCode(wc, c.getSystem(), c.getCode());
645    }
646  }
647
648  private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, DataType value) {
649    for (ValueSetExpansionParameterComponent p : params) {
650      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false)) {
651        return true;
652      }
653    }
654    return false;
655  }
656
657  public ValueSetExpansionOutcome expand(ValueSet source, Parameters expParams) {
658    
659    allErrors.clear();
660    try {
661      opContext.seeContext(source.getVersionedUrl());
662      
663      return expandInternal(source, expParams);
664    } catch (NoTerminologyServiceException e) {
665      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
666      // that might fail too, but it might not, later.
667      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.NOSERVICE, allErrors, false);
668    } catch (CodeSystemProviderExtension e) {
669      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
670      // that might fail too, but it might not, later.
671      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.INTERNAL_ERROR, allErrors, false);
672    } catch (TerminologyServiceProtectionException e) {
673      if (opContext.isOriginal()) {
674        return new ValueSetExpansionOutcome(e.getMessage(), e.getError(), allErrors, false);
675      } else {
676        throw e;
677      }
678    } catch (ETooCostly e) {
679      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.TOO_COSTLY, allErrors, false);
680    } catch (Exception e) {
681      if (debug) {
682        e.printStackTrace();
683      }
684      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.UNKNOWN, allErrors, e instanceof EFhirClientException || e instanceof TerminologyServiceException);
685    }
686  }
687  
688  public ValueSetExpansionOutcome expandInternal(ValueSet source, Parameters expParams) throws FHIRException, FileNotFoundException, ETooCostly, IOException, CodeSystemProviderExtension {
689      return doExpand(source, expParams);
690  }
691
692  private void processParameter(String name, DataType value) {
693    if (Utilities.existsInList(name, "includeDesignations", "excludeNested", "activeOnly", "offset", "count")) {
694      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
695      focus.getExpansion().addParameter().setName(name).setValue(value);
696    }
697    if ("displayLanguage".equals(name)) {
698      this.langs = new AcceptLanguageHeader(value.primitiveValue(), true);
699      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
700      focus.getExpansion().addParameter().setName(name).setValue(new CodeType(value.primitiveValue()));
701    }
702    if ("designation".equals(name)) {
703      String[] v = value.primitiveValue().split("\\|");
704      if (v.length != 2 || !Utilities.isAbsoluteUrl(v[0]) || Utilities.noString(v[1])) {
705        throw new NoTerminologyServiceException("Unable to understand designation parameter "+value.primitiveValue());
706      }
707      this.designations.add(new Token(v[0], v[1]));
708      focus.getExpansion().addParameter().setName(name).setValue(new StringType(value.primitiveValue()));
709    }
710    if ("offset".equals(name) && value instanceof IntegerType) {
711      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
712      focus.getExpansion().addParameter().setName(name).setValue(value);
713      dwc.setOffsetParam(((IntegerType) value).getValue());
714      if (dwc.getOffsetParam() < 0) {
715        dwc.setOffsetParam(0);
716      }
717    }
718    if ("count".equals(name)) {
719      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
720      focus.getExpansion().addParameter().setName(name).setValue(value);
721      dwc.setCountParam(((IntegerType) value).getValue());
722      if (dwc.getCountParam() < 0) {
723        dwc.setCountParam(0);
724      }
725    }
726  }
727  
728  public ValueSetExpansionOutcome doExpand(ValueSet source, Parameters expParams) throws FHIRException, ETooCostly, FileNotFoundException, IOException, CodeSystemProviderExtension {
729    if (expParams == null)
730      expParams = makeDefaultExpansion();
731    altCodeParams.seeParameters(expParams);
732    altCodeParams.seeValueSet(source);
733    source.checkNoModifiers("ValueSet", "expanding");
734    focus = source.copy();
735    focus.setIdBase(null);
736    focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
737    focus.getExpansion().setTimestampElement(DateTimeType.now());
738    focus.getExpansion().setIdentifier(Factory.createUUID()); 
739    checkCanonical(focus.getExpansion(), focus, focus);
740    for (Extension ext : focus.getCompose().getExtensionsByUrl("http://hl7.org/fhir/tools/StructureDefinion/valueset-expansion-param")) {
741      processParameter(ext.getExtensionString("name"), ext.getExtensionByUrl("value").getValue());
742    }
743    for (ParametersParameterComponent p : expParams.getParameter()) {
744      processParameter(p.getName(), p.getValue());
745    }
746    for (Extension s : focus.getExtensionsByUrl(ExtensionConstants.EXT_VSSUPPLEMENT)) {
747      requiredSupplements.add(s.getValue().primitiveValue());
748    }
749    if (langs == null && focus.hasLanguage()) {
750      langs = new AcceptLanguageHeader(focus.getLanguage(), true);
751    }
752
753    try {
754      if (source.hasCompose()) {
755//        ExtensionsUtils.stripExtensions(focus.getCompose()); - disabled 23/05/2023 GDG - why was this ever thought to be a good idea?
756        handleCompose(source.getCompose(), focus.getExpansion(), expParams, source.getUrl(), focus.getExpansion().getExtension(), source);
757      }
758    } catch (EFinished e) {
759      // nothing - we intended to trap this here
760    }
761
762    if (dwc.getCount() > maxExpansionSize && dwc.getOffsetParam() + dwc.getCountParam() == 0) {
763      if (dwc.isNoTotal()) {
764        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize)));        
765      } else {
766        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY_COUNT, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize), MessageFormat.format("{0,number,#}", dwc.getCount())));
767      }
768    } else if (dwc.isCanBeHierarchy() && ((dwc.getCountParam() == 0) || dwc.getCountParam() > dwc.getCodes().size())) {
769      for (ValueSetExpansionContainsComponent c : dwc.getRoots()) {
770        focus.getExpansion().getContains().add(c);
771      }
772    } else {
773      int i = 0;
774      int cc = 0;
775      for (ValueSetExpansionContainsComponent c : dwc.getCodes()) {
776        c.getContains().clear(); // make sure any hierarchy is wiped
777        if (dwc.getMap().containsKey(key(c)) && (includeAbstract || !c.getAbstract())) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
778          if (dwc.getOffsetParam() == 0 || i >= dwc.getOffsetParam()) {
779            focus.getExpansion().getContains().add(c);
780            cc++;
781            if (cc == dwc.getCountParam()) {
782              break;
783            }
784          }
785          i++;
786        }
787      }
788    }
789
790    if (dwc.hasOffsetParam()) {
791      focus.getExpansion().setOffset(dwc.getOffsetParam());
792    }
793    if (!dwc.isNoTotal()) {
794      focus.getExpansion().setTotal(dwc.getStatedTotal());
795    }
796    if (!requiredSupplements.isEmpty()) {      
797      return new ValueSetExpansionOutcome(context.formatMessagePlural(requiredSupplements.size(), I18nConstants.VALUESET_SUPPLEMENT_MISSING, CommaSeparatedStringBuilder.build(requiredSupplements)), TerminologyServiceErrorClass.BUSINESS_RULE, allErrors, false);
798    }
799    if (!expParams.hasParameter("includeDefinition") || !expParams.getParameterBool("includeDefinition")) {
800      focus.setCompose(null);
801      focus.getExtension().clear();
802      focus.setPublisher(null);
803      focus.setDescription(null);
804      focus.setPurpose(null);
805      focus.getContact().clear();
806      focus.setCopyright(null);
807      focus.setText(null);
808    }
809    return new ValueSetExpansionOutcome(focus);
810  }
811
812
813  private Parameters makeDefaultExpansion() {
814    Parameters res = new Parameters();
815    res.addParameter("excludeNested", true);
816    res.addParameter("includeDesignations", false);
817    return res;
818  }
819
820  private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
821    for (ConceptDefinitionComponent c : clist) {
822      if (code.equals(c.getCode()))
823        return c;
824      ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
825      if (v != null)
826        return v;
827    }
828    return null;
829  }
830
831  private void handleCompose(ValueSetComposeComponent compose, ValueSetExpansionComponent exp, Parameters expParams, String ctxt, List<Extension> extensions, ValueSet valueSet)
832      throws ETooCostly, FileNotFoundException, IOException, FHIRException, CodeSystemProviderExtension {
833    compose.checkNoModifiers("ValueSet.compose", "expanding");
834    // Exclude comes first because we build up a map of things to exclude
835    for (ConceptSetComponent inc : compose.getExclude()) {
836      excludeCodes(dwc, inc, expParams, exp, valueSet);
837    }
838    dwc.setCanBeHierarchy(!expParams.getParameterBool("excludeNested") && dwc.getExcludeKeys().isEmpty() && dwc.getExcludeSystems().isEmpty() && dwc.getOffsetParam() == 0);
839    includeAbstract = !expParams.getParameterBool("excludeNotForUI");
840    boolean first = true;
841    for (ConceptSetComponent inc : compose.getInclude()) {
842      if (first == true)
843        first = false;
844      else
845        dwc.setCanBeHierarchy(false);
846      includeCodes(inc, exp, expParams, dwc.isCanBeHierarchy(), compose.hasInactive() ? !compose.getInactive() : checkNoInActiveFromParam(expParams), extensions, valueSet);
847    }
848  }
849
850  /**
851   * returns true if activeOnly = true 
852   * @param expParams
853   * @return
854   */
855  private boolean checkNoInActiveFromParam(Parameters expParams) {
856    for (ParametersParameterComponent p : expParams.getParameter()) {
857      if (p.getName().equals("activeOnly")) {
858        return p.getValueBooleanType().getValue();
859      }
860    }
861    return false;
862  }
863
864  private ValueSet importValueSet(WorkingContext wc, String value, ValueSetExpansionComponent exp, Parameters expParams, boolean noInactive, ValueSet valueSet) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
865    if (value == null)
866      throw fail("unable to find value set with no identity");
867    ValueSet vs = context.findTxResource(ValueSet.class, value, valueSet);
868    if (vs == null) {
869      if (context.fetchResource(CodeSystem.class, value, valueSet) != null) {
870        throw fail("Cannot include value set "+value+" because it's actually a code system");
871      } else {
872        throw fail("Unable to find imported value set " + value);
873      }
874    }
875    checkCanonical(exp, vs, focus);
876    if (noInactive) {
877      expParams = expParams.copy();
878      expParams.addParameter("activeOnly", true);
879    }
880    ValueSetExpansionOutcome vso = new ValueSetExpander(context, opContext.copy(), allErrors).expand(vs, expParams);
881    if (vso.getError() != null) {
882      addErrors(vso.getAllErrors());
883      throw fail("Unable to expand imported value set "+vs.getUrl()+": " + vso.getError());
884    } else if (vso.getValueset() == null) {
885      throw fail("Unable to expand imported value set "+vs.getUrl()+" but no error");      
886    }
887    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
888      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
889      if (!existsInParams(exp.getParameter(), "used-valueset", u))
890        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
891    }
892    ValueSetExpansionComponent evs = vso.getValueset().getExpansion();
893    for (Extension ex : evs.getExtension()) {
894      if (ex.getUrl().equals(ToolingExtensions.EXT_EXP_TOOCOSTLY)) {
895        if (ex.getValue() instanceof BooleanType) {
896          exp.getExtension().add(new Extension(ToolingExtensions.EXT_EXP_TOOCOSTLY).setValue(new CanonicalType(value)));
897        } else {
898          exp.getExtension().add(ex);
899        }
900      } 
901    }
902    if (!evs.hasTotal()) {
903      // because if there's no total, we can't know if we got everything
904      dwc.setNoTotal(true);
905    }
906    for (ValueSetExpansionParameterComponent p : evs.getParameter()) {
907      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue()))
908        exp.getParameter().add(p);
909    }
910    if (isValueSetUnionImports(valueSet)) {
911      copyExpansion(wc, evs.getContains());
912    }
913    wc.setCanBeHierarchy(false); // if we're importing a value set, we have to be combining, so we won't try for a hierarchy
914    return vso.getValueset();
915  }
916  
917
918  private ValueSet importValueSetForExclude(WorkingContext wc, String value, ValueSetExpansionComponent exp, Parameters expParams, boolean noInactive, ValueSet valueSet) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
919    if (value == null)
920      throw fail("unable to find value set with no identity");
921    ValueSet vs = context.findTxResource(ValueSet.class, value, valueSet);
922    if (vs == null) {
923      if (context.fetchResource(CodeSystem.class, value, valueSet) != null) {
924        throw fail("Cannot include value set "+value+" because it's actually a code system");
925      } else {
926        throw fail("Unable to find imported value set " + value);
927      }
928    }
929    checkCanonical(exp, vs, focus);
930    if (noInactive) {
931      expParams = expParams.copy();
932      expParams.addParameter("activeOnly", true);
933    }
934    ValueSetExpansionOutcome vso = new ValueSetExpander(context, opContext.copy(), allErrors).expand(vs, expParams);
935    if (vso.getError() != null) {
936      addErrors(vso.getAllErrors());
937      throw fail("Unable to expand imported value set "+vs.getUrl()+": " + vso.getError());
938    }
939    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
940      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
941      if (!existsInParams(exp.getParameter(), "used-valueset", u))
942        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
943    }
944    for (Extension ex : vso.getValueset().getExpansion().getExtension()) {
945      if (ex.getUrl().equals(ToolingExtensions.EXT_EXP_TOOCOSTLY)) {
946        throw fail("Unable to expand imported value set "+vs.getUrl()+" for exclude: too costly");
947      } 
948    }
949    return vso.getValueset();
950  }
951  
952  protected boolean isValueSetUnionImports(ValueSet valueSet) {
953    PackageInformation p = valueSet.getSourcePackage();
954    if (p != null) {
955      return p.getDate().before(new GregorianCalendar(2022, Calendar.MARCH, 31).getTime());
956    } else {
957      return false;
958    }
959  }
960
961  public void copyExpansion(WorkingContext wc,List<ValueSetExpansionContainsComponent> list) {
962    opContext.deadCheck("copyExpansion");
963    for (ValueSetExpansionContainsComponent cc : list) {
964       ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
965       n.setSystem(cc.getSystem());
966       n.setCode(cc.getCode());
967       n.setAbstract(cc.getAbstract());
968       n.setInactive(cc.getInactive());
969       n.setDisplay(cc.getDisplay());
970       n.getDesignation().addAll(cc.getDesignation());
971
972       String s = key(n);
973       if (!wc.getMap().containsKey(s) && !wc.getExcludeKeys().contains(s)) {
974         wc.getCodes().add(n);
975         wc.getMap().put(s, n);
976       }
977       copyExpansion(wc, cc.getContains());
978    }
979  }
980
981  private void addErrors(List<String> errs) {
982    for (String s : errs) {
983      if (!allErrors.contains(s)) {
984        allErrors.add(s);
985      }
986    }
987  }
988
989  private int copyImportContains(List<ValueSetExpansionContainsComponent> list, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filter, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp) throws FHIRException, ETooCostly {
990    int count = 0;
991    opContext.deadCheck("copyImportContains");
992    for (ValueSetExpansionContainsComponent c : list) {
993      c.checkNoModifiers("Imported Expansion in Code System", "expanding");
994      ValueSetExpansionContainsComponent np = addCode(dwc, c.getSystem(), c.getCode(), c.getDisplay(), vsSrc.getLanguage(), parent, null, expParams, c.getAbstract(), c.getInactive(), 
995          filter, noInactive, false, vsProps, makeCSProps(c.getExtensionString(ToolingExtensions.EXT_DEFINITION), null), null, c.getProperty(), null, c.getExtension(), exp);
996      if (np != null) {
997        count++;
998      }
999      count = count + copyImportContains(c.getContains(), np, expParams, filter, noInactive, vsProps, vsSrc, exp);
1000    }
1001    return count;
1002  }
1003
1004  private void includeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, boolean heirarchical, boolean noInactive, List<Extension> extensions, ValueSet valueSet) throws ETooCostly, FileNotFoundException, IOException, FHIRException, CodeSystemProviderExtension {
1005    opContext.deadCheck("includeCodes");
1006    inc.checkNoModifiers("Compose.include", "expanding");
1007    List<ValueSet> imports = new ArrayList<ValueSet>();
1008    for (CanonicalType imp : inc.getValueSet()) {
1009      imports.add(importValueSet(dwc, imp.getValue(), exp, expParams, noInactive, valueSet));
1010    }
1011
1012    if (!inc.hasSystem()) {
1013      if (imports.isEmpty()) // though this is not supposed to be the case
1014        return;
1015      ValueSet base = imports.get(0);
1016      checkCanonical(exp, base, focus);
1017      imports.remove(0);
1018      base.checkNoModifiers("Imported ValueSet", "expanding");
1019      copyImportContains(base.getExpansion().getContains(), null, expParams, imports, noInactive, base.getExpansion().getProperty(), base, exp);
1020    } else {
1021      CodeSystem cs = context.fetchSupplementedCodeSystem(inc.getSystem());
1022      if (ValueSetUtilities.isServerSide(inc.getSystem()) || (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT))) {
1023        doServerIncludeCodes(inc, heirarchical, exp, imports, expParams, extensions, noInactive, valueSet.getExpansion().getProperty());
1024      } else {
1025        if (cs.hasUserData("supplements.installed")) {
1026          for (String s : cs.getUserString("supplements.installed").split("\\,")) {
1027            requiredSupplements.remove(s);
1028          }
1029        }
1030        doInternalIncludeCodes(inc, exp, expParams, imports, cs, noInactive, valueSet);
1031      }
1032    }
1033  }
1034
1035  private void doServerIncludeCodes(ConceptSetComponent inc, boolean heirarchical, ValueSetExpansionComponent exp, List<ValueSet> imports, Parameters expParams, List<Extension> extensions, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps) throws FHIRException, CodeSystemProviderExtension, ETooCostly {
1036    opContext.deadCheck("doServerIncludeCodes");
1037    CodeSystemProvider csp = CodeSystemProvider.factory(inc.getSystem());
1038    if (csp != null) {
1039      csp.includeCodes(inc, heirarchical, exp, imports, expParams, extensions, noInactive, vsProps);
1040      return;
1041    }
1042    
1043    ValueSetExpansionOutcome vso = context.expandVS(inc, heirarchical, noInactive);
1044    if (vso.getError() != null) {
1045      throw failTSE("Unable to expand imported value set: " + vso.getError());
1046    }
1047    ValueSet vs = vso.getValueset();
1048    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
1049      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
1050      if (!existsInParams(exp.getParameter(), "used-valueset", u)) {
1051        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
1052      }
1053    }
1054    if (vs.getExpansion().hasTotal()) {
1055      // 0 for now... dwc.incExtraCount(!vs.getExpansion().getTotal());
1056    } else {
1057      dwc.setNoTotal(true);
1058    }
1059    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
1060      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue())) {
1061        exp.getParameter().add(p);
1062      }
1063    }
1064    for (Extension ex : vs.getExpansion().getExtension()) {
1065      if (Utilities.existsInList(ex.getUrl(), ToolingExtensions.EXT_EXP_TOOCOSTLY, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed")) {
1066        if (!ExtensionsUtils.hasExtension(extensions, ex.getUrl())) {
1067          extensions.add(ex);
1068        }
1069      }
1070    }
1071    for (ValueSetExpansionContainsComponent cc : vs.getExpansion().getContains()) {
1072      addCodeAndDescendents(dwc, cc, null, expParams, imports, noInactive, vsProps, vs, exp);
1073    }
1074  }
1075
1076
1077  public void doInternalIncludeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, List<ValueSet> imports, CodeSystem cs, boolean noInactive, Resource vsSrc) throws NoTerminologyServiceException, TerminologyServiceException, FHIRException, ETooCostly {
1078    opContext.deadCheck("doInternalIncludeCodes");
1079    if (cs == null) {
1080      if (context.isNoTerminologyServer())
1081        throw failTSE("Unable to find code system " + inc.getSystem().toString());
1082      else
1083        throw failTSE("Unable to find code system " + inc.getSystem().toString());
1084    }
1085    checkCanonical(exp, cs, focus);
1086    cs.checkNoModifiers("Code System", "expanding");
1087    if (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT)
1088      throw failTSE("Code system " + inc.getSystem().toString() + " is incomplete");
1089    if (cs.hasVersion() || REPORT_VERSION_ANYWAY) {
1090      UriType u = new UriType(cs.getUrl() + (cs.hasVersion() ? "|"+cs.getVersion() : ""));
1091      if (!existsInParams(exp.getParameter(), "used-codesystem", u))
1092        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-codesystem").setValue(u));
1093      if (cs.hasUserData("supplements.installed")) {
1094        for (String s : cs.getUserString("supplements.installed").split("\\,")) {
1095          u = new UriType(s);
1096          if (!existsInParams(exp.getParameter(), "used-supplement", u)) {
1097            exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-supplement").setValue(u));
1098          }
1099        }
1100      }
1101    }
1102    if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
1103      // special case - add all the code system
1104      for (ConceptDefinitionComponent def : cs.getConcept()) {
1105        addCodeAndDescendents(dwc, cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), null, exp);
1106      }
1107      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1108        addFragmentWarning(exp, cs);
1109      }
1110      if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
1111        addExampleWarning(exp, cs);
1112      }      
1113    }
1114
1115    if (!inc.getConcept().isEmpty()) {
1116      dwc.setCanBeHierarchy(false);
1117      for (ConceptReferenceComponent c : inc.getConcept()) {
1118        c.checkNoModifiers("Code in Value Set", "expanding");
1119        ConceptDefinitionComponent def = CodeSystemUtilities.findCodeOrAltCode(cs.getConcept(), c.getCode(), null);
1120        boolean inactive = false; // default is true if we're a fragment and  
1121        boolean isAbstract = false;
1122        if (def == null) {
1123          if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1124            addFragmentWarning(exp, cs);
1125          } else if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
1126              addExampleWarning(exp, cs);
1127          } else {
1128            if (checkCodesWhenExpanding) {
1129              throw failTSE("Unable to find code '" + c.getCode() + "' in code system " + cs.getUrl());
1130            }
1131          }
1132        } else {
1133          def.checkNoModifiers("Code in Code System", "expanding");
1134          inactive = CodeSystemUtilities.isInactive(cs, def);
1135          isAbstract = CodeSystemUtilities.isNotSelectable(cs, def);
1136          addCode(dwc, inc.getSystem(), c.getCode(), !Utilities.noString(c.getDisplay()) ? c.getDisplay() : def.getDisplay(), c.hasDisplay() ? vsSrc.getLanguage() : cs.getLanguage(), null, mergeDesignations(def, convertDesignations(c.getDesignation())), 
1137              expParams, isAbstract, inactive, imports, noInactive, false, exp.getProperty(), makeCSProps(def.getDefinition(), def.getProperty()), cs, null, def.getExtension(), c.getExtension(), exp);
1138        }
1139      }
1140    }
1141    if (inc.getFilter().size() > 0) {
1142      if (inc.getFilter().size() > 1) {
1143        dwc.setCanBeHierarchy(false); // which will be the case if we get around to supporting this
1144      }
1145      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1146        addFragmentWarning(exp, cs);
1147      }
1148      List<WorkingContext> filters = new ArrayList<>();
1149      for (int i = 1; i < inc.getFilter().size(); i++) {
1150        WorkingContext wc = new WorkingContext();
1151        filters.add(wc);
1152        processFilter(inc, exp, expParams, imports, cs, noInactive, inc.getFilter().get(i), wc, null, false);
1153      }
1154      ConceptSetFilterComponent fc = inc.getFilter().get(0);
1155      WorkingContext wc = dwc;
1156      processFilter(inc, exp, expParams, imports, cs, noInactive, fc, wc, filters, false);
1157    }
1158  }
1159
1160  private void processFilter(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, List<ValueSet> imports, CodeSystem cs, boolean noInactive, 
1161      ConceptSetFilterComponent fc, WorkingContext wc, List<WorkingContext> filters, boolean exclude)
1162      throws ETooCostly {
1163    opContext.deadCheck("processFilter");
1164    if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
1165      // special: all codes in the target code system under the value
1166      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1167      if (def == null)
1168        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1169      if (exclude) {
1170        excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1171      } else {
1172        addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1173      }
1174    } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISNOTA) {
1175      // special: all codes in the target code system that are not under the value
1176      ConceptDefinitionComponent defEx = getConceptForCode(cs.getConcept(), fc.getValue());
1177      if (defEx == null)
1178        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1179      for (ConceptDefinitionComponent def : cs.getConcept()) {
1180        if (exclude) {
1181          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, defEx, new AllConceptsFilter(allErrors), filters, exp);
1182        } else {
1183          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, defEx, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1184        }
1185      }
1186    } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
1187      // special: all codes in the target code system under the value
1188      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1189      if (def == null)
1190        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1191      for (ConceptDefinitionComponent c : def.getConcept())
1192        if (exclude) {
1193          excludeCodeAndDescendents(wc, cs, inc.getSystem(), c, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1194        } else {
1195          addCodeAndDescendents(wc, cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1196        }
1197      if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
1198        List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
1199        for (ConceptDefinitionComponent c : children) {
1200          if (exclude) {
1201            excludeCodeAndDescendents(wc, cs, inc.getSystem(), c, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1202          } else {
1203            addCodeAndDescendents(wc, cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1204          }
1205        }
1206      }
1207
1208    } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
1209      // gg; note: wtf is this: if the filter is display=v, look up the code 'v', and see if it's display is 'v'?
1210      dwc.setCanBeHierarchy(false);
1211      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1212      if (def != null) {
1213        if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
1214          if (def.getDisplay().contains(fc.getValue()) && passesOtherFilters(filters, cs, def.getCode())) {
1215            for (String code : getCodesForConcept(def, expParams)) {
1216              opContext.deadCheck("processFilter2");
1217              if (exclude) {
1218                excludeCode(wc, inc.getSystem(), code);
1219              } else {
1220                addCode(wc, inc.getSystem(), code, def.getDisplay(), cs.getLanguage(), null, def.getDesignation(), expParams, CodeSystemUtilities.isNotSelectable(cs, def), CodeSystemUtilities.isInactive(cs, def),
1221                  imports, noInactive, false, exp.getProperty(), makeCSProps(def.getDefinition(), def.getProperty()), cs, null, def.getExtension(), null, exp);
1222              }
1223            }
1224          }
1225        }
1226      }
1227    } else if (CodeSystemUtilities.isDefinedProperty(cs, fc.getProperty())) {
1228      for (ConceptDefinitionComponent def : cs.getConcept()) {
1229        PropertyFilter pf = new PropertyFilter(allErrors, fc, CodeSystemUtilities.getPropertyDefinition(cs, fc.getProperty()));
1230        if (exclude) {
1231          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, pf, filters, exp);
1232        } else {
1233          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, pf, noInactive, exp.getProperty(), filters, exp);
1234        }
1235      }
1236    } else if (isKnownProperty(fc.getProperty(), cs)) {
1237      for (ConceptDefinitionComponent def : cs.getConcept()) {
1238        KnownPropertyFilter pf = new KnownPropertyFilter(allErrors, fc, fc.getProperty());
1239        if (exclude) {
1240          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, pf, filters, exp);
1241        } else {
1242          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, pf, noInactive, exp.getProperty(), filters, exp);
1243        }
1244      }
1245    } else if ("code".equals(fc.getProperty()) && fc.getOp() == FilterOperator.REGEX) {
1246      for (ConceptDefinitionComponent def : cs.getConcept()) {
1247        if (exclude) {
1248          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, new RegexFilter(allErrors, fc.getValue()), filters, exp);
1249        } else {
1250          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, new RegexFilter(allErrors, fc.getValue()), noInactive, exp.getProperty(), filters, exp);
1251        }
1252      }
1253    } else {
1254      throw fail("Filter by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
1255    }
1256  }
1257
1258  private boolean isKnownProperty(String property, CodeSystem cs) {
1259    return Utilities.existsInList(property, "notSelectable");
1260  }
1261
1262  private List<ConceptDefinitionDesignationComponent> mergeDesignations(ConceptDefinitionComponent def,
1263      List<ConceptDefinitionDesignationComponent> list) {
1264    List<ConceptDefinitionDesignationComponent> res = new ArrayList<>();
1265    if (def != null) {
1266      res.addAll(def.getDesignation());
1267    }
1268    res.addAll(list);
1269    return res;
1270  }
1271
1272 
1273
1274  private void addFragmentWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
1275    String url = cs.getVersionedUrl();
1276    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
1277      if ("fragment".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
1278        return;
1279      }     
1280    }
1281    exp.addParameter().setName("fragment").setValue(new CanonicalType(url));
1282  }
1283
1284  private void addExampleWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
1285    String url = cs.getVersionedUrl();
1286    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
1287      if ("example".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
1288        return;
1289      }     
1290    }
1291    exp.addParameter().setName("example").setValue(new CanonicalType(url));
1292  }
1293  
1294  private List<ConceptDefinitionDesignationComponent> convertDesignations(List<ConceptReferenceDesignationComponent> list) {
1295    List<ConceptDefinitionDesignationComponent> res = new ArrayList<CodeSystem.ConceptDefinitionDesignationComponent>();
1296    for (ConceptReferenceDesignationComponent t : list) {
1297      ConceptDefinitionDesignationComponent c = new ConceptDefinitionDesignationComponent();
1298      c.setLanguage(t.getLanguage());
1299      c.setUse(t.getUse());
1300      c.setValue(t.getValue());
1301      c.getExtension().addAll(t.getExtension());
1302      res.add(c);
1303    }
1304    return res;
1305  }
1306
1307  private String key(String uri, String code) {
1308    return "{" + uri + "}" + code;
1309  }
1310
1311  private String key(ValueSetExpansionContainsComponent c) {
1312    return key(c.getSystem(), c.getCode());
1313  }
1314
1315  private FHIRException fail(String msg) {
1316    allErrors.add(msg);
1317    return new FHIRException(msg);
1318  }
1319
1320  private ETooCostly failCostly(String msg) {
1321    allErrors.add(msg);
1322    return new ETooCostly(msg);
1323  }
1324
1325  private TerminologyServiceException failTSE(String msg) {
1326    allErrors.add(msg);
1327    return new TerminologyServiceException(msg);
1328  }
1329
1330  public Collection<? extends String> getAllErrors() {
1331    return allErrors;
1332  }
1333
1334  public boolean isCheckCodesWhenExpanding() {
1335    return checkCodesWhenExpanding;
1336  }
1337
1338  public void setCheckCodesWhenExpanding(boolean checkCodesWhenExpanding) {
1339    this.checkCodesWhenExpanding = checkCodesWhenExpanding;
1340  }
1341
1342  private boolean passesOtherFilters(List<WorkingContext> otherFilters, CodeSystem cs, String code) {
1343    if (otherFilters == null) {
1344      return true;
1345    }
1346    String key = key(cs.getUrl(), code);
1347    for (WorkingContext wc : otherFilters) {
1348      if (!wc.getMap().containsKey(key)) {
1349        return false;
1350      }
1351    }
1352    return true;
1353  }
1354
1355  public boolean isDebug() {
1356    return debug;
1357  }
1358
1359  public ValueSetExpander setDebug(boolean debug) {
1360    this.debug = debug;
1361    return this;
1362  }
1363  
1364  
1365}