001package org.hl7.fhir.r5.terminologies.expansion;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import static org.apache.commons.lang3.StringUtils.isNotBlank;
035
036import java.io.FileNotFoundException;
037import java.io.IOException;
038import java.text.MessageFormat;
039
040/*
041 * Copyright (c) 2011+, HL7, Inc
042 * All rights reserved.
043 * 
044 * Redistribution and use in source and binary forms, with or without modification,
045 * are permitted provided that the following conditions are met:
046 * 
047 * Redistributions of source code must retain the above copyright notice, this
048 * list of conditions and the following disclaimer.
049 * Redistributions in binary form must reproduce the above copyright notice,
050 * this list of conditions and the following disclaimer in the documentation
051 * and/or other materials provided with the distribution.
052 * Neither the name of HL7 nor the names of its contributors may be used to
053 * endorse or promote products derived from this software without specific
054 * prior written permission.
055 * 
056 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
057 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
058 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
059 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
060 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
061 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
062 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
063 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
064 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
065 * POSSIBILITY OF SUCH DAMAGE.
066 * 
067 */
068
069import java.util.ArrayList;
070import java.util.Calendar;
071import java.util.Collection;
072import java.util.GregorianCalendar;
073import java.util.HashSet;
074import java.util.List;
075import java.util.Set;
076
077import org.hl7.fhir.exceptions.FHIRException;
078import org.hl7.fhir.exceptions.FHIRFormatError;
079import org.hl7.fhir.exceptions.NoTerminologyServiceException;
080import org.hl7.fhir.exceptions.TerminologyServiceException;
081import org.hl7.fhir.r5.context.IWorkerContext;
082import org.hl7.fhir.r5.elementmodel.LanguageUtils;
083import org.hl7.fhir.r5.extensions.ExtensionDefinitions;
084import org.hl7.fhir.r5.extensions.ExtensionDefinitions;
085import org.hl7.fhir.r5.extensions.ExtensionUtilities;
086import org.hl7.fhir.r5.model.BooleanType;
087import org.hl7.fhir.r5.model.CanonicalType;
088import org.hl7.fhir.r5.model.CodeSystem;
089import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
090import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
091import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
092import org.hl7.fhir.r5.model.CodeSystem.PropertyComponent;
093import org.hl7.fhir.r5.model.CodeType;
094import org.hl7.fhir.r5.model.Coding;
095import org.hl7.fhir.r5.model.DataType;
096import org.hl7.fhir.r5.model.DateTimeType;
097import org.hl7.fhir.r5.model.DecimalType;
098import org.hl7.fhir.r5.model.Enumerations.CodeSystemContentMode;
099import org.hl7.fhir.r5.model.Enumerations.FilterOperator;
100import org.hl7.fhir.r5.model.OperationOutcome.IssueType;
101import org.hl7.fhir.r5.model.OperationOutcome.OperationOutcomeIssueComponent;
102import org.hl7.fhir.r5.model.Extension;
103import org.hl7.fhir.r5.model.Factory;
104import org.hl7.fhir.r5.model.IntegerType;
105import org.hl7.fhir.r5.model.PackageInformation;
106import org.hl7.fhir.r5.model.Parameters;
107import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
108import org.hl7.fhir.r5.model.PrimitiveType;
109import org.hl7.fhir.r5.model.Resource;
110import org.hl7.fhir.r5.model.StringType;
111import org.hl7.fhir.r5.model.UriType;
112import org.hl7.fhir.r5.model.ValueSet;
113import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceComponent;
114import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceDesignationComponent;
115import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
116import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
117import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
118import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionComponent;
119import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
120import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionParameterComponent;
121import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionPropertyComponent;
122import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
123import org.hl7.fhir.r5.terminologies.ValueSetUtilities;
124import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpander.UnknownValueSetException;
125import org.hl7.fhir.r5.terminologies.providers.CodeSystemProvider;
126import org.hl7.fhir.r5.terminologies.providers.CodeSystemProviderExtension;
127import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext;
128import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext.TerminologyServiceProtectionException;
129import org.hl7.fhir.r5.terminologies.utilities.ValueSetProcessBase.OpIssueCode;
130import org.hl7.fhir.r5.terminologies.utilities.ValueSetProcessBase.TerminologyOperationDetails;
131import org.hl7.fhir.r5.terminologies.validation.VSCheckerException;
132import org.hl7.fhir.r5.terminologies.utilities.TerminologyServiceErrorClass;
133import org.hl7.fhir.r5.terminologies.utilities.ValueSetProcessBase;
134
135import org.hl7.fhir.r5.utils.UserDataNames;
136import org.hl7.fhir.r5.utils.client.EFhirClientException;
137import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
138import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
139import org.hl7.fhir.utilities.Utilities;
140import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader;
141import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader.LanguagePreference;
142import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
143import org.hl7.fhir.utilities.i18n.I18nConstants;
144
145@MarkedToMoveToAdjunctPackage
146public class ValueSetExpander extends ValueSetProcessBase {
147
148
149  public class UnknownValueSetException extends FHIRException {
150
151    protected UnknownValueSetException() {
152      super();
153    }
154
155    protected UnknownValueSetException(String message, Throwable cause) {
156      super(message, cause);
157    }
158    
159    protected UnknownValueSetException(String message) {
160      super(message);
161    }
162
163    protected UnknownValueSetException(Throwable cause) {
164      super(cause);
165    }
166
167  }
168
169  public class Token {
170    private String system;
171    private String code;
172    public Token(String system, String code) {
173      super();
174      this.system = system;
175      this.code = code;
176    }
177    public String getSystem() {
178      return system;
179    }
180    public String getCode() {
181      return code;
182    }
183    public boolean matches(Coding use) {
184      return system.equals(use.getSystem()) && code.equals(use.getCode());
185    }
186    public boolean matchesLang(String language) {
187      return system.equals("urn:ietf:bcp:47") && code.equals(language);
188    }
189  }
190
191  private static final boolean REPORT_VERSION_ANYWAY = true;
192
193  private static final String VS_EXP_IMPORT_ERROR_TOO_COSTLY = null;
194  
195  private ValueSet focus;
196  private List<String> allErrors = new ArrayList<>();
197  private int maxExpansionSize = 1000;
198  private WorkingContext dwc = new WorkingContext();
199  
200  private boolean checkCodesWhenExpanding;
201  private boolean includeAbstract = true;
202  private boolean debug;
203  private Set<String> sources = new HashSet<>();
204
205  private AcceptLanguageHeader langs;
206  private List<Token> designations = new ArrayList<>();
207
208  public ValueSetExpander(IWorkerContext context, TerminologyOperationContext opContext) {
209    super(context, opContext);
210  }
211
212  public ValueSetExpander(IWorkerContext context, TerminologyOperationContext opContext, List<String> allErrors) {
213    super(context, opContext);
214    this.allErrors = allErrors;
215  }
216
217  public void setMaxExpansionSize(int theMaxExpansionSize) {
218    maxExpansionSize = theMaxExpansionSize;
219  }
220  
221  private ValueSetExpansionContainsComponent addCode(WorkingContext wc, String system, String code, String display, String dispLang, ValueSetExpansionContainsComponent parent, List<ConceptDefinitionDesignationComponent> designations, Parameters expParams, 
222      boolean isAbstract, boolean inactive, List<ValueSet> filters, boolean noInactive, boolean deprecated, List<ValueSetExpansionPropertyComponent> vsProp, 
223      List<ConceptPropertyComponent> csProps, CodeSystem cs, List<org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent> expProps,
224      List<Extension> csExtList, List<Extension> vsExtList, ValueSetExpansionComponent exp, String vstatus) throws ETooCostly {
225    opContext.deadCheck("addCode"+code);
226    
227    if (filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code, exp))
228      return null;
229    if (noInactive && inactive) {
230      return null;
231    }
232    
233    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
234    n.setSystem(system);
235    n.setCode(code);
236    if (isAbstract) {
237      n.setAbstract(true);
238    }
239    if (inactive) {
240      n.setInactive(true);
241      ValueSetUtilities.addCodeProperty(focus, n, "http://hl7.org/fhir/concept-properties#status", "status", vstatus);
242    }
243    if (deprecated) {
244      ValueSetUtilities.setDeprecated(vsProp, n);
245    }
246    if (ExtensionUtilities.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-label")) {
247      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#label", "label", ExtensionUtilities.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-label"));
248    }
249    if (ExtensionUtilities.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-label")) {
250      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#label", "label", ExtensionUtilities.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-label"));
251    }
252    if (ExtensionUtilities.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder")) {
253      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#order", "order", convertToDecimal(ExtensionUtilities.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/codesystem-conceptOrder")));
254    }
255    if (ExtensionUtilities.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder")) {
256      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#order", "order", convertToDecimal(ExtensionUtilities.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/valueset-conceptOrder")));
257    }
258    if (ExtensionUtilities.hasExtension(csExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight")) {
259      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#itemWeight", "weight", ExtensionUtilities.getExtensionValue(csExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight"));
260    }
261    if (ExtensionUtilities.hasExtension(vsExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight")) {
262      ValueSetUtilities.addProperty(focus, n, "http://hl7.org/fhir/concept-properties#itemWeight", "weight", ExtensionUtilities.getExtensionValue(vsExtList, "http://hl7.org/fhir/StructureDefinition/itemWeight"));
263    }
264    ExtensionUtilities.copyExtensions(csExtList, n.getExtension(),
265        "http://hl7.org/fhir/StructureDefinition/coding-sctdescid", 
266        "http://hl7.org/fhir/StructureDefinition/rendering-style", 
267        "http://hl7.org/fhir/StructureDefinition/rendering-xhtml",
268        "http://hl7.org/fhir/StructureDefinition/codesystem-alternate");
269
270    ExtensionUtilities.copyExtensions(vsExtList, n.getExtension(),
271        "http://hl7.org/fhir/StructureDefinition/valueset-supplement", 
272        "http://hl7.org/fhir/StructureDefinition/valueset-deprecated",
273        "http://hl7.org/fhir/StructureDefinition/valueset-concept-definition",
274        "http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status",
275        "http://hl7.org/fhir/StructureDefinition/coding-sctdescid", 
276        "http://hl7.org/fhir/StructureDefinition/rendering-style", 
277        "http://hl7.org/fhir/StructureDefinition/rendering-xhtml");
278    
279    // display and designations
280    ConceptDefinitionDesignationComponent pref = null;
281    if (langs == null) {
282      n.setDisplay(display);
283    } else {
284      if (designations == null) {
285        designations = new ArrayList<>();
286      }
287      designations.add(new ConceptDefinitionDesignationComponent().setLanguage(dispLang).setValue(display).setUse(new Coding().setSystem("http://terminology.hl7.org/CodeSystem/hl7TermMaintInfra").setCode("preferredForLanguage")));
288      pref = findMatchingDesignation(designations);
289      if (pref != null) {
290        n.setDisplay(pref.getValue());
291      }
292    }
293    if (!n.hasDisplay() && display != null && langs != null && (langs.matches(dispLang) || Utilities.existsInList(langs.getSource(), "en", "en-US"))) {
294      n.setDisplay(display);      
295    }
296
297    if (expParams.getParameterBool("includeDesignations") && designations != null) {
298      
299      for (ConceptDefinitionDesignationComponent t : designations) {
300        if (t != pref && (t.hasLanguage() || t.hasUse()) && t.getValue() != null && passesDesignationFilter(t)) {
301          ConceptReferenceDesignationComponent d = n.addDesignation();
302          if (t.getLanguage() != null) {
303            d.setLanguage(t.getLanguage().trim());
304          }
305          if (t.getValue() != null) {
306            d.setValue(t.getValue().trim());
307          }
308          if (t.getUse() != null) {
309            d.setUse(t.getUse());
310          }
311          for (Extension ext : t.getExtension()) {
312            if (Utilities.existsInList(ext.getUrl(), "http://hl7.org/fhir/StructureDefinition/coding-sctdescid", "http://hl7.org/fhir/StructureDefinition/structuredefinition-standards-status")) {
313              d.addExtension(ext);
314            }
315          }
316        }
317      }
318    }
319    for (ParametersParameterComponent p : expParams.getParameter()) {
320      if ("property".equals(p.getName())) {
321        if (csProps != null && p.hasValue()) {
322          for (ConceptPropertyComponent cp : csProps) {
323            if (p.getValue().primitiveValue().equals(cp.getCode())) {
324              PropertyComponent pd = cs.getProperty(cp.getCode());
325              String url = pd == null ? null : pd.getUri();
326              if (url == null) {
327                if ("definition".equals(cp.getCode())) {
328                  url = "http://hl7.org/fhir/concept-properties#definition";
329                } else {
330                  // ??
331                }
332              }
333              ValueSetUtilities.addProperty(focus, n, url, cp.getCode(), cp.getValue()).copyExtensions(cp, "http://hl7.org/fhir/StructureDefinition/alternate-code-use", "http://hl7.org/fhir/StructureDefinition/alternate-code-status");
334            }
335          }
336        }
337        if (expProps != null && p.hasValue()) {
338          for (org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent cp : expProps) {
339            if (p.getValue().primitiveValue().equals(cp.getCode())) {
340              String url = null;
341              for (ValueSetExpansionPropertyComponent t : vsProp) {
342                if (t.hasCode() && t.getCode().equals(cp.getCode())) {
343                  url = t.getUri();
344                }
345              }
346              if (url == null) {
347                if ("definition".equals(cp.getCode())) {
348                  url = "http://hl7.org/fhir/concept-properties#definition";
349                } else {
350                  // TODO: try looking it up from the code system
351                }
352              }
353              ValueSetUtilities.addProperty(focus, n, url, cp.getCode(), cp.getValue()).copyExtensions(cp, "http://hl7.org/fhir/StructureDefinition/alternate-code-use", "http://hl7.org/fhir/StructureDefinition/alternate-code-status");
354            }
355          }
356        }        
357      }
358    }
359
360    String s = key(n);
361    if (wc.getExcludeKeys().contains(s)) {
362      return null;
363    } else if (wc.getMap().containsKey(s)) {
364      wc.setCanBeHierarchy(false);
365    } else {
366      wc.getCodes().add(n);
367      wc.getMap().put(s, n);
368//      if (wc == dwc && wc.getTotal() > maxExpansionSize) {
369//        if (wc.getOffset()+wc.getCount() > 0 && wc.getTotal() > wc.getOffset()+wc.getCount()) {
370//          wc.setTotal(-1);
371//          throw new EFinished();
372//        }
373//        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, focus.getUrl(), ">" + Integer.toString(maxExpansionSize)));
374//      }
375    }
376    if (wc.isCanBeHierarchy() && parent != null) {
377      parent.getContains().add(n);
378    } else if (!wc.getRootMap().containsKey(s)) {
379      wc.getRootMap().put(s, n);
380      wc.getRoots().add(n);
381    }
382    return n;
383  }
384
385  private String findParamValue(List<ValueSetExpansionParameterComponent> list, String name) {
386    for (ValueSetExpansionParameterComponent p : list) {
387      if (name.equals(p.getName())) {
388        return p.getValue().primitiveValue();
389      }
390    }
391    return null;
392  }
393
394  private DataType convertToDecimal(DataType v) {
395    if (v == null) {
396      return null;
397    } 
398    if (v instanceof DecimalType) {
399      return v;
400    } 
401    if (v instanceof IntegerType) {
402      return new DecimalType(((IntegerType) v).asStringValue());
403    }
404    return null;
405  }
406
407  private boolean passesDesignationFilter(ConceptDefinitionDesignationComponent d) {
408    if (designations.isEmpty()) {
409      return true;
410    }
411    for (Token t : designations) {
412      if ((d.hasUse() && t.matches(d.getUse())) || t.matchesLang(d.getLanguage())) {
413        return true;
414      }
415      for (Coding c : d.getAdditionalUse()) {
416        if (t.matches(c)) {
417          return true;
418        }
419      }
420    }
421    return false;
422  }
423
424  private ConceptDefinitionDesignationComponent findMatchingDesignation(List<ConceptDefinitionDesignationComponent> designations) {
425    if (langs == null) {
426      return null;
427    }
428    // we have a list of languages in priority order 
429    // we have a list of designations in no order 
430    // language exact match is preferred
431    // display is always preferred
432    
433    for (LanguagePreference lang : langs.getLangs()) {
434      if (lang.getValue() > 0) {
435        for (ConceptDefinitionDesignationComponent cd : designations) {
436          if (isDisplay(cd, false) && LanguageUtils.langsMatchExact(cd.getLanguage(), lang.getLang())) {
437            return cd;
438          }
439        }
440        for (ConceptDefinitionDesignationComponent cd : designations) {
441          if (isDisplay(cd, false) && LanguageUtils.langsMatch(lang.getLang(), cd.getLanguage())) {
442            return cd;
443          }
444        }
445        for (ConceptDefinitionDesignationComponent cd : designations) {
446          if (isDisplay(cd, false) && LanguageUtils.langsMatch(cd.getLanguage(), lang.getLang())) {
447            return cd;
448          }
449        }
450        for (ConceptDefinitionDesignationComponent cd : designations) {
451          if (isDisplay(cd, true) && LanguageUtils.langsMatchExact(cd.getLanguage(), lang.getLang())) {
452            return cd;
453          }
454        }
455        for (ConceptDefinitionDesignationComponent cd : designations) {
456          if (isDisplay(cd, true) && LanguageUtils.langsMatch(cd.getLanguage(), lang.getLang())) {
457            return cd;
458          }
459        }
460        for (ConceptDefinitionDesignationComponent cd : designations) {
461          if (isDisplay(cd, true) && LanguageUtils.langsMatch(lang.getLang(), cd.getLanguage())) {
462            return cd;
463          }
464        }
465        for (ConceptDefinitionDesignationComponent cd : designations) {
466          if (LanguageUtils.langsMatchExact(cd.getLanguage(), lang.getLang())) {
467            return cd;
468          }
469        }
470        for (ConceptDefinitionDesignationComponent cd : designations) {
471          if (LanguageUtils.langsMatch(cd.getLanguage(), lang.getLang())) {
472            return cd;
473          }
474        }
475        for (ConceptDefinitionDesignationComponent cd : designations) {
476          if (LanguageUtils.langsMatch(lang.getLang(), cd.getLanguage())) {
477            return cd;
478          }
479        }
480      }
481    }
482    return null;
483  }
484
485  private boolean isDisplay(ConceptDefinitionDesignationComponent cd, boolean def) {
486    return (def && !cd.hasUse())
487      || (cd.hasUse() && cd.getUse().is("http://terminology.hl7.org/CodeSystem/hl7TermMaintInfra", "preferredForLanguage"))
488      || (cd.hasUse() && cd.getUse().is("http://terminology.hl7.org/CodeSystem/designation-usage", "display"));
489  }
490
491  private boolean filterContainsCode(List<ValueSet> filters, String system, String code, ValueSetExpansionComponent exp) {
492    for (ValueSet vse : filters) {
493      checkCanonical(exp, vse, focus);
494      if (expansionContainsCode(vse.getExpansion().getContains(), system, code))
495        return true;
496    }
497    return false;
498  }
499
500  private boolean expansionContainsCode(List<ValueSetExpansionContainsComponent> contains, String system, String code) {
501    for (ValueSetExpansionContainsComponent cc : contains) {
502      if (system.equals(cc.getSystem()) && code.equals(cc.getCode()))
503        return true;
504      if (expansionContainsCode(cc.getContains(), system, code))
505        return true;
506    }
507    return false;
508  }
509
510  private ConceptDefinitionDesignationComponent getMatchingLang(List<ConceptDefinitionDesignationComponent> list, AcceptLanguageHeader langs) {
511    for (ConceptDefinitionDesignationComponent t : list) {
512      if (LanguageUtils.langsMatchExact(langs, t.getLanguage())) {
513        return t;
514      }
515    }
516    for (ConceptDefinitionDesignationComponent t : list) {
517      if (LanguageUtils.langsMatch(langs, t.getLanguage())) {
518        return t;
519      }
520    }
521    return null;
522  }
523
524  private void addCodeAndDescendents(WorkingContext wc, ValueSetExpansionContainsComponent focus, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp, String langDisplay)  throws FHIRException, ETooCostly {
525    opContext.deadCheck("addCodeAndDescendents");
526    focus.checkNoModifiers("Expansion.contains", "expanding");
527    ValueSetExpansionContainsComponent np = null;
528    for (String code : getCodesForConcept(focus, expParams)) {
529      String vstatus = determineStatus(vsSrc, focus);
530      ValueSetExpansionContainsComponent t = addCode(wc, focus.getSystem(), code, focus.getDisplay(), langDisplay, parent, 
531           convert(focus.getDesignation()), expParams, focus.getAbstract(), focus.getInactive(), filters, noInactive, false,
532        vsProps, makeCSProps(focus.getExtensionString(ExtensionDefinitions.EXT_DEFINITION), null), null, focus.getProperty(), null, focus.getExtension(), exp, vstatus);
533      if (np == null) {
534        np = t;
535      }
536    }
537    for (ValueSetExpansionContainsComponent c : focus.getContains())
538      addCodeAndDescendents(wc, c, np, expParams, filters, noInactive, vsProps, vsSrc, exp, langDisplay);
539  }
540
541  private String determineStatus(ValueSet vs, ValueSetExpansionContainsComponent focus) {
542    String vstatus = ValueSetUtilities.getProperty(vs.getExpansion().getProperty(), focus, "status", "http://hl7.org/fhir/concept-properties#status");
543    if (vstatus == null) {
544      vstatus = focus.getExtensionString(ExtensionDefinitions.EXT_STANDARDS_STATUS);
545    }
546    if (Utilities.existsInList(vstatus, "retired")) {
547      return null;
548    }
549    return vstatus;
550  }
551
552  private String getLang(ValueSet vsSrc) {
553    if (vsSrc.hasLanguage()) {
554      return vsSrc.getLanguage();
555    }
556    for (ValueSetExpansionParameterComponent p : vsSrc.getExpansion().getParameter()) {
557      if ("displayLanguage".equals(p.getName())) {
558        return p.getValue().primitiveValue();
559      }
560    }
561    return null;
562  }
563
564  private List<ConceptPropertyComponent> makeCSProps(String definition, List<ConceptPropertyComponent> list) {
565    List<ConceptPropertyComponent> res = new ArrayList<>();
566    if (!Utilities.noString(definition)) {
567      res.add(new ConceptPropertyComponent("definition", new StringType(definition)));
568    }
569    if (list != null) {
570      res.addAll(list);
571    }
572    return res;
573  }
574
575  private List<String> getCodesForConcept(ValueSetExpansionContainsComponent focus, Parameters expParams) {
576    List<String> codes = new ArrayList<>();
577    codes.add(focus.getCode());
578    for (org.hl7.fhir.r5.model.ValueSet.ConceptPropertyComponent p : focus.getProperty()) {
579      if ("alternateCode".equals(p.getCode()) && (altCodeParams.passes(p.getExtension())) && p.getValue().isPrimitive()) {
580        codes.add(p.getValue().primitiveValue());        
581      }
582    }
583    return codes;
584  }
585
586  private List<ConceptDefinitionDesignationComponent> convert(List<ConceptReferenceDesignationComponent> designations) {
587    List<ConceptDefinitionDesignationComponent> list = new ArrayList<ConceptDefinitionDesignationComponent>();
588    for (ConceptReferenceDesignationComponent d : designations) {
589      ConceptDefinitionDesignationComponent n = new ConceptDefinitionDesignationComponent();
590      n.setLanguage(d.getLanguage());
591      if (d.hasUse()) {
592        n.setUse(d.getUse());
593      }
594      n.setValue(d.getValue());
595      list.add(n);
596    }
597    return list;
598  }
599
600  private void addCodeAndDescendents(WorkingContext wc, CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filters, 
601        ConceptDefinitionComponent exclusion, ConceptFilter filterFunc, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, List<WorkingContext> otherFilters, ValueSetExpansionComponent exp)  throws FHIRException, ETooCostly {
602    opContext.deadCheck("addCodeAndDescendents");
603    def.checkNoModifiers("Code in Code System", "expanding");
604    if (exclusion != null) {
605      if (exclusion.getCode().equals(def.getCode()))
606        return; // excluded.
607    }
608    ValueSetExpansionContainsComponent np = null;
609    boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
610    boolean inc = CodeSystemUtilities.isInactive(cs, def);
611    boolean dep = CodeSystemUtilities.isDeprecated(cs, def, false);
612    String vstatus = determineStatus(cs, def);
613    if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def) && passesOtherFilters(otherFilters, cs, def.getCode())) {
614      for (String code : getCodesForConcept(def, expParams)) {
615        ValueSetExpansionContainsComponent t = addCode(wc, system, code, def.getDisplay(), cs.getLanguage(), parent, def.getDesignation(), expParams,
616          abs, inc, filters, noInactive, dep, vsProps, makeCSProps(def.getDefinition(), def.getProperty()), cs,
617          null, def.getExtension(), null, exp, vstatus) ;
618        if (np == null) {
619          np = t;
620        }
621      }
622    }
623    for (ConceptDefinitionComponent c : def.getConcept()) {
624      addCodeAndDescendents(wc, cs, system, c, np, expParams, filters, exclusion, filterFunc, noInactive, vsProps, otherFilters, exp);
625    }
626    if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
627      List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
628      for (ConceptDefinitionComponent c : children)
629        addCodeAndDescendents(wc, cs, system, c, np, expParams, filters, exclusion, filterFunc, noInactive, vsProps, otherFilters, exp);
630    }
631  }
632
633  private String determineStatus(CodeSystem cs, ConceptDefinitionComponent def) {
634    String vstatus = CodeSystemUtilities.getStatus(cs, def);
635    if (vstatus == null) {
636      vstatus = def.getExtensionString(ExtensionDefinitions.EXT_STANDARDS_STATUS);
637    }
638    if (Utilities.existsInList(vstatus, "retired")) {
639      return null;
640    }
641    return vstatus;
642  }
643
644
645  private void excludeCodeAndDescendents(WorkingContext wc, CodeSystem cs, String system, ConceptDefinitionComponent def, Parameters expParams, List<ValueSet> filters, 
646      ConceptDefinitionComponent exclusion, ConceptFilter filterFunc, List<WorkingContext> otherFilters, ValueSetExpansionComponent exp)  throws FHIRException, ETooCostly {
647    opContext.deadCheck("excludeCodeAndDescendents");
648    def.checkNoModifiers("Code in Code System", "expanding");
649    if (exclusion != null) {
650      if (exclusion.getCode().equals(def.getCode()))
651        return; // excluded.
652    }
653    boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
654    if ((includeAbstract || !abs)  && filterFunc.includeConcept(cs, def) && passesOtherFilters(otherFilters, cs, def.getCode())) {
655      for (String code : getCodesForConcept(def, expParams)) {
656        if (!(filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code, exp)))
657          excludeCode(wc, system, code);
658      }
659    }
660    for (ConceptDefinitionComponent c : def.getConcept()) {
661      excludeCodeAndDescendents(wc, cs, system, c, expParams, filters, exclusion, filterFunc, otherFilters, exp);
662    }
663    if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
664      List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
665      for (ConceptDefinitionComponent c : children)
666        excludeCodeAndDescendents(wc, cs, system, c, expParams, filters, exclusion, filterFunc, otherFilters, exp);
667    }
668  }
669
670
671  private List<String> getCodesForConcept(ConceptDefinitionComponent focus, Parameters expParams) {
672    List<String> codes = new ArrayList<>();
673    codes.add(focus.getCode());
674    for (ConceptPropertyComponent p : focus.getProperty()) {
675      if ("alternateCode".equals(p.getCode()) && (altCodeParams.passes(p.getExtension())) && p.getValue().isPrimitive()) {
676        codes.add(p.getValue().primitiveValue());        
677      }
678    }
679    return codes;
680  }
681
682  private static boolean hasUse(ConceptPropertyComponent p, List<String> uses) {
683    for (Extension ext : p.getExtensionsByUrl(ExtensionDefinitions.EXT_CS_ALTERNATE_USE)) {
684      if (ext.hasValueCoding() && Utilities.existsInList(ext.getValueCoding().getCode(), uses)) {
685        return true;
686      }
687    }
688    return false;
689  }
690
691  
692
693  private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params, Parameters expParams, List<ValueSet> filters, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp) throws ETooCostly, FHIRException {
694    if (expand != null) {
695      if (expand.getContains().size() > maxExpansionSize)
696        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, vsSrc.getUrl(), ">" + Integer.toString(expand.getContains().size())));
697      for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
698        if (!existsInParams(params, p.getName(), p.getValue()))
699          params.add(p);
700      }
701      
702      copyImportContains(expand.getContains(), null, expParams, filters, noInactive, vsProps, vsSrc, exp);
703    }
704  }
705
706  private void excludeCode(WorkingContext wc, String theSystem, String theCode) {
707    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
708    n.setSystem(theSystem);
709    n.setCode(theCode);
710    String s = key(n);
711    wc.getExcludeKeys().add(s);
712  }
713
714  private void excludeCodes(WorkingContext wc, ConceptSetComponent exc, Parameters expParams, ValueSetExpansionComponent exp, ValueSet vs, String vspath) throws FHIRException, FileNotFoundException, ETooCostly, IOException {
715    opContext.deadCheck("excludeCodes");
716    exc.checkNoModifiers("Compose.exclude", "expanding");
717    if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
718      wc.getExcludeSystems().add(exc.getSystem());
719    }
720
721    for (UriType imp : exc.getValueSet()) {
722      excludeCodes(wc, importValueSetForExclude(wc, imp.getValue(), exp, expParams, false, vs).getExpansion());
723    }
724    
725    if (exc.hasSystem()) {
726      String sv = exc.getSystem()+(exc.hasVersion() ? "#"+exc.getVersion(): "");
727      if (dwc.getCountIncompleteSystems().contains(sv)) {
728        dwc.setNoTotal(true);
729      }
730
731      CodeSystem cs = context.fetchSupplementedCodeSystem(exc.getSystem());
732      if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem())) {
733        ValueSetExpansionOutcome vse = context.expandVS(new TerminologyOperationDetails(requiredSupplements), exc, false, false);
734        ValueSet valueset = vse.getValueset();
735        if (valueset.hasUserData(UserDataNames.VS_EXPANSION_SOURCE)) {
736          sources.add(valueset.getUserString(UserDataNames.VS_EXPANSION_SOURCE));
737        }
738        if (valueset == null)
739          throw failTSE("Error Expanding ValueSet: "+vse.getError());
740        excludeCodes(wc, valueset.getExpansion());
741        return;
742      }
743
744      for (ConceptReferenceComponent c : exc.getConcept()) {
745        excludeCode(wc, exc.getSystem(), c.getCode());
746      }
747
748      if (exc.getFilter().size() > 0) {
749        if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
750          addFragmentWarning(exp, cs);
751        }
752        List<WorkingContext> filters = new ArrayList<>();
753        for (int i = 1; i < exc.getFilter().size(); i++) {
754          WorkingContext wc1 = new WorkingContext();
755          filters.add(wc1);
756          processFilter(exc, exp, expParams, null, cs, false, exc.getFilter().get(i), wc1, null, true, vspath+".filter["+i+"]");
757        }
758        ConceptSetFilterComponent fc = exc.getFilter().get(0);
759        WorkingContext wc1 = dwc;
760        processFilter(exc, exp, expParams, null, cs, false, fc, wc1, filters, true, vspath+".filter[0]");
761      }
762    }
763  }
764
765  private void excludeCodes(WorkingContext wc, ValueSetExpansionComponent expand) {
766    opContext.deadCheck("excludeCodes");
767    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
768      excludeCode(wc, c.getSystem(), c.getCode());
769    }
770  }
771
772  private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, DataType value) {
773    for (ValueSetExpansionParameterComponent p : params) {
774      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false)) {
775        return true;
776      }
777    }
778    return false;
779  }
780
781  public ValueSetExpansionOutcome expand(ValueSet source, Parameters expParams) {
782    allErrors.clear();
783    try {
784      opContext.seeContext(source.getVersionedUrl());
785      
786      return expandInternal(source, expParams);
787    } catch (NoTerminologyServiceException e) {
788      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
789      // that might fail too, but it might not, later.
790      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.NOSERVICE, allErrors, false);
791    } catch (CodeSystemProviderExtension e) {
792      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
793      // that might fail too, but it might not, later.
794      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.INTERNAL_ERROR, allErrors, false);
795    } catch (TerminologyServiceProtectionException e) {
796      if (opContext.isOriginal()) {
797        return new ValueSetExpansionOutcome(e.getMessage(), e.getError(), allErrors, false);
798      } else {
799        throw e;
800      }
801    } catch (ETooCostly e) {
802      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.TOO_COSTLY, allErrors, false);
803    } catch (UnknownValueSetException e) {
804      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.VALUESET_UNKNOWN, allErrors, false);
805    } catch (VSCheckerException e) {
806      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.UNKNOWN, allErrors, e.getIssues());      
807    } catch (Exception e) {
808      if (debug) {
809        e.printStackTrace();
810      }
811      return new ValueSetExpansionOutcome(e.getMessage(), TerminologyServiceErrorClass.UNKNOWN, allErrors, e instanceof EFhirClientException || e instanceof TerminologyServiceException);
812    }
813  }
814
815  public ValueSetExpansionOutcome expandInternal(ValueSet source, Parameters expParams) throws FHIRException, FileNotFoundException, ETooCostly, IOException, CodeSystemProviderExtension {
816    if (expParams == null)
817      expParams = makeDefaultExpansion();
818    altCodeParams.seeParameters(expParams);
819    altCodeParams.seeValueSet(source);
820    source.checkNoModifiers("ValueSet", "expanding");
821    focus = source.copy();
822    focus.setIdBase(null);
823    focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
824    focus.getExpansion().setTimestampElement(DateTimeType.now());
825    focus.getExpansion().setIdentifier(Factory.createUUID()); 
826    checkCanonical(focus.getExpansion(), focus, focus);
827    for (Extension ext : focus.getCompose().getExtensionsByUrl(ExtensionDefinitions.EXT_VS_EXP_PARAM_NEW, ExtensionDefinitions.EXT_VS_EXP_PARAM_OLD)) {
828      processParameter(ext.getExtensionString("name"), ext.getExtensionByUrl("value").getValue());
829    }
830    for (ParametersParameterComponent p : expParams.getParameter()) {
831      processParameter(p.getName(), p.getValue());
832    }
833    for (Extension s : focus.getExtensionsByUrl(ExtensionDefinitions.EXT_VS_CS_SUPPL_NEEDED)) {
834      requiredSupplements.add(s.getValue().primitiveValue());
835    }
836    if (langs == null && focus.hasLanguage()) {
837      langs = new AcceptLanguageHeader(focus.getLanguage(), true);
838    }
839
840    try {
841      if (source.hasCompose()) {
842        //        ExtensionsUtils.stripExtensions(focus.getCompose()); - disabled 23/05/2023 GDG - why was this ever thought to be a good idea?
843        handleCompose(source.getCompose(), focus.getExpansion(), expParams, source.getUrl(), focus.getExpansion().getExtension(), source);
844      }
845    } catch (EFinished e) {
846      // nothing - we intended to trap this here
847    }
848
849    if (dwc.getCount() > maxExpansionSize && dwc.getOffsetParam() + dwc.getCountParam() == 0) {
850      if (dwc.isNoTotal()) {
851        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize)));        
852      } else {
853        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY_COUNT, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize), MessageFormat.format("{0,number,#}", dwc.getCount())));
854      }
855    } else if (dwc.isCanBeHierarchy() && ((dwc.getCountParam() == 0) || dwc.getCountParam() > dwc.getCodes().size())) {
856      for (ValueSetExpansionContainsComponent c : dwc.getRoots()) {
857        focus.getExpansion().getContains().add(c);
858      }
859    } else {
860      int i = 0;
861      int cc = 0;
862      for (ValueSetExpansionContainsComponent c : dwc.getCodes()) {
863        c.getContains().clear(); // make sure any hierarchy is wiped
864        if (dwc.getMap().containsKey(key(c)) && (includeAbstract || !c.getAbstract())) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
865          if (dwc.getOffsetParam() == 0 || i >= dwc.getOffsetParam()) {
866            focus.getExpansion().getContains().add(c);
867            cc++;
868            if (cc == dwc.getCountParam()) {
869              break;
870            }
871          }
872          i++;
873        }
874      }
875    }
876
877    if (dwc.hasOffsetParam()) {
878      focus.getExpansion().setOffset(dwc.getOffsetParam());
879    }
880    if (!dwc.isNoTotal()) {
881      focus.getExpansion().setTotal(dwc.getStatedTotal());
882    }
883    if (!requiredSupplements.isEmpty()) {      
884      return new ValueSetExpansionOutcome(context.formatMessagePlural(requiredSupplements.size(), I18nConstants.VALUESET_SUPPLEMENT_MISSING, CommaSeparatedStringBuilder.build(requiredSupplements)), TerminologyServiceErrorClass.BUSINESS_RULE, allErrors, false);
885    }
886    if (!expParams.hasParameter("includeDefinition") || !expParams.getParameterBool("includeDefinition")) {
887      focus.setCompose(null);
888      focus.getExtension().clear();
889      focus.setPublisher(null);
890      focus.setDescription(null);
891      focus.setPurpose(null);
892      focus.getContact().clear();
893      focus.setCopyright(null);
894      focus.setText(null);
895    }
896    return new ValueSetExpansionOutcome(focus);
897  }
898
899  private void processParameter(String name, DataType value) {
900    if (Utilities.existsInList(name, "includeDesignations", "excludeNested", "activeOnly", "offset", "count")) {
901      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
902      focus.getExpansion().addParameter().setName(name).setValue(value);
903    }
904    if ("displayLanguage".equals(name)) {
905      this.langs = new AcceptLanguageHeader(value.primitiveValue(), true);
906      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
907      focus.getExpansion().addParameter().setName(name).setValue(new CodeType(value.primitiveValue()));
908    }
909    if ("designation".equals(name)) {
910      String[] v = value.primitiveValue().split("\\|");
911      if (v.length != 2 || !Utilities.isAbsoluteUrl(v[0]) || Utilities.noString(v[1])) {
912        throw new NoTerminologyServiceException("Unable to understand designation parameter "+value.primitiveValue());
913      }
914      this.designations.add(new Token(v[0], v[1]));
915      focus.getExpansion().addParameter().setName(name).setValue(new StringType(value.primitiveValue()));
916    }
917    if ("offset".equals(name) && value instanceof IntegerType) {
918      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
919      focus.getExpansion().addParameter().setName(name).setValue(value);
920      dwc.setOffsetParam(((IntegerType) value).getValue());
921      if (dwc.getOffsetParam() < 0) {
922        dwc.setOffsetParam(0);
923      }
924    }
925    if ("count".equals(name)) {
926      focus.getExpansion().getParameter().removeIf(p -> p.getName().equals(name));
927      focus.getExpansion().addParameter().setName(name).setValue(value);
928      dwc.setCountParam(((IntegerType) value).getValue());
929      if (dwc.getCountParam() < 0) {
930        dwc.setCountParam(0);
931      }
932    }
933  }
934  
935  public ValueSetExpansionOutcome doExpand(ValueSet source, Parameters expParams) throws FHIRException, ETooCostly, FileNotFoundException, IOException, CodeSystemProviderExtension {
936    if (expParams == null)
937      expParams = makeDefaultExpansion();
938    altCodeParams.seeParameters(expParams);
939    altCodeParams.seeValueSet(source);
940    source.checkNoModifiers("ValueSet", "expanding");
941    focus = source.copy();
942    focus.setIdBase(null);
943    focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
944    focus.getExpansion().setTimestampElement(DateTimeType.now());
945    focus.getExpansion().setIdentifier(Factory.createUUID()); 
946    checkCanonical(focus.getExpansion(), focus, focus);
947    for (Extension ext : focus.getCompose().getExtensionsByUrl(ExtensionDefinitions.EXT_VS_EXP_PARAM_NEW, ExtensionDefinitions.EXT_VS_EXP_PARAM_OLD)) {
948      processParameter(ext.getExtensionString("name"), ext.getExtensionByUrl("value").getValue());
949    }
950    for (ParametersParameterComponent p : expParams.getParameter()) {
951      processParameter(p.getName(), p.getValue());
952    }
953    for (Extension s : focus.getExtensionsByUrl(ExtensionDefinitions.EXT_VS_CS_SUPPL_NEEDED)) {
954      requiredSupplements.add(s.getValue().primitiveValue());
955    }
956    if (langs == null && focus.hasLanguage()) {
957      langs = new AcceptLanguageHeader(focus.getLanguage(), true);
958    }
959
960    try {
961      if (source.hasCompose()) {
962//        ExtensionsUtils.stripExtensions(focus.getCompose()); - disabled 23/05/2023 GDG - why was this ever thought to be a good idea?
963        handleCompose(source.getCompose(), focus.getExpansion(), expParams, source.getUrl(), focus.getExpansion().getExtension(), source);
964      }
965    } catch (EFinished e) {
966      // nothing - we intended to trap this here
967    }
968
969    if (dwc.getCount() > maxExpansionSize && dwc.getOffsetParam() + dwc.getCountParam() == 0) {
970      if (dwc.isNoTotal()) {
971        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize)));        
972      } else {
973        throw failCostly(context.formatMessage(I18nConstants.VALUESET_TOO_COSTLY_COUNT, focus.getVersionedUrl(), ">" + MessageFormat.format("{0,number,#}", maxExpansionSize), MessageFormat.format("{0,number,#}", dwc.getCount())));
974      }
975    } else if (dwc.isCanBeHierarchy() && ((dwc.getCountParam() == 0) || dwc.getCountParam() > dwc.getCodes().size())) {
976      for (ValueSetExpansionContainsComponent c : dwc.getRoots()) {
977        focus.getExpansion().getContains().add(c);
978      }
979    } else {
980      int i = 0;
981      int cc = 0;
982      for (ValueSetExpansionContainsComponent c : dwc.getCodes()) {
983        c.getContains().clear(); // make sure any hierarchy is wiped
984        if (dwc.getMap().containsKey(key(c)) && (includeAbstract || !c.getAbstract())) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
985          if (dwc.getOffsetParam() == 0 || i >= dwc.getOffsetParam()) {
986            focus.getExpansion().getContains().add(c);
987            cc++;
988            if (cc == dwc.getCountParam()) {
989              break;
990            }
991          }
992          i++;
993        }
994      }
995    }
996
997    if (dwc.hasOffsetParam()) {
998      focus.getExpansion().setOffset(dwc.getOffsetParam());
999    }
1000    if (!dwc.isNoTotal()) {
1001      focus.getExpansion().setTotal(dwc.getStatedTotal());
1002    }
1003    if (!requiredSupplements.isEmpty()) {      
1004      return new ValueSetExpansionOutcome(context.formatMessagePlural(requiredSupplements.size(), I18nConstants.VALUESET_SUPPLEMENT_MISSING, CommaSeparatedStringBuilder.build(requiredSupplements)), TerminologyServiceErrorClass.BUSINESS_RULE, allErrors, false);
1005    }
1006    if (!expParams.hasParameter("includeDefinition") || !expParams.getParameterBool("includeDefinition")) {
1007      focus.setCompose(null);
1008      focus.getExtension().clear();
1009      focus.setPublisher(null);
1010      focus.setDescription(null);
1011      focus.setPurpose(null);
1012      focus.getContact().clear();
1013      focus.setCopyright(null);
1014      focus.setText(null);
1015    }
1016    return new ValueSetExpansionOutcome(focus);
1017  }
1018
1019
1020  private Parameters makeDefaultExpansion() {
1021    Parameters res = new Parameters();
1022    res.addParameter("excludeNested", true);
1023    res.addParameter("includeDesignations", false);
1024    return res;
1025  }
1026
1027  private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
1028    for (ConceptDefinitionComponent c : clist) {
1029      if (code.equals(c.getCode()))
1030        return c;
1031      ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
1032      if (v != null)
1033        return v;
1034    }
1035    return null;
1036  }
1037
1038  private void handleCompose(ValueSetComposeComponent compose, ValueSetExpansionComponent exp, Parameters expParams, String ctxt, List<Extension> extensions, ValueSet valueSet)
1039      throws ETooCostly, FileNotFoundException, IOException, FHIRException, CodeSystemProviderExtension {
1040    compose.checkNoModifiers("ValueSet.compose", "expanding");
1041    String vspath = "ValueSet["+valueSet.getVersionedUrl()+"].compose";
1042    
1043    // Exclude comes first because we build up a map of things to exclude
1044    int c = 0;
1045    for (ConceptSetComponent inc : compose.getExclude()) {
1046      excludeCodes(dwc, inc, expParams, exp, valueSet, vspath+".include["+c+"]");
1047      c++;
1048    }
1049    dwc.setCanBeHierarchy(!expParams.getParameterBool("excludeNested") && dwc.getExcludeKeys().isEmpty() && dwc.getExcludeSystems().isEmpty() && dwc.getOffsetParam() == 0);
1050    includeAbstract = !expParams.getParameterBool("excludeNotForUI");
1051    boolean first = true;
1052    c = 0;
1053    for (ConceptSetComponent inc : compose.getInclude()) {
1054      if (first == true)
1055        first = false;
1056      else
1057        dwc.setCanBeHierarchy(false);
1058      includeCodes(inc, exp, expParams, dwc.isCanBeHierarchy(), compose.hasInactive() ? !compose.getInactive() : checkNoInActiveFromParam(expParams), extensions, valueSet, vspath+".include["+c+"]");
1059      c++;
1060    }
1061  }
1062
1063  /**
1064   * returns true if activeOnly = true 
1065   * @param expParams
1066   * @return
1067   */
1068  private boolean checkNoInActiveFromParam(Parameters expParams) {
1069    for (ParametersParameterComponent p : expParams.getParameter()) {
1070      if (p.getName().equals("activeOnly")) {
1071        return p.getValueBooleanType().getValue();
1072      }
1073    }
1074    return false;
1075  }
1076
1077  private ValueSet importValueSet(WorkingContext wc, String value, ValueSetExpansionComponent exp, Parameters expParams, boolean noInactive, ValueSet valueSet) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
1078    if (value == null)
1079      throw fail(I18nConstants.VS_EXP_IMPORT_NULL, true);
1080    String url = getCu().pinValueSet(value, expParams);
1081    ValueSet vs = context.findTxResource(ValueSet.class, url, valueSet);
1082    if (vs == null) {
1083      boolean pinned = !url.equals(value);
1084      String ver = pinned ? url.substring(value.length()+1) : null;
1085      if (context.fetchResource(CodeSystem.class, url, valueSet) != null) {
1086        throw failNotFound(pinned ? I18nConstants.VS_EXP_IMPORT_CS_PINNED : I18nConstants.VS_EXP_IMPORT_CS, true, value, ver);
1087      } else  {
1088        throw failNotFound(pinned ? I18nConstants.VS_EXP_IMPORT_UNK_PINNED : I18nConstants.VS_EXP_IMPORT_UNK, true, value, ver);
1089      }
1090    }
1091    checkCanonical(exp, vs, focus);
1092    if (noInactive) {
1093      expParams = expParams.copy();
1094      expParams.addParameter("activeOnly", true);
1095    }
1096    ValueSetExpander expander = new ValueSetExpander(context, opContext.copy(), allErrors);
1097    ValueSetExpansionOutcome vso = expander.expand(vs, expParams);
1098    if (vso.getError() != null) {
1099      addErrors(vso.getAllErrors());
1100      if (vso.getErrorClass() == TerminologyServiceErrorClass.VALUESET_UNKNOWN) {  
1101        throw failUnk(I18nConstants.VS_EXP_IMPORT_ERROR, true, vs.getUrl(), vso.getError());
1102      } else {
1103        throw fail(I18nConstants.VS_EXP_IMPORT_ERROR, true, vs.getUrl(), vso.getError());
1104      }
1105    } else if (vso.getValueset() == null) {
1106      throw fail(I18nConstants.VS_EXP_IMPORT_FAIL, true, vs.getUrl());      
1107    }
1108    sources.addAll(expander.sources);
1109    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
1110      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
1111      if (!existsInParams(exp.getParameter(), "used-valueset", u))
1112        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
1113    }
1114    ValueSetExpansionComponent evs = vso.getValueset().getExpansion();
1115    for (Extension ex : evs.getExtension()) {
1116      if (ex.getUrl().equals(ExtensionDefinitions.EXT_EXP_TOOCOSTLY)) {
1117        if (ex.getValue() instanceof BooleanType) {
1118          exp.getExtension().add(new Extension(ExtensionDefinitions.EXT_EXP_TOOCOSTLY).setValue(new CanonicalType(value)));
1119        } else {
1120          exp.getExtension().add(ex);
1121        }
1122      } 
1123    }
1124    if (!evs.hasTotal()) {
1125      // because if there's no total, we can't know if we got everything
1126      dwc.setNoTotal(true);
1127    }
1128    for (ValueSetExpansionParameterComponent p : evs.getParameter()) {
1129      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue()))
1130        exp.getParameter().add(p);
1131    }
1132    if (isValueSetUnionImports(valueSet)) {
1133      copyExpansion(wc, evs.getContains());
1134    }
1135    wc.setCanBeHierarchy(false); // if we're importing a value set, we have to be combining, so we won't try for a hierarchy
1136    return vso.getValueset();
1137  }
1138  
1139
1140
1141  private ValueSet importValueSetForExclude(WorkingContext wc, String value, ValueSetExpansionComponent exp, Parameters expParams, boolean noInactive, ValueSet valueSet) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
1142    if (value == null)
1143      throw fail(I18nConstants.VS_EXP_IMPORT_NULL_X, true);  
1144    String url = getCu().pinValueSet(value, expParams);
1145    ValueSet vs = context.findTxResource(ValueSet.class, url, valueSet);
1146    if (vs == null) {
1147      boolean pinned = !url.equals(value);
1148      String ver = pinned ? url.substring(value.length()+1) : null;
1149      if (context.fetchResource(CodeSystem.class, url, valueSet) != null) {
1150        throw fail(pinned ? I18nConstants.VS_EXP_IMPORT_CS_PINNED_X : I18nConstants.VS_EXP_IMPORT_CS_X, true, value, ver);
1151      } else  {
1152        throw fail(pinned ? I18nConstants.VS_EXP_IMPORT_UNK_PINNED_X : I18nConstants.VS_EXP_IMPORT_UNK_X, true, value, ver);
1153      }
1154    }
1155    checkCanonical(exp, vs, focus);
1156    if (noInactive) {
1157      expParams = expParams.copy();
1158      expParams.addParameter("activeOnly", true);
1159    }
1160    ValueSetExpander expander = new ValueSetExpander(context, opContext.copy(), allErrors);
1161    ValueSetExpansionOutcome vso = expander.expand(vs, expParams);
1162    sources.addAll(expander.sources);
1163    if (vso.getError() != null) {
1164      addErrors(vso.getAllErrors());
1165      throw fail(I18nConstants.VS_EXP_IMPORT_ERROR_X, true, vs.getUrl(), vso.getError());
1166    } else if (vso.getValueset() == null) {
1167      throw fail(I18nConstants.VS_EXP_IMPORT_FAIL_X, true, vs.getUrl());      
1168    }
1169    
1170    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
1171      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
1172      if (!existsInParams(exp.getParameter(), "used-valueset", u))
1173        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
1174    }
1175    for (Extension ex : vso.getValueset().getExpansion().getExtension()) {
1176      if (ex.getUrl().equals(ExtensionDefinitions.EXT_EXP_TOOCOSTLY)) {
1177        throw fail(VS_EXP_IMPORT_ERROR_TOO_COSTLY, true, vs.getUrl());
1178      } 
1179    }
1180    return vso.getValueset();
1181  }
1182  
1183  protected boolean isValueSetUnionImports(ValueSet valueSet) {
1184    PackageInformation p = valueSet.getSourcePackage();
1185    if (p != null) {
1186      return p.getDate().before(new GregorianCalendar(2022, Calendar.MARCH, 31).getTime());
1187    } else {
1188      return false;
1189    }
1190  }
1191
1192  public void copyExpansion(WorkingContext wc,List<ValueSetExpansionContainsComponent> list) {
1193    opContext.deadCheck("copyExpansion");
1194    for (ValueSetExpansionContainsComponent cc : list) {
1195       ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
1196       n.setSystem(cc.getSystem());
1197       n.setCode(cc.getCode());
1198       n.setAbstract(cc.getAbstract());
1199       n.setInactive(cc.getInactive());
1200       n.setDisplay(cc.getDisplay());
1201       n.getDesignation().addAll(cc.getDesignation());
1202
1203       String s = key(n);
1204       if (!wc.getMap().containsKey(s) && !wc.getExcludeKeys().contains(s)) {
1205         wc.getCodes().add(n);
1206         wc.getMap().put(s, n);
1207       }
1208       copyExpansion(wc, cc.getContains());
1209    }
1210  }
1211
1212  private void addErrors(List<String> errs) {
1213    for (String s : errs) {
1214      if (!allErrors.contains(s)) {
1215        allErrors.add(s);
1216      }
1217    }
1218  }
1219
1220  private int copyImportContains(List<ValueSetExpansionContainsComponent> list, ValueSetExpansionContainsComponent parent, Parameters expParams, List<ValueSet> filter, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps, ValueSet vsSrc, ValueSetExpansionComponent exp) throws FHIRException, ETooCostly {
1221    int count = 0;
1222    opContext.deadCheck("copyImportContains");
1223    
1224    String lang = vsSrc.getLanguage();
1225    if (lang == null) { 
1226      lang = findParamValue(vsSrc.getExpansion().getParameter(), "displayLanguage");
1227    }
1228    for (ValueSetExpansionContainsComponent c : list) {
1229      c.checkNoModifiers("Imported Expansion in Code System", "expanding");
1230      String vstatus = determineStatus(vsSrc, c);
1231      ValueSetExpansionContainsComponent np = addCode(dwc, c.getSystem(), c.getCode(), c.getDisplay(), lang, parent, translateDesignations(c), expParams,
1232            c.getAbstract(), c.getInactive(), filter, noInactive, false, vsProps, makeCSProps(c.getExtensionString(ExtensionDefinitions.EXT_DEFINITION), null),
1233        null, c.getProperty(), null, c.getExtension(), exp, vstatus);
1234      if (np != null) {
1235        count++;
1236      }
1237      count = count + copyImportContains(c.getContains(), np, expParams, filter, noInactive, vsProps, vsSrc, exp);
1238    }
1239    return count;
1240  }
1241
1242  private List<ConceptDefinitionDesignationComponent> translateDesignations(ValueSetExpansionContainsComponent c) {
1243    if (!c.hasDesignation()) {
1244      return null;
1245    }
1246    List<ConceptDefinitionDesignationComponent> list = new ArrayList<>();
1247    for (ConceptReferenceDesignationComponent d : c.getDesignation()) {
1248      ConceptDefinitionDesignationComponent d2 = new ConceptDefinitionDesignationComponent();
1249      d2.setLanguage(d.getLanguage());
1250      d2.setUse(d.getUse());
1251      d2.setAdditionalUse(d.getAdditionalUse());
1252      d2.setValue(d.getValue());
1253      list.add(d2);
1254    }
1255    return list;
1256  }
1257
1258  private void includeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, boolean heirarchical, boolean noInactive, List<Extension> extensions, ValueSet valueSet, String vspath) throws ETooCostly, FileNotFoundException, IOException, FHIRException, CodeSystemProviderExtension {
1259    opContext.deadCheck("includeCodes");
1260    inc.checkNoModifiers("Compose.include", "expanding");
1261    List<ValueSet> imports = new ArrayList<ValueSet>();
1262    for (CanonicalType imp : inc.getValueSet()) {
1263      imports.add(importValueSet(dwc, imp.getValue(), exp, expParams, noInactive, valueSet));
1264    }
1265
1266    if (!inc.hasSystem()) {
1267      if (imports.isEmpty()) // though this is not supposed to be the case
1268        return;
1269      ValueSet base = imports.get(0);
1270      checkCanonical(exp, base, focus);
1271      imports.remove(0);
1272      base.checkNoModifiers("Imported ValueSet", "expanding");
1273      copyImportContains(base.getExpansion().getContains(), null, expParams, imports, noInactive, base.getExpansion().getProperty(), base, exp);
1274    } else {
1275      String sv = inc.getSystem()+(inc.hasVersion() ? "#"+inc.getVersion(): "");
1276      if (dwc.getCountIncompleteSystems().contains(sv)) {
1277        dwc.setNoTotal(true);
1278      }
1279      CodeSystem cs = context.fetchSupplementedCodeSystem(inc.getSystem());
1280      if (ValueSetUtilities.isServerSide(inc.getSystem()) || (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT))) {
1281        doServerIncludeCodes(inc, heirarchical, exp, imports, expParams, extensions, noInactive, valueSet.getExpansion().getProperty());
1282      } else {
1283        if (cs.hasUserData(UserDataNames.tx_known_supplements)) {
1284          for (String s : cs.getUserString(UserDataNames.tx_known_supplements).split("\\,")) {
1285            requiredSupplements.remove(s);
1286          }
1287        }
1288        doInternalIncludeCodes(inc, exp, expParams, imports, cs, noInactive, valueSet, vspath);
1289      }
1290    }
1291  }
1292
1293  private void doServerIncludeCodes(ConceptSetComponent inc, boolean heirarchical, ValueSetExpansionComponent exp, List<ValueSet> imports, Parameters expParams, List<Extension> extensions, boolean noInactive, List<ValueSetExpansionPropertyComponent> vsProps) throws FHIRException, CodeSystemProviderExtension, ETooCostly {
1294    opContext.deadCheck("doServerIncludeCodes");
1295    CodeSystemProvider csp = CodeSystemProvider.factory(inc.getSystem());
1296    if (csp != null) {
1297      csp.includeCodes(inc, heirarchical, exp, imports, expParams, extensions, noInactive, vsProps);
1298      return;
1299    }
1300    
1301    ValueSetExpansionOutcome vso = context.expandVS(new TerminologyOperationDetails(requiredSupplements), inc, heirarchical, noInactive);
1302    if (vso.getError() != null) {
1303      throw failTSE("Unable to expand imported value set: " + vso.getError());
1304    }
1305    ValueSet vs = vso.getValueset();
1306    if (vs.hasUserData(UserDataNames.VS_EXPANSION_SOURCE)) {
1307      sources.add(vs.getUserString(UserDataNames.VS_EXPANSION_SOURCE));
1308    }
1309    if (vs.hasVersion() || REPORT_VERSION_ANYWAY) {
1310      UriType u = new UriType(vs.getUrl() + (vs.hasVersion() ? "|"+vs.getVersion() : ""));
1311      if (!existsInParams(exp.getParameter(), "used-valueset", u)) {
1312        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-valueset").setValue(u));
1313      }
1314    }
1315    if (vs.getExpansion().hasTotal()) {
1316      dwc.incExtraCount(vs.getExpansion().getTotal() - countContains(vs.getExpansion().getContains()));
1317      dwc.getCountIncompleteSystems().add(inc.getSystem()+(inc.hasVersion() ? "#"+inc.getVersion(): ""));
1318    } else {
1319      dwc.setNoTotal(true);
1320    }
1321    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
1322      if (!existsInParams(exp.getParameter(), p.getName(), p.getValue())) {
1323        exp.getParameter().add(p);
1324      }
1325    }
1326    for (Extension ex : vs.getExpansion().getExtension()) {
1327      if (Utilities.existsInList(ex.getUrl(), ExtensionDefinitions.EXT_EXP_TOOCOSTLY, "http://hl7.org/fhir/StructureDefinition/valueset-unclosed")) {
1328        if (!ExtensionUtilities.hasExtension(extensions, ex.getUrl())) {
1329          extensions.add(ex);
1330        }
1331      }
1332    }
1333    for (ValueSetExpansionContainsComponent cc : vs.getExpansion().getContains()) {
1334      addCodeAndDescendents(dwc, cc, null, expParams, imports, noInactive, vsProps, vs, exp, getLang(vs));
1335    }
1336  }
1337
1338
1339  private int countContains(List<ValueSetExpansionContainsComponent> contains) {
1340    int count = contains.size();
1341    for (ValueSetExpansionContainsComponent cc : contains) {
1342      if (cc.hasContains()) {
1343        count += countContains(cc.getContains());
1344      }
1345    }
1346    return count;
1347  }
1348
1349  public void doInternalIncludeCodes(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, List<ValueSet> imports, CodeSystem cs, boolean noInactive, Resource vsSrc, String vspath) throws NoTerminologyServiceException, TerminologyServiceException, FHIRException, ETooCostly {
1350    opContext.deadCheck("doInternalIncludeCodes");
1351    if (cs == null) {
1352      if (context.isNoTerminologyServer())
1353        throw failTSE("Unable to find code system " + inc.getSystem().toString());
1354      else
1355        throw failTSE("Unable to find code system " + inc.getSystem().toString());
1356    }
1357    checkCanonical(exp, cs, focus);
1358    cs.checkNoModifiers("Code System", "expanding");
1359    if (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT)
1360      throw failTSE("Code system " + inc.getSystem().toString() + " is incomplete");
1361    if (cs.hasVersion() || REPORT_VERSION_ANYWAY) {
1362      UriType u = new UriType(cs.getUrl() + (cs.hasVersion() ? "|"+cs.getVersion() : ""));
1363      if (!existsInParams(exp.getParameter(), "used-codesystem", u))
1364        exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-codesystem").setValue(u));
1365      if (cs.hasUserData(UserDataNames.tx_known_supplements)) {
1366        for (String s : cs.getUserString(UserDataNames.tx_known_supplements).split("\\,")) {
1367          u = new UriType(s);
1368          if (!existsInParams(exp.getParameter(), "used-supplement", u)) {
1369            exp.getParameter().add(new ValueSetExpansionParameterComponent().setName("used-supplement").setValue(u));
1370          }
1371        }
1372      }
1373    }
1374    if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
1375      // special case - add all the code system
1376      for (ConceptDefinitionComponent def : cs.getConcept()) {
1377        addCodeAndDescendents(dwc, cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), null, exp);
1378      }
1379      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1380        addFragmentWarning(exp, cs);
1381      }
1382      if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
1383        addExampleWarning(exp, cs);
1384      }      
1385    }
1386
1387    if (!inc.getConcept().isEmpty()) {
1388      dwc.setCanBeHierarchy(false);
1389      for (ConceptReferenceComponent c : inc.getConcept()) {
1390        c.checkNoModifiers("Code in Value Set", "expanding");
1391        ConceptDefinitionComponent def = CodeSystemUtilities.findCodeOrAltCode(cs.getConcept(), c.getCode(), null);
1392        boolean inactive = false; // default is true if we're a fragment and  
1393        boolean isAbstract = false;
1394        if (def == null) {
1395          if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1396            addFragmentWarning(exp, cs);
1397          } else if (cs.getContent() == CodeSystemContentMode.EXAMPLE) {
1398              addExampleWarning(exp, cs);
1399          } else {
1400            if (checkCodesWhenExpanding) {
1401              throw failTSE("Unable to find code '" + c.getCode() + "' in code system " + cs.getUrl());
1402            }
1403          }
1404        } else {
1405          def.checkNoModifiers("Code in Code System", "expanding");
1406          inactive = CodeSystemUtilities.isInactive(cs, def);
1407          isAbstract = CodeSystemUtilities.isNotSelectable(cs, def);
1408          String vstatus = determineStatus(cs, def);
1409          addCode(dwc, inc.getSystem(), c.getCode(), !Utilities.noString(c.getDisplay()) ? c.getDisplay() : def.getDisplay(), c.hasDisplay() ? vsSrc.getLanguage() : cs.getLanguage(),
1410            null, mergeDesignations(def, convertDesignations(c.getDesignation())), expParams, isAbstract, inactive, imports, noInactive, false,
1411            exp.getProperty(), makeCSProps(def.getDefinition(), def.getProperty()), cs, null, def.getExtension(), c.getExtension(), exp, vstatus);
1412        }
1413      }
1414    }
1415    if (inc.getFilter().size() > 0) {
1416      if (inc.getFilter().size() > 1) {
1417        dwc.setCanBeHierarchy(false); // which will be the case if we get around to supporting this
1418      }
1419      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
1420        addFragmentWarning(exp, cs);
1421      }
1422      List<WorkingContext> filters = new ArrayList<>();
1423      for (int i = 1; i < inc.getFilter().size(); i++) {
1424        WorkingContext wc = new WorkingContext();
1425        filters.add(wc);
1426        processFilter(inc, exp, expParams, imports, cs, noInactive, inc.getFilter().get(i), wc, null, false, vspath+".filter["+i+"]");
1427      }
1428      ConceptSetFilterComponent fc = inc.getFilter().get(0);
1429      WorkingContext wc = dwc;
1430      processFilter(inc, exp, expParams, imports, cs, noInactive, fc, wc, filters, false, vspath+".filter[0]");
1431    }
1432  }
1433
1434  private void processFilter(ConceptSetComponent inc, ValueSetExpansionComponent exp, Parameters expParams, List<ValueSet> imports, CodeSystem cs, boolean noInactive, 
1435      ConceptSetFilterComponent fc, WorkingContext wc, List<WorkingContext> filters, boolean exclude, String vspath)
1436      throws ETooCostly {
1437    
1438    if (!fc.hasValue() || fc.getValue() == null) {
1439      List<OperationOutcomeIssueComponent> issues = new ArrayList<>();
1440      issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.INVALID, vspath+".value", context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE,
1441        cs.getUrl(), fc.getProperty(), fc.getOp().toCode()), OpIssueCode.VSProcessing, null, I18nConstants.UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE));
1442      throw new VSCheckerException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE, cs.getUrl(), fc.getProperty(), fc.getOp().toCode()), issues, TerminologyServiceErrorClass.INTERNAL_ERROR);
1443    }
1444    opContext.deadCheck("processFilter");
1445    if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
1446      // special: all codes in the target code system under the value
1447      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1448      if (def == null)
1449        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1450      if (exclude) {
1451        excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1452      } else {
1453        addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1454      }
1455    } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISNOTA) {
1456      // special: all codes in the target code system that are not under the value
1457      ConceptDefinitionComponent defEx = getConceptForCode(cs.getConcept(), fc.getValue());
1458      if (defEx == null)
1459        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1460      for (ConceptDefinitionComponent def : cs.getConcept()) {
1461        if (exclude) {
1462          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, defEx, new AllConceptsFilter(allErrors), filters, exp);
1463        } else {
1464          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, defEx, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1465        }
1466      }
1467    } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
1468      // special: all codes in the target code system under the value
1469      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1470      if (def == null)
1471        throw failTSE("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
1472      for (ConceptDefinitionComponent c : def.getConcept())
1473        if (exclude) {
1474          excludeCodeAndDescendents(wc, cs, inc.getSystem(), c, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1475        } else {
1476          addCodeAndDescendents(wc, cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1477        }
1478      if (def.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
1479        List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) def.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
1480        for (ConceptDefinitionComponent c : children) {
1481          if (exclude) {
1482            excludeCodeAndDescendents(wc, cs, inc.getSystem(), c, null, imports, null, new AllConceptsFilter(allErrors), filters, exp);
1483          } else {
1484            addCodeAndDescendents(wc, cs, inc.getSystem(), c, null, expParams, imports, null, new AllConceptsFilter(allErrors), noInactive, exp.getProperty(), filters, exp);
1485          }
1486        }
1487      }
1488
1489    } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
1490      // gg; note: wtf is this: if the filter is display=v, look up the code 'v', and see if it's display is 'v'?
1491      dwc.setCanBeHierarchy(false);
1492      ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
1493      if (def != null) {
1494        if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
1495          if (def.getDisplay().contains(fc.getValue()) && passesOtherFilters(filters, cs, def.getCode())) {
1496            for (String code : getCodesForConcept(def, expParams)) {
1497              opContext.deadCheck("processFilter2");
1498              if (exclude) {
1499                excludeCode(wc, inc.getSystem(), code);
1500              } else {
1501                String vstatus = determineStatus(cs, def);
1502                addCode(wc, inc.getSystem(), code, def.getDisplay(), cs.getLanguage(), null, def.getDesignation(), expParams, CodeSystemUtilities.isNotSelectable(cs, def), CodeSystemUtilities.isInactive(cs, def),
1503                  imports, noInactive, false, exp.getProperty(), makeCSProps(def.getDefinition(), def.getProperty()), cs, null, def.getExtension(), null, exp, vstatus);
1504              }
1505            }
1506          }
1507        }
1508      }
1509    } else if (CodeSystemUtilities.isDefinedProperty(cs, fc.getProperty())) {
1510      for (ConceptDefinitionComponent def : cs.getConcept()) {
1511        PropertyFilter pf = new PropertyFilter(allErrors, fc, CodeSystemUtilities.getPropertyDefinition(cs, fc.getProperty()));
1512        if (exclude) {
1513          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, pf, filters, exp);
1514        } else {
1515          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, pf, noInactive, exp.getProperty(), filters, exp);
1516        }
1517      }
1518    } else if (isKnownProperty(fc.getProperty(), cs)) {
1519      for (ConceptDefinitionComponent def : cs.getConcept()) {
1520        KnownPropertyFilter pf = new KnownPropertyFilter(allErrors, fc, fc.getProperty());
1521        if (exclude) {
1522          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, pf, filters, exp);
1523        } else {
1524          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, pf, noInactive, exp.getProperty(), filters, exp);
1525        }
1526      }
1527    } else if ("code".equals(fc.getProperty()) && fc.getOp() == FilterOperator.REGEX) {
1528      for (ConceptDefinitionComponent def : cs.getConcept()) {
1529        if (exclude) {
1530          excludeCodeAndDescendents(wc, cs, inc.getSystem(), def, null, imports, null, new RegexFilter(allErrors, fc.getValue()), filters, exp);
1531        } else {
1532          addCodeAndDescendents(wc, cs, inc.getSystem(), def, null, expParams, imports, null, new RegexFilter(allErrors, fc.getValue()), noInactive, exp.getProperty(), filters, exp);
1533        }
1534      }
1535    } else {
1536      throw fail(I18nConstants.VS_EXP_FILTER_UNK, true, focus.getVersionedUrl(), fc.getProperty(), fc.getOp());
1537    }
1538  }
1539
1540  private boolean isKnownProperty(String property, CodeSystem cs) {
1541    return Utilities.existsInList(property, "notSelectable");
1542  }
1543
1544  private List<ConceptDefinitionDesignationComponent> mergeDesignations(ConceptDefinitionComponent def,
1545      List<ConceptDefinitionDesignationComponent> list) {
1546    List<ConceptDefinitionDesignationComponent> res = new ArrayList<>();
1547    if (def != null) {
1548      res.addAll(def.getDesignation());
1549    }
1550    res.addAll(list);
1551    return res;
1552  }
1553
1554 
1555
1556  private void addFragmentWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
1557    String url = cs.getVersionedUrl();
1558    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
1559      if ("fragment".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
1560        return;
1561      }     
1562    }
1563    exp.addParameter().setName("fragment").setValue(new CanonicalType(url));
1564  }
1565
1566  private void addExampleWarning(ValueSetExpansionComponent exp, CodeSystem cs) {
1567    String url = cs.getVersionedUrl();
1568    for (ValueSetExpansionParameterComponent p : exp.getParameter()) {
1569      if ("example".equals(p.getName()) && p.hasValueUriType() && url.equals(p.getValue().primitiveValue())) { 
1570        return;
1571      }     
1572    }
1573    exp.addParameter().setName("example").setValue(new CanonicalType(url));
1574  }
1575  
1576  private List<ConceptDefinitionDesignationComponent> convertDesignations(List<ConceptReferenceDesignationComponent> list) {
1577    List<ConceptDefinitionDesignationComponent> res = new ArrayList<CodeSystem.ConceptDefinitionDesignationComponent>();
1578    for (ConceptReferenceDesignationComponent t : list) {
1579      ConceptDefinitionDesignationComponent c = new ConceptDefinitionDesignationComponent();
1580      c.setLanguage(t.getLanguage());
1581      if (t.hasUse()) {
1582        c.setUse(t.getUse());
1583      }
1584      c.setValue(t.getValue());
1585      c.getExtension().addAll(t.getExtension());
1586      res.add(c);
1587    }
1588    return res;
1589  }
1590
1591  private String key(String uri, String code) {
1592    return "{" + uri + "}" + code;
1593  }
1594
1595  private String key(ValueSetExpansionContainsComponent c) {
1596    return key(c.getSystem(), c.getCode());
1597  }
1598
1599  private FHIRException fail(String msgId, boolean check, Object... params) {
1600    String msg = context.formatMessage(msgId, params);
1601    allErrors.add(msg);
1602    return new FHIRException(msg);
1603  }
1604
1605  private UnknownValueSetException failUnk(String msgId, boolean check, Object... params) {
1606    String msg = context.formatMessage(msgId, params);
1607    allErrors.add(msg);
1608    return new UnknownValueSetException(msg);
1609  }
1610
1611  private UnknownValueSetException failNotFound(String msgId, boolean check, Object... params) {
1612    String msg = context.formatMessage(msgId, params);
1613    allErrors.add(msg);
1614    return new UnknownValueSetException(msg);
1615  }
1616
1617  private ETooCostly failCostly(String msg) {
1618    allErrors.add(msg);
1619    return new ETooCostly(msg);
1620  }
1621
1622  private TerminologyServiceException failTSE(String msg) {
1623    allErrors.add(msg);
1624    return new TerminologyServiceException(msg);
1625  }
1626
1627  public Collection<? extends String> getAllErrors() {
1628    return allErrors;
1629  }
1630
1631  public boolean isCheckCodesWhenExpanding() {
1632    return checkCodesWhenExpanding;
1633  }
1634
1635  public void setCheckCodesWhenExpanding(boolean checkCodesWhenExpanding) {
1636    this.checkCodesWhenExpanding = checkCodesWhenExpanding;
1637  }
1638
1639  private boolean passesOtherFilters(List<WorkingContext> otherFilters, CodeSystem cs, String code) {
1640    if (otherFilters == null) {
1641      return true;
1642    }
1643    String key = key(cs.getUrl(), code);
1644    for (WorkingContext wc : otherFilters) {
1645      if (!wc.getMap().containsKey(key)) {
1646        return false;
1647      }
1648    }
1649    return true;
1650  }
1651
1652  public boolean isDebug() {
1653    return debug;
1654  }
1655
1656  public ValueSetExpander setDebug(boolean debug) {
1657    this.debug = debug;
1658    return this;
1659  }
1660
1661  public String getSource() {
1662    if (sources.isEmpty()) {
1663      return "internal";
1664    } else {
1665      return CommaSeparatedStringBuilder.join(", ", Utilities.sorted(sources));
1666    }
1667  }
1668  
1669  
1670}