Source code

001package org.hl7.fhir.dstu3.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.ByteArrayOutputStream;
035import java.io.File;
036import java.io.FileInputStream;
037import java.io.FileNotFoundException;
038import java.io.FileOutputStream;
039import java.io.IOException;
040import java.util.ArrayList;
041import java.util.HashMap;
042import java.util.HashSet;
043import java.util.List;
044import java.util.Locale;
045import java.util.Map;
046import java.util.ResourceBundle;
047import java.util.Set;
048
049import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
050import org.hl7.fhir.dstu3.formats.JsonParser;
051import org.hl7.fhir.dstu3.model.BooleanType;
052import org.hl7.fhir.dstu3.model.Bundle;
053import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
054import org.hl7.fhir.dstu3.model.CodeSystem;
055import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemHierarchyMeaning;
056import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
057import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
058import org.hl7.fhir.dstu3.model.CodeableConcept;
059import org.hl7.fhir.dstu3.model.Coding;
060import org.hl7.fhir.dstu3.model.ConceptMap;
061import org.hl7.fhir.dstu3.model.DataElement;
062import org.hl7.fhir.dstu3.model.ExpansionProfile;
063import org.hl7.fhir.dstu3.model.IntegerType;
064import org.hl7.fhir.dstu3.model.OperationDefinition;
065import org.hl7.fhir.dstu3.model.OperationOutcome;
066import org.hl7.fhir.dstu3.model.Parameters;
067import org.hl7.fhir.dstu3.model.Parameters.ParametersParameterComponent;
068import org.hl7.fhir.dstu3.model.PrimitiveType;
069import org.hl7.fhir.dstu3.model.Questionnaire;
070import org.hl7.fhir.dstu3.model.Reference;
071import org.hl7.fhir.dstu3.model.Resource;
072import org.hl7.fhir.dstu3.model.SearchParameter;
073import org.hl7.fhir.dstu3.model.StringType;
074import org.hl7.fhir.dstu3.model.StructureDefinition;
075import org.hl7.fhir.dstu3.model.StructureDefinition.TypeDerivationRule;
076import org.hl7.fhir.dstu3.model.StructureMap;
077import org.hl7.fhir.dstu3.model.UriType;
078import org.hl7.fhir.dstu3.model.ValueSet;
079import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
080import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
081import org.hl7.fhir.dstu3.model.ValueSet.ValueSetComposeComponent;
082import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
083import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
084import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ETooCostly;
085import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
086import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
087import org.hl7.fhir.dstu3.terminologies.ValueSetExpanderFactory;
088import org.hl7.fhir.dstu3.terminologies.ValueSetExpansionCache;
089import org.hl7.fhir.dstu3.utils.ToolingExtensions;
090import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
091import org.hl7.fhir.exceptions.FHIRException;
092import org.hl7.fhir.exceptions.NoTerminologyServiceException;
093import org.hl7.fhir.exceptions.TerminologyServiceException;
094import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
095import org.hl7.fhir.utilities.TextFile;
096import org.hl7.fhir.utilities.Utilities;
097import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
098import org.hl7.fhir.utilities.i18n.I18nBase;
099import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
100import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
101
102import com.google.gson.JsonObject;
103import com.google.gson.JsonSyntaxException;
104
105import ca.uhn.fhir.rest.api.Constants;
106
107public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext {
108
109  // all maps are to the full URI
110  protected Map<String, CodeSystem> codeSystems = new HashMap<String, CodeSystem>();
111  protected Set<String> nonSupportedCodeSystems = new HashSet<String>();
112  protected Map<String, ValueSet> valueSets = new HashMap<String, ValueSet>();
113  protected Map<String, ConceptMap> maps = new HashMap<String, ConceptMap>();
114  protected Map<String, StructureMap> transforms = new HashMap<String, StructureMap>();
115  protected Map<String, DataElement> dataElements = new HashMap<String, DataElement>();
116  protected Map<String, StructureDefinition> profiles = new HashMap<String, StructureDefinition>();
117  protected Map<String, SearchParameter> searchParameters = new HashMap<String, SearchParameter>();
118  protected Map<String, StructureDefinition> extensionDefinitions = new HashMap<String, StructureDefinition>();
119  protected Map<String, Questionnaire> questionnaires = new HashMap<String, Questionnaire>();
120  protected Map<String, OperationDefinition> operations = new HashMap<String, OperationDefinition>();
121
122  protected ValueSetExpanderFactory expansionCache = new ValueSetExpansionCache(this);
123  protected boolean cacheValidation; // if true, do an expansion and cache the expansion
124  private Set<String> failed = new HashSet<String>(); // value sets for which we don't try to do expansion, since the first attempt to get a comprehensive expansion was not successful
125  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String, ValidationResult>>();
126  protected String tsServer;
127  protected String validationCachePath;
128  protected String name;
129
130  // private ValueSetExpansionCache expansionCache; //   
131
132  protected FHIRToolingClient txServer;
133  private Bundle bndCodeSystems;
134  private boolean canRunWithoutTerminology;
135  protected boolean allowLoadingDuplicates;
136  protected boolean noTerminologyServer;
137  protected String cache;
138  private int expandCodesLimit = 1000;
139  protected ILoggingService logger;
140  protected ExpansionProfile expProfile;
141  private Locale locale;
142  private ResourceBundle i18Nmessages;
143
144  public Map<String, CodeSystem> getCodeSystems() {
145    return codeSystems;
146  }
147
148  public Map<String, DataElement> getDataElements() {
149    return dataElements;
150  }
151
152  public Map<String, ValueSet> getValueSets() {
153    return valueSets;
154  }
155
156  public Map<String, ConceptMap> getMaps() {
157    return maps;
158  }
159
160  public Map<String, StructureDefinition> getProfiles() {
161    return profiles;
162  }
163
164  public Map<String, StructureDefinition> getExtensionDefinitions() {
165    return extensionDefinitions;
166  }
167
168  public Map<String, Questionnaire> getQuestionnaires() {
169    return questionnaires;
170  }
171
172  public Map<String, OperationDefinition> getOperations() {
173    return operations;
174  }
175
176  public void seeExtensionDefinition(String url, StructureDefinition ed) throws Exception {
177    if (extensionDefinitions.get(ed.getUrl()) != null) {
178      throw new Exception("duplicate extension definition: " + ed.getUrl());
179    }
180    extensionDefinitions.put(ed.getId(), ed);
181    extensionDefinitions.put(url, ed);
182    extensionDefinitions.put(ed.getUrl(), ed);
183  }
184
185  public void dropExtensionDefinition(String id) {
186    StructureDefinition sd = extensionDefinitions.get(id);
187    extensionDefinitions.remove(id);
188    if (sd != null) {
189      extensionDefinitions.remove(sd.getUrl());
190    }
191  }
192
193  public void seeQuestionnaire(String url, Questionnaire theQuestionnaire) throws Exception {
194    if (questionnaires.get(theQuestionnaire.getId()) != null) {
195      throw new Exception("duplicate extension definition: " + theQuestionnaire.getId());
196    }
197    questionnaires.put(theQuestionnaire.getId(), theQuestionnaire);
198    questionnaires.put(url, theQuestionnaire);
199  }
200
201  public void seeOperation(OperationDefinition opd) throws Exception {
202    if (operations.get(opd.getUrl()) != null) {
203      throw new Exception("duplicate extension definition: " + opd.getUrl());
204    }
205    operations.put(opd.getUrl(), opd);
206    operations.put(opd.getId(), opd);
207  }
208
209  public void seeValueSet(String url, ValueSet vs) throws Exception {
210    if (valueSets.containsKey(vs.getUrl()) && !allowLoadingDuplicates) {
211      throw new Exception("Duplicate value set " + vs.getUrl());
212    }
213    valueSets.put(vs.getId(), vs);
214    valueSets.put(url, vs);
215    valueSets.put(vs.getUrl(), vs);
216  }
217
218  public void dropValueSet(String id) {
219    ValueSet vs = valueSets.get(id);
220    valueSets.remove(id);
221    if (vs != null) {
222      valueSets.remove(vs.getUrl());
223    }
224  }
225
226  public void seeCodeSystem(String url, CodeSystem cs) throws FHIRException {
227    if (codeSystems.containsKey(cs.getUrl()) && !allowLoadingDuplicates) {
228      throw new FHIRException("Duplicate code system " + cs.getUrl());
229    }
230    codeSystems.put(cs.getId(), cs);
231    codeSystems.put(url, cs);
232    codeSystems.put(cs.getUrl(), cs);
233  }
234
235  public void dropCodeSystem(String id) {
236    CodeSystem cs = codeSystems.get(id);
237    codeSystems.remove(id);
238    if (cs != null) {
239      codeSystems.remove(cs.getUrl());
240    }
241  }
242
243  public void seeProfile(String url, StructureDefinition p) throws Exception {
244    if (profiles.containsKey(p.getUrl())) {
245      throw new Exception("Duplicate Profile " + p.getUrl());
246    }
247    profiles.put(p.getId(), p);
248    profiles.put(url, p);
249    profiles.put(p.getUrl(), p);
250  }
251
252  public void dropProfile(String id) {
253    StructureDefinition sd = profiles.get(id);
254    profiles.remove(id);
255    if (sd != null) {
256      profiles.remove(sd.getUrl());
257    }
258  }
259
260  @Override
261  public CodeSystem fetchCodeSystem(String system) {
262    return codeSystems.get(system);
263  }
264
265  @Override
266  public boolean supportsSystem(String system) throws TerminologyServiceException {
267    if (codeSystems.containsKey(system)) {
268      return true;
269    } else if (nonSupportedCodeSystems.contains(system)) {
270      return false;
271    } else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com")
272      || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:")) {
273      return false;
274    } else {
275      if (noTerminologyServer) {
276        return false;
277      }
278      if (bndCodeSystems == null) {
279        try {
280          tlog("Terminology server: Check for supported code systems for " + system);
281          bndCodeSystems = txServer.fetchFeed(txServer.getAddress()
282            + "/CodeSystem?content-mode=not-present&_summary=true&_count=1000");
283        } catch (Exception e) {
284          if (canRunWithoutTerminology) {
285            noTerminologyServer = true;
286            log("==============!! Running without terminology server !!============== (" + e
287              .getMessage() + ")");
288            return false;
289          } else {
290            throw new TerminologyServiceException(e);
291          }
292        }
293      }
294      if (bndCodeSystems != null) {
295        for (BundleEntryComponent be : bndCodeSystems.getEntry()) {
296          CodeSystem cs = (CodeSystem) be.getResource();
297          if (!codeSystems.containsKey(cs.getUrl())) {
298            codeSystems.put(cs.getUrl(), null);
299          }
300        }
301      }
302      if (codeSystems.containsKey(system)) {
303        return true;
304      }
305    }
306    nonSupportedCodeSystems.add(system);
307    return false;
308  }
309
310  private void log(String message) {
311    if (logger != null) {
312      logger.logMessage(message);
313    } else {
314      System.out.println(message);
315    }
316  }
317
318  @Override
319  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
320    try {
321      if (vs.hasExpansion()) {
322        return new ValueSetExpansionOutcome(vs.copy());
323      }
324      String cacheFn = null;
325      if (cache != null) {
326        cacheFn = Utilities.path(cache, determineCacheId(vs, heirarchical) + ".json");
327        if (ManagedFileAccess.file(cacheFn).exists()) {
328          return loadFromCache(vs.copy(), cacheFn);
329        }
330      }
331      if (cacheOk && vs.hasUrl()) {
332        if (expProfile == null) {
333          throw new Exception("No ExpansionProfile provided");
334        }
335        ValueSetExpansionOutcome vse = expansionCache.getExpander()
336          .expand(vs, expProfile.setExcludeNested(!heirarchical));
337        if (vse.getValueset() != null) {
338          if (cache != null) {
339            FileOutputStream s = ManagedFileAccess.outStream(cacheFn);
340            newJsonParser().compose(ManagedFileAccess.outStream(cacheFn), vse.getValueset());
341            s.close();
342          }
343        }
344        return vse;
345      } else {
346        ValueSetExpansionOutcome res = expandOnServer(vs, cacheFn);
347        if (cacheFn != null) {
348          if (res.getValueset() != null) {
349            saveToCache(res.getValueset(), cacheFn);
350          } else {
351            OperationOutcome oo = new OperationOutcome();
352            oo.addIssue().getDetails().setText(res.getError());
353            saveToCache(oo, cacheFn);
354          }
355        }
356        return res;
357      }
358    } catch (NoTerminologyServiceException e) {
359      return new ValueSetExpansionOutcome(
360        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
361        TerminologyServiceErrorClass.NOSERVICE);
362    } catch (Exception e) {
363      return new ValueSetExpansionOutcome(
364        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
365        TerminologyServiceErrorClass.UNKNOWN);
366    }
367  }
368
369  private ValueSetExpansionOutcome loadFromCache(ValueSet vs, String cacheFn)
370    throws FileNotFoundException, Exception {
371    JsonParser parser = new JsonParser();
372    Resource r = parser.parse(ManagedFileAccess.inStream(cacheFn));
373    if (r instanceof OperationOutcome) {
374      return new ValueSetExpansionOutcome(
375        ((OperationOutcome) r).getIssue().get(0).getDetails().getText(),
376        TerminologyServiceErrorClass.UNKNOWN);
377    } else {
378      vs.setExpansion(((ValueSet) r)
379        .getExpansion()); // because what is cached might be from a different value set
380      return new ValueSetExpansionOutcome(vs);
381    }
382  }
383
384  private void saveToCache(Resource res, String cacheFn) throws FileNotFoundException, Exception {
385    JsonParser parser = new JsonParser();
386    parser.compose(ManagedFileAccess.outStream(cacheFn), res);
387  }
388
389  private String determineCacheId(ValueSet vs, boolean heirarchical) throws Exception {
390    // just the content logical definition is hashed
391    ValueSet vsid = new ValueSet();
392    vsid.setCompose(vs.getCompose());
393    JsonParser parser = new JsonParser();
394    parser.setOutputStyle(OutputStyle.NORMAL);
395    ByteArrayOutputStream b = new ByteArrayOutputStream();
396    parser.compose(b, vsid);
397    b.close();
398    String s = new String(b.toByteArray(), Constants.CHARSET_UTF8);
399    // any code systems we can find, we add these too. 
400    for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
401      CodeSystem cs = fetchCodeSystem(inc.getSystem());
402      if (cs != null) {
403        String css = cacheValue(cs);
404        s = s + css;
405      }
406    }
407    s = s + "-" + Boolean.toString(heirarchical);
408    String r = Integer.toString(s.hashCode());
409    //    TextFile.stringToFile(s, Utilities.path(cache, r+".id.json"));
410    return r;
411  }
412
413
414  private String cacheValue(CodeSystem cs) throws IOException {
415    CodeSystem csid = new CodeSystem();
416    csid.setId(cs.getId());
417    csid.setVersion(cs.getVersion());
418    csid.setContent(cs.getContent());
419    csid.setHierarchyMeaning(CodeSystemHierarchyMeaning.GROUPEDBY);
420    for (ConceptDefinitionComponent cc : cs.getConcept()) {
421      csid.getConcept().add(processCSConcept(cc));
422    }
423    JsonParser parser = new JsonParser();
424    parser.setOutputStyle(OutputStyle.NORMAL);
425    ByteArrayOutputStream b = new ByteArrayOutputStream();
426    parser.compose(b, csid);
427    b.close();
428    return new String(b.toByteArray(), Constants.CHARSET_UTF8);
429  }
430
431
432  private ConceptDefinitionComponent processCSConcept(ConceptDefinitionComponent cc) {
433    ConceptDefinitionComponent ccid = new ConceptDefinitionComponent();
434    ccid.setCode(cc.getCode());
435    ccid.setDisplay(cc.getDisplay());
436    for (ConceptDefinitionComponent cci : cc.getConcept()) {
437      ccid.getConcept().add(processCSConcept(cci));
438    }
439    return ccid;
440  }
441
442  public ValueSetExpansionOutcome expandOnServer(ValueSet vs, String fn) throws Exception {
443    if (noTerminologyServer) {
444      return new ValueSetExpansionOutcome(
445        "Error expanding ValueSet: running without terminology services",
446        TerminologyServiceErrorClass.NOSERVICE);
447    }
448    if (expProfile == null) {
449      throw new Exception("No ExpansionProfile provided");
450    }
451
452    try {
453      Parameters params = new Parameters();
454      params.addParameter().setName("profile").setResource(expProfile.setIncludeDefinition(false));
455      params.addParameter().setName("_limit").setValue(new IntegerType(expandCodesLimit));
456      params.addParameter().setName("_incomplete").setValue(new BooleanType("true"));
457      tlog("Terminology Server: $expand on " + getVSSummary(vs));
458      ValueSet result = txServer.expandValueset(vs, params);
459      return new ValueSetExpansionOutcome(result);
460    } catch (Exception e) {
461      return new ValueSetExpansionOutcome(
462        "Error expanding ValueSet \"" + vs.getUrl() + ": " + e.getMessage(),
463        TerminologyServiceErrorClass.UNKNOWN);
464    }
465  }
466
467  private String getVSSummary(ValueSet vs) {
468    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
469    for (ConceptSetComponent cc : vs.getCompose().getInclude()) {
470      b.append("Include " + getIncSummary(cc));
471    }
472    for (ConceptSetComponent cc : vs.getCompose().getExclude()) {
473      b.append("Exclude " + getIncSummary(cc));
474    }
475    return b.toString();
476  }
477
478  private String getIncSummary(ConceptSetComponent cc) {
479    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
480    for (UriType vs : cc.getValueSet()) {
481      b.append(vs.asStringValue());
482    }
483    String vsd =
484      b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
485    String system = cc.getSystem();
486    if (cc.hasConcept()) {
487      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
488    }
489    if (cc.hasFilter()) {
490      String s = "";
491      for (ConceptSetFilterComponent f : cc.getFilter()) {
492        if (!Utilities.noString(s)) {
493          s = s + " & ";
494        }
495        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
496      }
497      return "from " + system + " where " + s + vsd;
498    }
499    return "All codes from " + system + vsd;
500  }
501
502  private ValidationResult handleByCache(ValueSet vs, Coding coding, boolean tryCache) {
503    String cacheId = cacheId(coding);
504    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
505    if (cache == null) {
506      cache = new HashMap<String, IWorkerContext.ValidationResult>();
507      validationCache.put(vs.getUrl(), cache);
508    }
509    if (cache.containsKey(cacheId)) {
510      return cache.get(cacheId);
511    }
512    if (!tryCache) {
513      return null;
514    }
515    if (!cacheValidation) {
516      return null;
517    }
518    if (failed.contains(vs.getUrl())) {
519      return null;
520    }
521    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
522    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
523      failed.add(vs.getUrl());
524      return null;
525    }
526
527    ValidationResult res = validateCode(coding, vse.getValueset());
528    cache.put(cacheId, res);
529    return res;
530  }
531
532  private boolean notcomplete(ValueSet vs) {
533    if (!vs.hasExpansion()) {
534      return true;
535    }
536    if (!vs.getExpansion()
537      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-unclosed").isEmpty()) {
538      return true;
539    }
540    if (!vs.getExpansion()
541      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-toocostly").isEmpty()) {
542      return true;
543    }
544    return false;
545  }
546
547  private ValidationResult handleByCache(ValueSet vs, CodeableConcept concept, boolean tryCache) {
548    String cacheId = cacheId(concept);
549    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
550    if (cache == null) {
551      cache = new HashMap<String, IWorkerContext.ValidationResult>();
552      validationCache.put(vs.getUrl(), cache);
553    }
554    if (cache.containsKey(cacheId)) {
555      return cache.get(cacheId);
556    }
557
558    if (validationCache.containsKey(vs.getUrl()) && validationCache.get(vs.getUrl())
559      .containsKey(cacheId)) {
560      return validationCache.get(vs.getUrl()).get(cacheId);
561    }
562    if (!tryCache) {
563      return null;
564    }
565    if (!cacheValidation) {
566      return null;
567    }
568    if (failed.contains(vs.getUrl())) {
569      return null;
570    }
571    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
572    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
573      failed.add(vs.getUrl());
574      return null;
575    }
576    ValidationResult res = validateCode(concept, vse.getValueset());
577    cache.put(cacheId, res);
578    return res;
579  }
580
581  private String cacheId(Coding coding) {
582    return "|" + coding.getSystem() + "|" + coding.getVersion() + "|" + coding.getCode() + "|"
583      + coding.getDisplay();
584  }
585
586  private String cacheId(CodeableConcept cc) {
587    StringBuilder b = new StringBuilder();
588    for (Coding c : cc.getCoding()) {
589      b.append("#");
590      b.append(cacheId(c));
591    }
592    return b.toString();
593  }
594
595  private ValidationResult verifyCodeExternal(ValueSet vs, Coding coding, boolean tryCache)
596    throws Exception {
597    ValidationResult res = vs == null ? null : handleByCache(vs, coding, tryCache);
598    if (res != null) {
599      return res;
600    }
601    Parameters pin = new Parameters();
602    pin.addParameter().setName("coding").setValue(coding);
603    if (vs != null) {
604      pin.addParameter().setName("valueSet").setResource(vs);
605    }
606    res = serverValidateCode(pin, vs == null);
607    if (vs != null) {
608      Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
609      cache.put(cacheId(coding), res);
610    }
611    return res;
612  }
613
614  private ValidationResult verifyCodeExternal(ValueSet vs, CodeableConcept cc, boolean tryCache)
615    throws Exception {
616    ValidationResult res = handleByCache(vs, cc, tryCache);
617    if (res != null) {
618      return res;
619    }
620    Parameters pin = new Parameters();
621    pin.addParameter().setName("codeableConcept").setValue(cc);
622    pin.addParameter().setName("valueSet").setResource(vs);
623    res = serverValidateCode(pin, false);
624    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
625    cache.put(cacheId(cc), res);
626    return res;
627  }
628
629  private ValidationResult serverValidateCode(Parameters pin, boolean doCache) throws Exception {
630    if (noTerminologyServer) {
631      return new ValidationResult(null, null, TerminologyServiceErrorClass.NOSERVICE);
632    }
633    String cacheName = doCache ? generateCacheName(pin) : null;
634    ValidationResult res = loadFromCache(cacheName);
635    if (res != null) {
636      return res;
637    }
638    tlog("Terminology Server: $validate-code " + describeValidationParameters(pin));
639    for (ParametersParameterComponent pp : pin.getParameter()) {
640      if (pp.getName().equals("profile")) {
641        throw new Error("Can only specify profile in the context");
642      }
643    }
644    if (expProfile == null) {
645      throw new Exception("No ExpansionProfile provided");
646    }
647    pin.addParameter().setName("profile").setResource(expProfile);
648
649    Parameters pout = txServer.operateType(ValueSet.class, "validate-code", pin);
650    boolean ok = false;
651    String message = "No Message returned";
652    String display = null;
653    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
654    for (ParametersParameterComponent p : pout.getParameter()) {
655      if (p.getName().equals("result")) {
656        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
657      } else if (p.getName().equals("message")) {
658        message = ((StringType) p.getValue()).getValue();
659      } else if (p.getName().equals("display")) {
660        display = ((StringType) p.getValue()).getValue();
661      } else if (p.getName().equals("cause")) {
662        try {
663          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
664          if (it == IssueType.UNKNOWN) {
665            err = TerminologyServiceErrorClass.UNKNOWN;
666          } else if (it == IssueType.NOTSUPPORTED) {
667            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
668          }
669        } catch (FHIRException e) {
670        }
671      }
672    }
673    if (!ok) {
674      res = new ValidationResult(IssueSeverity.ERROR, message, err);
675    } else if (display != null) {
676      res = new ValidationResult(new ConceptDefinitionComponent().setDisplay(display));
677    } else {
678      res = new ValidationResult(new ConceptDefinitionComponent());
679    }
680    saveToCache(res, cacheName);
681    return res;
682  }
683
684
685  private void tlog(String msg) {
686    //    log(msg);
687  }
688
689  @SuppressWarnings("rawtypes")
690  private String describeValidationParameters(Parameters pin) {
691    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
692    for (ParametersParameterComponent p : pin.getParameter()) {
693      if (p.hasValue() && p.getValue() instanceof PrimitiveType) {
694        b.append(p.getName() + "=" + ((PrimitiveType) p.getValue()).asStringValue());
695      } else if (p.hasValue() && p.getValue() instanceof Coding) {
696        b.append("system=" + ((Coding) p.getValue()).getSystem());
697        b.append("code=" + ((Coding) p.getValue()).getCode());
698        b.append("display=" + ((Coding) p.getValue()).getDisplay());
699      } else if (p.hasValue() && p.getValue() instanceof CodeableConcept) {
700        if (((CodeableConcept) p.getValue()).hasCoding()) {
701          Coding c = ((CodeableConcept) p.getValue()).getCodingFirstRep();
702          b.append("system=" + c.getSystem());
703          b.append("code=" + c.getCode());
704          b.append("display=" + c.getDisplay());
705        } else if (((CodeableConcept) p.getValue()).hasText()) {
706          b.append("text=" + ((CodeableConcept) p.getValue()).getText());
707        }
708      } else if (p.hasResource() && (p.getResource() instanceof ValueSet)) {
709        b.append("valueset=" + getVSSummary((ValueSet) p.getResource()));
710      }
711    }
712    return b.toString();
713  }
714
715  private ValidationResult loadFromCache(String fn) throws FileNotFoundException, IOException {
716    if (fn == null) {
717      return null;
718    }
719    if (!(ManagedFileAccess.file(fn).exists())) {
720      return null;
721    }
722    String cnt = TextFile.fileToString(fn);
723    if (cnt.startsWith("!error: ")) {
724      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(8));
725    } else if (cnt.startsWith("!warning: ")) {
726      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(10));
727    } else {
728      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(cnt));
729    }
730  }
731
732  private void saveToCache(ValidationResult res, String cacheName) throws IOException {
733    if (cacheName == null) {
734      return;
735    }
736    if (res.getDisplay() != null) {
737      TextFile.stringToFile(res.getDisplay(), cacheName);
738    } else if (res.getMessage() != null) {
739      if (res.getSeverity() == IssueSeverity.WARNING) {
740        TextFile.stringToFile("!warning: " + res.getMessage(), cacheName);
741      } else {
742        TextFile.stringToFile("!error: " + res.getMessage(), cacheName);
743      }
744    }
745  }
746
747  private String generateCacheName(Parameters pin) throws IOException {
748    if (cache == null) {
749      return null;
750    }
751    String json = new JsonParser().composeString(pin);
752    return Utilities.path(cache, "vc" + Integer.toString(json.hashCode()) + ".json");
753  }
754
755  @Override
756  public ValueSetExpansionComponent expandVS(ConceptSetComponent inc, boolean heirachical)
757    throws TerminologyServiceException {
758    ValueSet vs = new ValueSet();
759    vs.setCompose(new ValueSetComposeComponent());
760    vs.getCompose().getInclude().add(inc);
761    ValueSetExpansionOutcome vse = expandVS(vs, true, heirachical);
762    ValueSet valueset = vse.getValueset();
763    if (valueset == null) {
764      throw new TerminologyServiceException("Error Expanding ValueSet: " + vse.getError());
765    }
766    return valueset.getExpansion();
767  }
768
769  @Override
770  public ValidationResult validateCode(String system, String code, String display) {
771    try {
772      if (codeSystems.containsKey(system) && codeSystems.get(system) != null) {
773        return verifyCodeInCodeSystem(codeSystems.get(system), system, code, display);
774      } else {
775        return verifyCodeExternal(null,
776          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
777      }
778    } catch (Exception e) {
779      return new ValidationResult(IssueSeverity.FATAL,
780        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
781    }
782  }
783
784
785  @Override
786  public ValidationResult validateCode(Coding code, ValueSet vs) {
787    if (codeSystems.containsKey(code.getSystem()) && codeSystems.get(code.getSystem()) != null) {
788      try {
789        return verifyCodeInCodeSystem(codeSystems.get(code.getSystem()), code.getSystem(),
790          code.getCode(), code.getDisplay());
791      } catch (Exception e) {
792        return new ValidationResult(IssueSeverity.FATAL,
793          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
794            .getMessage());
795      }
796    } else if (vs.hasExpansion()) {
797      try {
798        return verifyCodeInternal(vs, code.getSystem(), code.getCode(), code.getDisplay());
799      } catch (Exception e) {
800        return new ValidationResult(IssueSeverity.FATAL,
801          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
802            .getMessage());
803      }
804    } else {
805      try {
806        return verifyCodeExternal(vs, code, true);
807      } catch (Exception e) {
808        return new ValidationResult(IssueSeverity.WARNING,
809          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
810            .getMessage());
811      }
812    }
813  }
814
815  @Override
816  public ValidationResult validateCode(CodeableConcept code, ValueSet vs) {
817    try {
818      if (vs.hasExpansion()) {
819        return verifyCodeInternal(vs, code);
820      } else {
821        // we'll try expanding first; if that doesn't work, then we'll just pass it to the server to validate 
822        // ... could be a problem if the server doesn't have the code systems we have locally, so we try not to depend on the server
823        try {
824          ValueSetExpansionOutcome vse = expandVS(vs, true, false);
825          if (vse.getValueset() != null && !hasTooCostlyExpansion(vse.getValueset())) {
826            return verifyCodeInternal(vse.getValueset(), code);
827          }
828        } catch (Exception e) {
829          // failed? we'll just try the server
830        }
831        return verifyCodeExternal(vs, code, true);
832      }
833    } catch (Exception e) {
834      return new ValidationResult(IssueSeverity.FATAL,
835        "Error validating code \"" + code.toString() + "\": " + e.getMessage(),
836        TerminologyServiceErrorClass.SERVER_ERROR);
837    }
838  }
839
840
841  private boolean hasTooCostlyExpansion(ValueSet valueset) {
842    return valueset != null && valueset.hasExpansion() && ToolingExtensions
843      .hasExtension(valueset.getExpansion(),
844        "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
845  }
846
847  @Override
848  public ValidationResult validateCode(String system, String code, String display, ValueSet vs) {
849    try {
850      if (system == null && display == null) {
851        return verifyCodeInternal(vs, code);
852      }
853      if ((codeSystems.containsKey(system) && codeSystems.get(system) != null) || vs
854        .hasExpansion()) {
855        return verifyCodeInternal(vs, system, code, display);
856      } else {
857        return verifyCodeExternal(vs,
858          new Coding().setSystem(system).setCode(code).setDisplay(display), true);
859      }
860    } catch (Exception e) {
861      return new ValidationResult(IssueSeverity.FATAL,
862        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage(),
863        TerminologyServiceErrorClass.SERVER_ERROR);
864    }
865  }
866
867  @Override
868  public ValidationResult validateCode(String system, String code, String display,
869    ConceptSetComponent vsi) {
870    try {
871      ValueSet vs = new ValueSet();
872      vs.setUrl(Utilities.makeUuidUrn());
873      vs.getCompose().addInclude(vsi);
874      return verifyCodeExternal(vs,
875        new Coding().setSystem(system).setCode(code).setDisplay(display), true);
876    } catch (Exception e) {
877      return new ValidationResult(IssueSeverity.FATAL,
878        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
879    }
880  }
881
882  public void initTS(String cachePath, String tsServer) throws Exception {
883    cache = cachePath;
884    this.tsServer = tsServer;
885    expansionCache = new ValueSetExpansionCache(this, null);
886    validationCachePath = Utilities.path(cachePath, "validation.cache");
887    try {
888      loadValidationCache();
889    } catch (Exception e) {
890      e.printStackTrace();
891    }
892  }
893
894  protected void loadValidationCache() throws JsonSyntaxException, Exception {
895  }
896
897  @Override
898  public List<ConceptMap> findMapsForSource(String url) {
899    List<ConceptMap> res = new ArrayList<ConceptMap>();
900    for (ConceptMap map : maps.values()) {
901      if (((Reference) map.getSource()).getReference().equals(url)) {
902        res.add(map);
903      }
904    }
905    return res;
906  }
907
908  private ValidationResult verifyCodeInternal(ValueSet vs, CodeableConcept code) throws Exception {
909    for (Coding c : code.getCoding()) {
910      ValidationResult res = verifyCodeInternal(vs, c.getSystem(), c.getCode(), c.getDisplay());
911      if (res.isOk()) {
912        return res;
913      }
914    }
915    if (code.getCoding().isEmpty()) {
916      return new ValidationResult(IssueSeverity.ERROR, "None code provided");
917    } else {
918      return new ValidationResult(IssueSeverity.ERROR,
919        "None of the codes are in the specified value set");
920    }
921  }
922
923  private ValidationResult verifyCodeInternal(ValueSet vs, String system, String code,
924    String display) throws Exception {
925    if (vs.hasExpansion()) {
926      return verifyCodeInExpansion(vs, system, code, display);
927    } else {
928      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
929      if (vse.getValueset() != null) {
930        return verifyCodeExternal(vs,
931          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
932      } else {
933        return verifyCodeInExpansion(vse.getValueset(), system, code, display);
934      }
935    }
936  }
937
938  private ValidationResult verifyCodeInternal(ValueSet vs, String code)
939    throws FileNotFoundException, ETooCostly, IOException, FHIRException {
940    if (vs.hasExpansion()) {
941      return verifyCodeInExpansion(vs, code);
942    } else {
943      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
944      if (vse.getValueset() == null) {
945        return new ValidationResult(IssueSeverity.ERROR, vse.getError(), vse.getErrorClass());
946      } else {
947        return verifyCodeInExpansion(vse.getValueset(), code);
948      }
949    }
950  }
951
952  private ValidationResult verifyCodeInCodeSystem(CodeSystem cs, String system, String code,
953    String display) throws Exception {
954    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code);
955    if (cc == null) {
956      if (cs.getContent().equals(CodeSystem.CodeSystemContentMode.COMPLETE)) {
957        return new ValidationResult(IssueSeverity.ERROR,
958          "Unknown Code " + code + " in " + cs.getUrl());
959      } else if (!cs.getContent().equals(CodeSystem.CodeSystemContentMode.NOTPRESENT)) {
960        return new ValidationResult(IssueSeverity.WARNING,
961          "Unknown Code " + code + " in partial code list of " + cs.getUrl());
962      } else {
963        return verifyCodeExternal(null,
964          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
965      }
966    }
967    //
968    //        return new ValidationResult(IssueSeverity.WARNING, "A definition was found for "+cs.getUrl()+", but it has no codes in the definition");
969    //      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
970    if (display == null) {
971      return new ValidationResult(cc);
972    }
973    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
974    if (cc.hasDisplay()) {
975      b.append(cc.getDisplay());
976      if (display.equalsIgnoreCase(cc.getDisplay())) {
977        return new ValidationResult(cc);
978      }
979    }
980    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
981      b.append(ds.getValue());
982      if (display.equalsIgnoreCase(ds.getValue())) {
983        return new ValidationResult(cc);
984      }
985    }
986    return new ValidationResult(IssueSeverity.WARNING,
987      "Display Name for " + code + " must be one of '" + b.toString() + "'", cc);
988  }
989
990
991  private ValidationResult verifyCodeInExpansion(ValueSet vs, String system, String code,
992    String display) {
993    ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
994    if (cc == null) {
995      return new ValidationResult(IssueSeverity.ERROR,
996        "Unknown Code " + code + " in " + vs.getUrl());
997    }
998    if (display == null) {
999      return new ValidationResult(
1000        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1001    }
1002    if (cc.hasDisplay()) {
1003      if (display.equalsIgnoreCase(cc.getDisplay())) {
1004        return new ValidationResult(
1005          new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1006      }
1007      return new ValidationResult(IssueSeverity.WARNING,
1008        "Display Name for " + code + " must be '" + cc.getDisplay() + "'",
1009        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1010    }
1011    return null;
1012  }
1013
1014  private ValidationResult verifyCodeInExpansion(ValueSet vs, String code) throws FHIRException {
1015    if (vs.getExpansion()
1016      .hasExtension("http://hl7.org/fhir/StructureDefinition/valueset-toocostly")) {
1017      throw new FHIRException("Unable to validate core - value set is too costly to expand");
1018    } else {
1019      ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
1020      if (cc == null) {
1021        return new ValidationResult(IssueSeverity.ERROR,
1022          "Unknown Code " + code + " in " + vs.getUrl());
1023      }
1024      return null;
1025    }
1026  }
1027
1028  private ValueSetExpansionContainsComponent findCode(
1029    List<ValueSetExpansionContainsComponent> contains, String code) {
1030    for (ValueSetExpansionContainsComponent cc : contains) {
1031      if (code.equals(cc.getCode())) {
1032        return cc;
1033      }
1034      ValueSetExpansionContainsComponent c = findCode(cc.getContains(), code);
1035      if (c != null) {
1036        return c;
1037      }
1038    }
1039    return null;
1040  }
1041
1042  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept,
1043    String code) {
1044    for (ConceptDefinitionComponent cc : concept) {
1045      if (code.equals(cc.getCode())) {
1046        return cc;
1047      }
1048      ConceptDefinitionComponent c = findCodeInConcept(cc.getConcept(), code);
1049      if (c != null) {
1050        return c;
1051      }
1052    }
1053    return null;
1054  }
1055
1056  public Set<String> getNonSupportedCodeSystems() {
1057    return nonSupportedCodeSystems;
1058  }
1059
1060  public boolean isCanRunWithoutTerminology() {
1061    return canRunWithoutTerminology;
1062  }
1063
1064  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
1065    this.canRunWithoutTerminology = canRunWithoutTerminology;
1066  }
1067
1068  public int getExpandCodesLimit() {
1069    return expandCodesLimit;
1070  }
1071
1072  public void setExpandCodesLimit(int expandCodesLimit) {
1073    this.expandCodesLimit = expandCodesLimit;
1074  }
1075
1076  public void setLogger(ILoggingService logger) {
1077    this.logger = logger;
1078  }
1079
1080  public ExpansionProfile getExpansionProfile() {
1081    return expProfile;
1082  }
1083
1084  public void setExpansionProfile(ExpansionProfile expProfile) {
1085    this.expProfile = expProfile;
1086  }
1087
1088  @Override
1089  public boolean isNoTerminologyServer() {
1090    return noTerminologyServer;
1091  }
1092
1093  public String getName() {
1094    return name;
1095  }
1096
1097  public void setName(String name) {
1098    this.name = name;
1099  }
1100
1101  @Override
1102  public Set<String> getResourceNamesAsSet() {
1103    Set<String> res = new HashSet<String>();
1104    res.addAll(getResourceNames());
1105    return res;
1106  }
1107
1108  public void reportStatus(JsonObject json) {
1109    json.addProperty("codeystem-count", codeSystems.size());
1110    json.addProperty("valueset-count", valueSets.size());
1111    json.addProperty("conceptmap-count", maps.size());
1112    json.addProperty("transforms-count", transforms.size());
1113    json.addProperty("structures-count", profiles.size());
1114  }
1115
1116  public void cacheResource(Resource r) throws Exception {
1117    if (r instanceof ValueSet) {
1118      seeValueSet(((ValueSet) r).getUrl(), (ValueSet) r);
1119    } else if (r instanceof CodeSystem) {
1120      seeCodeSystem(((CodeSystem) r).getUrl(), (CodeSystem) r);
1121    } else if (r instanceof StructureDefinition) {
1122      StructureDefinition sd = (StructureDefinition) r;
1123      if ("http://hl7.org/fhir/StructureDefinition/Extension".equals(sd.getBaseDefinition())) {
1124        seeExtensionDefinition(sd.getUrl(), sd);
1125      } else if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) {
1126        seeProfile(sd.getUrl(), sd);
1127      }
1128    }
1129  }
1130
1131  public void dropResource(String type, String id) throws FHIRException {
1132    if (type.equals("ValueSet")) {
1133      dropValueSet(id);
1134    }
1135    if (type.equals("CodeSystem")) {
1136      dropCodeSystem(id);
1137    }
1138    if (type.equals("StructureDefinition")) {
1139      dropProfile(id);
1140      dropExtensionDefinition(id);
1141    }
1142  }
1143
1144  public boolean isAllowLoadingDuplicates() {
1145    return allowLoadingDuplicates;
1146  }
1147
1148  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
1149    this.allowLoadingDuplicates = allowLoadingDuplicates;
1150  }
1151
1152  @Override
1153  public StructureDefinition fetchTypeDefinition(String typeName) {
1154    return fetchResource(StructureDefinition.class,
1155      "http://hl7.org/fhir/StructureDefinition/" + typeName);
1156  }
1157}