001package org.hl7.fhir.dstu3.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.ByteArrayOutputStream;
035import java.io.File;
036import java.io.FileInputStream;
037import java.io.FileNotFoundException;
038import java.io.FileOutputStream;
039import java.io.IOException;
040import java.util.ArrayList;
041import java.util.HashMap;
042import java.util.HashSet;
043import java.util.List;
044import java.util.Locale;
045import java.util.Map;
046import java.util.ResourceBundle;
047import java.util.Set;
048
049import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
050import org.hl7.fhir.dstu3.formats.JsonParser;
051import org.hl7.fhir.dstu3.model.BooleanType;
052import org.hl7.fhir.dstu3.model.Bundle;
053import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
054import org.hl7.fhir.dstu3.model.CodeSystem;
055import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemHierarchyMeaning;
056import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
057import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
058import org.hl7.fhir.dstu3.model.CodeableConcept;
059import org.hl7.fhir.dstu3.model.Coding;
060import org.hl7.fhir.dstu3.model.ConceptMap;
061import org.hl7.fhir.dstu3.model.DataElement;
062import org.hl7.fhir.dstu3.model.ExpansionProfile;
063import org.hl7.fhir.dstu3.model.OperationDefinition;
064import org.hl7.fhir.dstu3.model.OperationOutcome;
065import org.hl7.fhir.dstu3.model.Parameters;
066import org.hl7.fhir.dstu3.model.Parameters.ParametersParameterComponent;
067import org.hl7.fhir.dstu3.model.PrimitiveType;
068import org.hl7.fhir.dstu3.model.Questionnaire;
069import org.hl7.fhir.dstu3.model.Reference;
070import org.hl7.fhir.dstu3.model.Resource;
071import org.hl7.fhir.dstu3.model.SearchParameter;
072import org.hl7.fhir.dstu3.model.StringType;
073import org.hl7.fhir.dstu3.model.StructureDefinition;
074import org.hl7.fhir.dstu3.model.StructureDefinition.TypeDerivationRule;
075import org.hl7.fhir.dstu3.model.StructureMap;
076import org.hl7.fhir.dstu3.model.UriType;
077import org.hl7.fhir.dstu3.model.ValueSet;
078import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
079import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
080import org.hl7.fhir.dstu3.model.ValueSet.ValueSetComposeComponent;
081import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
082import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
083import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ETooCostly;
084import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
085import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
086import org.hl7.fhir.dstu3.terminologies.ValueSetExpanderFactory;
087import org.hl7.fhir.dstu3.terminologies.ValueSetExpansionCache;
088import org.hl7.fhir.dstu3.utils.ToolingExtensions;
089import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
090import org.hl7.fhir.exceptions.FHIRException;
091import org.hl7.fhir.exceptions.NoTerminologyServiceException;
092import org.hl7.fhir.exceptions.TerminologyServiceException;
093import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
094import org.hl7.fhir.utilities.TextFile;
095import org.hl7.fhir.utilities.Utilities;
096import org.hl7.fhir.utilities.i18n.I18nBase;
097import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
098import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
099
100import com.google.gson.JsonObject;
101import com.google.gson.JsonSyntaxException;
102
103import ca.uhn.fhir.rest.api.Constants;
104
105public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext {
106
107  // all maps are to the full URI
108  protected Map<String, CodeSystem> codeSystems = new HashMap<String, CodeSystem>();
109  protected Set<String> nonSupportedCodeSystems = new HashSet<String>();
110  protected Map<String, ValueSet> valueSets = new HashMap<String, ValueSet>();
111  protected Map<String, ConceptMap> maps = new HashMap<String, ConceptMap>();
112  protected Map<String, StructureMap> transforms = new HashMap<String, StructureMap>();
113  protected Map<String, DataElement> dataElements = new HashMap<String, DataElement>();
114  protected Map<String, StructureDefinition> profiles = new HashMap<String, StructureDefinition>();
115  protected Map<String, SearchParameter> searchParameters = new HashMap<String, SearchParameter>();
116  protected Map<String, StructureDefinition> extensionDefinitions = new HashMap<String, StructureDefinition>();
117  protected Map<String, Questionnaire> questionnaires = new HashMap<String, Questionnaire>();
118  protected Map<String, OperationDefinition> operations = new HashMap<String, OperationDefinition>();
119
120  protected ValueSetExpanderFactory expansionCache = new ValueSetExpansionCache(this);
121  protected boolean cacheValidation; // if true, do an expansion and cache the expansion
122  private Set<String> failed = new HashSet<String>(); // value sets for which we don't try to do expansion, since the first attempt to get a comprehensive expansion was not successful
123  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String, ValidationResult>>();
124  protected String tsServer;
125  protected String validationCachePath;
126  protected String name;
127
128  // private ValueSetExpansionCache expansionCache; //   
129
130  protected FHIRToolingClient txServer;
131  private Bundle bndCodeSystems;
132  private boolean canRunWithoutTerminology;
133  protected boolean allowLoadingDuplicates;
134  protected boolean noTerminologyServer;
135  protected String cache;
136  private int expandCodesLimit = 1000;
137  protected ILoggingService logger;
138  protected ExpansionProfile expProfile;
139  private Locale locale;
140  private ResourceBundle i18Nmessages;
141
142  public Map<String, CodeSystem> getCodeSystems() {
143    return codeSystems;
144  }
145
146  public Map<String, DataElement> getDataElements() {
147    return dataElements;
148  }
149
150  public Map<String, ValueSet> getValueSets() {
151    return valueSets;
152  }
153
154  public Map<String, ConceptMap> getMaps() {
155    return maps;
156  }
157
158  public Map<String, StructureDefinition> getProfiles() {
159    return profiles;
160  }
161
162  public Map<String, StructureDefinition> getExtensionDefinitions() {
163    return extensionDefinitions;
164  }
165
166  public Map<String, Questionnaire> getQuestionnaires() {
167    return questionnaires;
168  }
169
170  public Map<String, OperationDefinition> getOperations() {
171    return operations;
172  }
173
174  public void seeExtensionDefinition(String url, StructureDefinition ed) throws Exception {
175    if (extensionDefinitions.get(ed.getUrl()) != null) {
176      throw new Exception("duplicate extension definition: " + ed.getUrl());
177    }
178    extensionDefinitions.put(ed.getId(), ed);
179    extensionDefinitions.put(url, ed);
180    extensionDefinitions.put(ed.getUrl(), ed);
181  }
182
183  public void dropExtensionDefinition(String id) {
184    StructureDefinition sd = extensionDefinitions.get(id);
185    extensionDefinitions.remove(id);
186    if (sd != null) {
187      extensionDefinitions.remove(sd.getUrl());
188    }
189  }
190
191  public void seeQuestionnaire(String url, Questionnaire theQuestionnaire) throws Exception {
192    if (questionnaires.get(theQuestionnaire.getId()) != null) {
193      throw new Exception("duplicate extension definition: " + theQuestionnaire.getId());
194    }
195    questionnaires.put(theQuestionnaire.getId(), theQuestionnaire);
196    questionnaires.put(url, theQuestionnaire);
197  }
198
199  public void seeOperation(OperationDefinition opd) throws Exception {
200    if (operations.get(opd.getUrl()) != null) {
201      throw new Exception("duplicate extension definition: " + opd.getUrl());
202    }
203    operations.put(opd.getUrl(), opd);
204    operations.put(opd.getId(), opd);
205  }
206
207  public void seeValueSet(String url, ValueSet vs) throws Exception {
208    if (valueSets.containsKey(vs.getUrl()) && !allowLoadingDuplicates) {
209      throw new Exception("Duplicate value set " + vs.getUrl());
210    }
211    valueSets.put(vs.getId(), vs);
212    valueSets.put(url, vs);
213    valueSets.put(vs.getUrl(), vs);
214  }
215
216  public void dropValueSet(String id) {
217    ValueSet vs = valueSets.get(id);
218    valueSets.remove(id);
219    if (vs != null) {
220      valueSets.remove(vs.getUrl());
221    }
222  }
223
224  public void seeCodeSystem(String url, CodeSystem cs) throws FHIRException {
225    if (codeSystems.containsKey(cs.getUrl()) && !allowLoadingDuplicates) {
226      throw new FHIRException("Duplicate code system " + cs.getUrl());
227    }
228    codeSystems.put(cs.getId(), cs);
229    codeSystems.put(url, cs);
230    codeSystems.put(cs.getUrl(), cs);
231  }
232
233  public void dropCodeSystem(String id) {
234    CodeSystem cs = codeSystems.get(id);
235    codeSystems.remove(id);
236    if (cs != null) {
237      codeSystems.remove(cs.getUrl());
238    }
239  }
240
241  public void seeProfile(String url, StructureDefinition p) throws Exception {
242    if (profiles.containsKey(p.getUrl())) {
243      throw new Exception("Duplicate Profile " + p.getUrl());
244    }
245    profiles.put(p.getId(), p);
246    profiles.put(url, p);
247    profiles.put(p.getUrl(), p);
248  }
249
250  public void dropProfile(String id) {
251    StructureDefinition sd = profiles.get(id);
252    profiles.remove(id);
253    if (sd != null) {
254      profiles.remove(sd.getUrl());
255    }
256  }
257
258  @Override
259  public CodeSystem fetchCodeSystem(String system) {
260    return codeSystems.get(system);
261  }
262
263  @Override
264  public boolean supportsSystem(String system) throws TerminologyServiceException {
265    if (codeSystems.containsKey(system)) {
266      return true;
267    } else if (nonSupportedCodeSystems.contains(system)) {
268      return false;
269    } else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com")
270      || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:")) {
271      return false;
272    } else {
273      if (noTerminologyServer) {
274        return false;
275      }
276      if (bndCodeSystems == null) {
277        try {
278          tlog("Terminology server: Check for supported code systems for " + system);
279          bndCodeSystems = txServer.fetchFeed(txServer.getAddress()
280            + "/CodeSystem?content-mode=not-present&_summary=true&_count=1000");
281        } catch (Exception e) {
282          if (canRunWithoutTerminology) {
283            noTerminologyServer = true;
284            log("==============!! Running without terminology server !!============== (" + e
285              .getMessage() + ")");
286            return false;
287          } else {
288            throw new TerminologyServiceException(e);
289          }
290        }
291      }
292      if (bndCodeSystems != null) {
293        for (BundleEntryComponent be : bndCodeSystems.getEntry()) {
294          CodeSystem cs = (CodeSystem) be.getResource();
295          if (!codeSystems.containsKey(cs.getUrl())) {
296            codeSystems.put(cs.getUrl(), null);
297          }
298        }
299      }
300      if (codeSystems.containsKey(system)) {
301        return true;
302      }
303    }
304    nonSupportedCodeSystems.add(system);
305    return false;
306  }
307
308  private void log(String message) {
309    if (logger != null) {
310      logger.logMessage(message);
311    } else {
312      System.out.println(message);
313    }
314  }
315
316  @Override
317  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
318    try {
319      if (vs.hasExpansion()) {
320        return new ValueSetExpansionOutcome(vs.copy());
321      }
322      String cacheFn = null;
323      if (cache != null) {
324        cacheFn = Utilities.path(cache, determineCacheId(vs, heirarchical) + ".json");
325        if (new File(cacheFn).exists()) {
326          return loadFromCache(vs.copy(), cacheFn);
327        }
328      }
329      if (cacheOk && vs.hasUrl()) {
330        if (expProfile == null) {
331          throw new Exception("No ExpansionProfile provided");
332        }
333        ValueSetExpansionOutcome vse = expansionCache.getExpander()
334          .expand(vs, expProfile.setExcludeNested(!heirarchical));
335        if (vse.getValueset() != null) {
336          if (cache != null) {
337            FileOutputStream s = new FileOutputStream(cacheFn);
338            newJsonParser().compose(new FileOutputStream(cacheFn), vse.getValueset());
339            s.close();
340          }
341        }
342        return vse;
343      } else {
344        ValueSetExpansionOutcome res = expandOnServer(vs, cacheFn);
345        if (cacheFn != null) {
346          if (res.getValueset() != null) {
347            saveToCache(res.getValueset(), cacheFn);
348          } else {
349            OperationOutcome oo = new OperationOutcome();
350            oo.addIssue().getDetails().setText(res.getError());
351            saveToCache(oo, cacheFn);
352          }
353        }
354        return res;
355      }
356    } catch (NoTerminologyServiceException e) {
357      return new ValueSetExpansionOutcome(
358        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
359        TerminologyServiceErrorClass.NOSERVICE);
360    } catch (Exception e) {
361      return new ValueSetExpansionOutcome(
362        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
363        TerminologyServiceErrorClass.UNKNOWN);
364    }
365  }
366
367  private ValueSetExpansionOutcome loadFromCache(ValueSet vs, String cacheFn)
368    throws FileNotFoundException, Exception {
369    JsonParser parser = new JsonParser();
370    Resource r = parser.parse(new FileInputStream(cacheFn));
371    if (r instanceof OperationOutcome) {
372      return new ValueSetExpansionOutcome(
373        ((OperationOutcome) r).getIssue().get(0).getDetails().getText(),
374        TerminologyServiceErrorClass.UNKNOWN);
375    } else {
376      vs.setExpansion(((ValueSet) r)
377        .getExpansion()); // because what is cached might be from a different value set
378      return new ValueSetExpansionOutcome(vs);
379    }
380  }
381
382  private void saveToCache(Resource res, String cacheFn) throws FileNotFoundException, Exception {
383    JsonParser parser = new JsonParser();
384    parser.compose(new FileOutputStream(cacheFn), res);
385  }
386
387  private String determineCacheId(ValueSet vs, boolean heirarchical) throws Exception {
388    // just the content logical definition is hashed
389    ValueSet vsid = new ValueSet();
390    vsid.setCompose(vs.getCompose());
391    JsonParser parser = new JsonParser();
392    parser.setOutputStyle(OutputStyle.NORMAL);
393    ByteArrayOutputStream b = new ByteArrayOutputStream();
394    parser.compose(b, vsid);
395    b.close();
396    String s = new String(b.toByteArray(), Constants.CHARSET_UTF8);
397    // any code systems we can find, we add these too. 
398    for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
399      CodeSystem cs = fetchCodeSystem(inc.getSystem());
400      if (cs != null) {
401        String css = cacheValue(cs);
402        s = s + css;
403      }
404    }
405    s = s + "-" + Boolean.toString(heirarchical);
406    String r = Integer.toString(s.hashCode());
407    //    TextFile.stringToFile(s, Utilities.path(cache, r+".id.json"));
408    return r;
409  }
410
411
412  private String cacheValue(CodeSystem cs) throws IOException {
413    CodeSystem csid = new CodeSystem();
414    csid.setId(cs.getId());
415    csid.setVersion(cs.getVersion());
416    csid.setContent(cs.getContent());
417    csid.setHierarchyMeaning(CodeSystemHierarchyMeaning.GROUPEDBY);
418    for (ConceptDefinitionComponent cc : cs.getConcept()) {
419      csid.getConcept().add(processCSConcept(cc));
420    }
421    JsonParser parser = new JsonParser();
422    parser.setOutputStyle(OutputStyle.NORMAL);
423    ByteArrayOutputStream b = new ByteArrayOutputStream();
424    parser.compose(b, csid);
425    b.close();
426    return new String(b.toByteArray(), Constants.CHARSET_UTF8);
427  }
428
429
430  private ConceptDefinitionComponent processCSConcept(ConceptDefinitionComponent cc) {
431    ConceptDefinitionComponent ccid = new ConceptDefinitionComponent();
432    ccid.setCode(cc.getCode());
433    ccid.setDisplay(cc.getDisplay());
434    for (ConceptDefinitionComponent cci : cc.getConcept()) {
435      ccid.getConcept().add(processCSConcept(cci));
436    }
437    return ccid;
438  }
439
440  public ValueSetExpansionOutcome expandOnServer(ValueSet vs, String fn) throws Exception {
441    if (noTerminologyServer) {
442      return new ValueSetExpansionOutcome(
443        "Error expanding ValueSet: running without terminology services",
444        TerminologyServiceErrorClass.NOSERVICE);
445    }
446    if (expProfile == null) {
447      throw new Exception("No ExpansionProfile provided");
448    }
449
450    try {
451      Map<String, String> params = new HashMap<String, String>();
452      params.put("_limit", Integer.toString(expandCodesLimit));
453      params.put("_incomplete", "true");
454      tlog("Terminology Server: $expand on " + getVSSummary(vs));
455      ValueSet result = txServer.expandValueset(vs, expProfile.setIncludeDefinition(false), params);
456      return new ValueSetExpansionOutcome(result);
457    } catch (Exception e) {
458      return new ValueSetExpansionOutcome(
459        "Error expanding ValueSet \"" + vs.getUrl() + ": " + e.getMessage(),
460        TerminologyServiceErrorClass.UNKNOWN);
461    }
462  }
463
464  private String getVSSummary(ValueSet vs) {
465    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
466    for (ConceptSetComponent cc : vs.getCompose().getInclude()) {
467      b.append("Include " + getIncSummary(cc));
468    }
469    for (ConceptSetComponent cc : vs.getCompose().getExclude()) {
470      b.append("Exclude " + getIncSummary(cc));
471    }
472    return b.toString();
473  }
474
475  private String getIncSummary(ConceptSetComponent cc) {
476    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
477    for (UriType vs : cc.getValueSet()) {
478      b.append(vs.asStringValue());
479    }
480    String vsd =
481      b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
482    String system = cc.getSystem();
483    if (cc.hasConcept()) {
484      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
485    }
486    if (cc.hasFilter()) {
487      String s = "";
488      for (ConceptSetFilterComponent f : cc.getFilter()) {
489        if (!Utilities.noString(s)) {
490          s = s + " & ";
491        }
492        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
493      }
494      return "from " + system + " where " + s + vsd;
495    }
496    return "All codes from " + system + vsd;
497  }
498
499  private ValidationResult handleByCache(ValueSet vs, Coding coding, boolean tryCache) {
500    String cacheId = cacheId(coding);
501    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
502    if (cache == null) {
503      cache = new HashMap<String, IWorkerContext.ValidationResult>();
504      validationCache.put(vs.getUrl(), cache);
505    }
506    if (cache.containsKey(cacheId)) {
507      return cache.get(cacheId);
508    }
509    if (!tryCache) {
510      return null;
511    }
512    if (!cacheValidation) {
513      return null;
514    }
515    if (failed.contains(vs.getUrl())) {
516      return null;
517    }
518    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
519    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
520      failed.add(vs.getUrl());
521      return null;
522    }
523
524    ValidationResult res = validateCode(coding, vse.getValueset());
525    cache.put(cacheId, res);
526    return res;
527  }
528
529  private boolean notcomplete(ValueSet vs) {
530    if (!vs.hasExpansion()) {
531      return true;
532    }
533    if (!vs.getExpansion()
534      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-unclosed").isEmpty()) {
535      return true;
536    }
537    if (!vs.getExpansion()
538      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-toocostly").isEmpty()) {
539      return true;
540    }
541    return false;
542  }
543
544  private ValidationResult handleByCache(ValueSet vs, CodeableConcept concept, boolean tryCache) {
545    String cacheId = cacheId(concept);
546    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
547    if (cache == null) {
548      cache = new HashMap<String, IWorkerContext.ValidationResult>();
549      validationCache.put(vs.getUrl(), cache);
550    }
551    if (cache.containsKey(cacheId)) {
552      return cache.get(cacheId);
553    }
554
555    if (validationCache.containsKey(vs.getUrl()) && validationCache.get(vs.getUrl())
556      .containsKey(cacheId)) {
557      return validationCache.get(vs.getUrl()).get(cacheId);
558    }
559    if (!tryCache) {
560      return null;
561    }
562    if (!cacheValidation) {
563      return null;
564    }
565    if (failed.contains(vs.getUrl())) {
566      return null;
567    }
568    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
569    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
570      failed.add(vs.getUrl());
571      return null;
572    }
573    ValidationResult res = validateCode(concept, vse.getValueset());
574    cache.put(cacheId, res);
575    return res;
576  }
577
578  private String cacheId(Coding coding) {
579    return "|" + coding.getSystem() + "|" + coding.getVersion() + "|" + coding.getCode() + "|"
580      + coding.getDisplay();
581  }
582
583  private String cacheId(CodeableConcept cc) {
584    StringBuilder b = new StringBuilder();
585    for (Coding c : cc.getCoding()) {
586      b.append("#");
587      b.append(cacheId(c));
588    }
589    return b.toString();
590  }
591
592  private ValidationResult verifyCodeExternal(ValueSet vs, Coding coding, boolean tryCache)
593    throws Exception {
594    ValidationResult res = vs == null ? null : handleByCache(vs, coding, tryCache);
595    if (res != null) {
596      return res;
597    }
598    Parameters pin = new Parameters();
599    pin.addParameter().setName("coding").setValue(coding);
600    if (vs != null) {
601      pin.addParameter().setName("valueSet").setResource(vs);
602    }
603    res = serverValidateCode(pin, vs == null);
604    if (vs != null) {
605      Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
606      cache.put(cacheId(coding), res);
607    }
608    return res;
609  }
610
611  private ValidationResult verifyCodeExternal(ValueSet vs, CodeableConcept cc, boolean tryCache)
612    throws Exception {
613    ValidationResult res = handleByCache(vs, cc, tryCache);
614    if (res != null) {
615      return res;
616    }
617    Parameters pin = new Parameters();
618    pin.addParameter().setName("codeableConcept").setValue(cc);
619    pin.addParameter().setName("valueSet").setResource(vs);
620    res = serverValidateCode(pin, false);
621    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
622    cache.put(cacheId(cc), res);
623    return res;
624  }
625
626  private ValidationResult serverValidateCode(Parameters pin, boolean doCache) throws Exception {
627    if (noTerminologyServer) {
628      return new ValidationResult(null, null, TerminologyServiceErrorClass.NOSERVICE);
629    }
630    String cacheName = doCache ? generateCacheName(pin) : null;
631    ValidationResult res = loadFromCache(cacheName);
632    if (res != null) {
633      return res;
634    }
635    tlog("Terminology Server: $validate-code " + describeValidationParameters(pin));
636    for (ParametersParameterComponent pp : pin.getParameter()) {
637      if (pp.getName().equals("profile")) {
638        throw new Error("Can only specify profile in the context");
639      }
640    }
641    if (expProfile == null) {
642      throw new Exception("No ExpansionProfile provided");
643    }
644    pin.addParameter().setName("profile").setResource(expProfile);
645
646    Parameters pout = txServer.operateType(ValueSet.class, "validate-code", pin);
647    boolean ok = false;
648    String message = "No Message returned";
649    String display = null;
650    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
651    for (ParametersParameterComponent p : pout.getParameter()) {
652      if (p.getName().equals("result")) {
653        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
654      } else if (p.getName().equals("message")) {
655        message = ((StringType) p.getValue()).getValue();
656      } else if (p.getName().equals("display")) {
657        display = ((StringType) p.getValue()).getValue();
658      } else if (p.getName().equals("cause")) {
659        try {
660          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
661          if (it == IssueType.UNKNOWN) {
662            err = TerminologyServiceErrorClass.UNKNOWN;
663          } else if (it == IssueType.NOTSUPPORTED) {
664            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
665          }
666        } catch (FHIRException e) {
667        }
668      }
669    }
670    if (!ok) {
671      res = new ValidationResult(IssueSeverity.ERROR, message, err);
672    } else if (display != null) {
673      res = new ValidationResult(new ConceptDefinitionComponent().setDisplay(display));
674    } else {
675      res = new ValidationResult(new ConceptDefinitionComponent());
676    }
677    saveToCache(res, cacheName);
678    return res;
679  }
680
681
682  private void tlog(String msg) {
683    //    log(msg);
684  }
685
686  @SuppressWarnings("rawtypes")
687  private String describeValidationParameters(Parameters pin) {
688    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
689    for (ParametersParameterComponent p : pin.getParameter()) {
690      if (p.hasValue() && p.getValue() instanceof PrimitiveType) {
691        b.append(p.getName() + "=" + ((PrimitiveType) p.getValue()).asStringValue());
692      } else if (p.hasValue() && p.getValue() instanceof Coding) {
693        b.append("system=" + ((Coding) p.getValue()).getSystem());
694        b.append("code=" + ((Coding) p.getValue()).getCode());
695        b.append("display=" + ((Coding) p.getValue()).getDisplay());
696      } else if (p.hasValue() && p.getValue() instanceof CodeableConcept) {
697        if (((CodeableConcept) p.getValue()).hasCoding()) {
698          Coding c = ((CodeableConcept) p.getValue()).getCodingFirstRep();
699          b.append("system=" + c.getSystem());
700          b.append("code=" + c.getCode());
701          b.append("display=" + c.getDisplay());
702        } else if (((CodeableConcept) p.getValue()).hasText()) {
703          b.append("text=" + ((CodeableConcept) p.getValue()).getText());
704        }
705      } else if (p.hasResource() && (p.getResource() instanceof ValueSet)) {
706        b.append("valueset=" + getVSSummary((ValueSet) p.getResource()));
707      }
708    }
709    return b.toString();
710  }
711
712  private ValidationResult loadFromCache(String fn) throws FileNotFoundException, IOException {
713    if (fn == null) {
714      return null;
715    }
716    if (!(new File(fn).exists())) {
717      return null;
718    }
719    String cnt = TextFile.fileToString(fn);
720    if (cnt.startsWith("!error: ")) {
721      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(8));
722    } else if (cnt.startsWith("!warning: ")) {
723      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(10));
724    } else {
725      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(cnt));
726    }
727  }
728
729  private void saveToCache(ValidationResult res, String cacheName) throws IOException {
730    if (cacheName == null) {
731      return;
732    }
733    if (res.getDisplay() != null) {
734      TextFile.stringToFile(res.getDisplay(), cacheName);
735    } else if (res.getMessage() != null) {
736      if (res.getSeverity() == IssueSeverity.WARNING) {
737        TextFile.stringToFile("!warning: " + res.getMessage(), cacheName);
738      } else {
739        TextFile.stringToFile("!error: " + res.getMessage(), cacheName);
740      }
741    }
742  }
743
744  private String generateCacheName(Parameters pin) throws IOException {
745    if (cache == null) {
746      return null;
747    }
748    String json = new JsonParser().composeString(pin);
749    return Utilities.path(cache, "vc" + Integer.toString(json.hashCode()) + ".json");
750  }
751
752  @Override
753  public ValueSetExpansionComponent expandVS(ConceptSetComponent inc, boolean heirachical)
754    throws TerminologyServiceException {
755    ValueSet vs = new ValueSet();
756    vs.setCompose(new ValueSetComposeComponent());
757    vs.getCompose().getInclude().add(inc);
758    ValueSetExpansionOutcome vse = expandVS(vs, true, heirachical);
759    ValueSet valueset = vse.getValueset();
760    if (valueset == null) {
761      throw new TerminologyServiceException("Error Expanding ValueSet: " + vse.getError());
762    }
763    return valueset.getExpansion();
764  }
765
766  @Override
767  public ValidationResult validateCode(String system, String code, String display) {
768    try {
769      if (codeSystems.containsKey(system) && codeSystems.get(system) != null) {
770        return verifyCodeInCodeSystem(codeSystems.get(system), system, code, display);
771      } else {
772        return verifyCodeExternal(null,
773          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
774      }
775    } catch (Exception e) {
776      return new ValidationResult(IssueSeverity.FATAL,
777        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
778    }
779  }
780
781
782  @Override
783  public ValidationResult validateCode(Coding code, ValueSet vs) {
784    if (codeSystems.containsKey(code.getSystem()) && codeSystems.get(code.getSystem()) != null) {
785      try {
786        return verifyCodeInCodeSystem(codeSystems.get(code.getSystem()), code.getSystem(),
787          code.getCode(), code.getDisplay());
788      } catch (Exception e) {
789        return new ValidationResult(IssueSeverity.FATAL,
790          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
791            .getMessage());
792      }
793    } else if (vs.hasExpansion()) {
794      try {
795        return verifyCodeInternal(vs, code.getSystem(), code.getCode(), code.getDisplay());
796      } catch (Exception e) {
797        return new ValidationResult(IssueSeverity.FATAL,
798          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
799            .getMessage());
800      }
801    } else {
802      try {
803        return verifyCodeExternal(vs, code, true);
804      } catch (Exception e) {
805        return new ValidationResult(IssueSeverity.WARNING,
806          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
807            .getMessage());
808      }
809    }
810  }
811
812  @Override
813  public ValidationResult validateCode(CodeableConcept code, ValueSet vs) {
814    try {
815      if (vs.hasExpansion()) {
816        return verifyCodeInternal(vs, code);
817      } else {
818        // we'll try expanding first; if that doesn't work, then we'll just pass it to the server to validate 
819        // ... could be a problem if the server doesn't have the code systems we have locally, so we try not to depend on the server
820        try {
821          ValueSetExpansionOutcome vse = expandVS(vs, true, false);
822          if (vse.getValueset() != null && !hasTooCostlyExpansion(vse.getValueset())) {
823            return verifyCodeInternal(vse.getValueset(), code);
824          }
825        } catch (Exception e) {
826          // failed? we'll just try the server
827        }
828        return verifyCodeExternal(vs, code, true);
829      }
830    } catch (Exception e) {
831      return new ValidationResult(IssueSeverity.FATAL,
832        "Error validating code \"" + code.toString() + "\": " + e.getMessage(),
833        TerminologyServiceErrorClass.SERVER_ERROR);
834    }
835  }
836
837
838  private boolean hasTooCostlyExpansion(ValueSet valueset) {
839    return valueset != null && valueset.hasExpansion() && ToolingExtensions
840      .hasExtension(valueset.getExpansion(),
841        "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
842  }
843
844  @Override
845  public ValidationResult validateCode(String system, String code, String display, ValueSet vs) {
846    try {
847      if (system == null && display == null) {
848        return verifyCodeInternal(vs, code);
849      }
850      if ((codeSystems.containsKey(system) && codeSystems.get(system) != null) || vs
851        .hasExpansion()) {
852        return verifyCodeInternal(vs, system, code, display);
853      } else {
854        return verifyCodeExternal(vs,
855          new Coding().setSystem(system).setCode(code).setDisplay(display), true);
856      }
857    } catch (Exception e) {
858      return new ValidationResult(IssueSeverity.FATAL,
859        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage(),
860        TerminologyServiceErrorClass.SERVER_ERROR);
861    }
862  }
863
864  @Override
865  public ValidationResult validateCode(String system, String code, String display,
866    ConceptSetComponent vsi) {
867    try {
868      ValueSet vs = new ValueSet();
869      vs.setUrl(Utilities.makeUuidUrn());
870      vs.getCompose().addInclude(vsi);
871      return verifyCodeExternal(vs,
872        new Coding().setSystem(system).setCode(code).setDisplay(display), true);
873    } catch (Exception e) {
874      return new ValidationResult(IssueSeverity.FATAL,
875        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
876    }
877  }
878
879  public void initTS(String cachePath, String tsServer) throws Exception {
880    cache = cachePath;
881    this.tsServer = tsServer;
882    expansionCache = new ValueSetExpansionCache(this, null);
883    validationCachePath = Utilities.path(cachePath, "validation.cache");
884    try {
885      loadValidationCache();
886    } catch (Exception e) {
887      e.printStackTrace();
888    }
889  }
890
891  protected void loadValidationCache() throws JsonSyntaxException, Exception {
892  }
893
894  @Override
895  public List<ConceptMap> findMapsForSource(String url) {
896    List<ConceptMap> res = new ArrayList<ConceptMap>();
897    for (ConceptMap map : maps.values()) {
898      if (((Reference) map.getSource()).getReference().equals(url)) {
899        res.add(map);
900      }
901    }
902    return res;
903  }
904
905  private ValidationResult verifyCodeInternal(ValueSet vs, CodeableConcept code) throws Exception {
906    for (Coding c : code.getCoding()) {
907      ValidationResult res = verifyCodeInternal(vs, c.getSystem(), c.getCode(), c.getDisplay());
908      if (res.isOk()) {
909        return res;
910      }
911    }
912    if (code.getCoding().isEmpty()) {
913      return new ValidationResult(IssueSeverity.ERROR, "None code provided");
914    } else {
915      return new ValidationResult(IssueSeverity.ERROR,
916        "None of the codes are in the specified value set");
917    }
918  }
919
920  private ValidationResult verifyCodeInternal(ValueSet vs, String system, String code,
921    String display) throws Exception {
922    if (vs.hasExpansion()) {
923      return verifyCodeInExpansion(vs, system, code, display);
924    } else {
925      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
926      if (vse.getValueset() != null) {
927        return verifyCodeExternal(vs,
928          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
929      } else {
930        return verifyCodeInExpansion(vse.getValueset(), system, code, display);
931      }
932    }
933  }
934
935  private ValidationResult verifyCodeInternal(ValueSet vs, String code)
936    throws FileNotFoundException, ETooCostly, IOException, FHIRException {
937    if (vs.hasExpansion()) {
938      return verifyCodeInExpansion(vs, code);
939    } else {
940      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
941      if (vse.getValueset() == null) {
942        return new ValidationResult(IssueSeverity.ERROR, vse.getError(), vse.getErrorClass());
943      } else {
944        return verifyCodeInExpansion(vse.getValueset(), code);
945      }
946    }
947  }
948
949  private ValidationResult verifyCodeInCodeSystem(CodeSystem cs, String system, String code,
950    String display) throws Exception {
951    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code);
952    if (cc == null) {
953      if (cs.getContent().equals(CodeSystem.CodeSystemContentMode.COMPLETE)) {
954        return new ValidationResult(IssueSeverity.ERROR,
955          "Unknown Code " + code + " in " + cs.getUrl());
956      } else if (!cs.getContent().equals(CodeSystem.CodeSystemContentMode.NOTPRESENT)) {
957        return new ValidationResult(IssueSeverity.WARNING,
958          "Unknown Code " + code + " in partial code list of " + cs.getUrl());
959      } else {
960        return verifyCodeExternal(null,
961          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
962      }
963    }
964    //
965    //        return new ValidationResult(IssueSeverity.WARNING, "A definition was found for "+cs.getUrl()+", but it has no codes in the definition");
966    //      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
967    if (display == null) {
968      return new ValidationResult(cc);
969    }
970    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
971    if (cc.hasDisplay()) {
972      b.append(cc.getDisplay());
973      if (display.equalsIgnoreCase(cc.getDisplay())) {
974        return new ValidationResult(cc);
975      }
976    }
977    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
978      b.append(ds.getValue());
979      if (display.equalsIgnoreCase(ds.getValue())) {
980        return new ValidationResult(cc);
981      }
982    }
983    return new ValidationResult(IssueSeverity.WARNING,
984      "Display Name for " + code + " must be one of '" + b.toString() + "'", cc);
985  }
986
987
988  private ValidationResult verifyCodeInExpansion(ValueSet vs, String system, String code,
989    String display) {
990    ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
991    if (cc == null) {
992      return new ValidationResult(IssueSeverity.ERROR,
993        "Unknown Code " + code + " in " + vs.getUrl());
994    }
995    if (display == null) {
996      return new ValidationResult(
997        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
998    }
999    if (cc.hasDisplay()) {
1000      if (display.equalsIgnoreCase(cc.getDisplay())) {
1001        return new ValidationResult(
1002          new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1003      }
1004      return new ValidationResult(IssueSeverity.WARNING,
1005        "Display Name for " + code + " must be '" + cc.getDisplay() + "'",
1006        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1007    }
1008    return null;
1009  }
1010
1011  private ValidationResult verifyCodeInExpansion(ValueSet vs, String code) throws FHIRException {
1012    if (vs.getExpansion()
1013      .hasExtension("http://hl7.org/fhir/StructureDefinition/valueset-toocostly")) {
1014      throw new FHIRException("Unable to validate core - value set is too costly to expand");
1015    } else {
1016      ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
1017      if (cc == null) {
1018        return new ValidationResult(IssueSeverity.ERROR,
1019          "Unknown Code " + code + " in " + vs.getUrl());
1020      }
1021      return null;
1022    }
1023  }
1024
1025  private ValueSetExpansionContainsComponent findCode(
1026    List<ValueSetExpansionContainsComponent> contains, String code) {
1027    for (ValueSetExpansionContainsComponent cc : contains) {
1028      if (code.equals(cc.getCode())) {
1029        return cc;
1030      }
1031      ValueSetExpansionContainsComponent c = findCode(cc.getContains(), code);
1032      if (c != null) {
1033        return c;
1034      }
1035    }
1036    return null;
1037  }
1038
1039  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept,
1040    String code) {
1041    for (ConceptDefinitionComponent cc : concept) {
1042      if (code.equals(cc.getCode())) {
1043        return cc;
1044      }
1045      ConceptDefinitionComponent c = findCodeInConcept(cc.getConcept(), code);
1046      if (c != null) {
1047        return c;
1048      }
1049    }
1050    return null;
1051  }
1052
1053  public Set<String> getNonSupportedCodeSystems() {
1054    return nonSupportedCodeSystems;
1055  }
1056
1057  public boolean isCanRunWithoutTerminology() {
1058    return canRunWithoutTerminology;
1059  }
1060
1061  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
1062    this.canRunWithoutTerminology = canRunWithoutTerminology;
1063  }
1064
1065  public int getExpandCodesLimit() {
1066    return expandCodesLimit;
1067  }
1068
1069  public void setExpandCodesLimit(int expandCodesLimit) {
1070    this.expandCodesLimit = expandCodesLimit;
1071  }
1072
1073  public void setLogger(ILoggingService logger) {
1074    this.logger = logger;
1075  }
1076
1077  public ExpansionProfile getExpansionProfile() {
1078    return expProfile;
1079  }
1080
1081  public void setExpansionProfile(ExpansionProfile expProfile) {
1082    this.expProfile = expProfile;
1083  }
1084
1085  @Override
1086  public boolean isNoTerminologyServer() {
1087    return noTerminologyServer;
1088  }
1089
1090  public String getName() {
1091    return name;
1092  }
1093
1094  public void setName(String name) {
1095    this.name = name;
1096  }
1097
1098  @Override
1099  public Set<String> getResourceNamesAsSet() {
1100    Set<String> res = new HashSet<String>();
1101    res.addAll(getResourceNames());
1102    return res;
1103  }
1104
1105  public void reportStatus(JsonObject json) {
1106    json.addProperty("codeystem-count", codeSystems.size());
1107    json.addProperty("valueset-count", valueSets.size());
1108    json.addProperty("conceptmap-count", maps.size());
1109    json.addProperty("transforms-count", transforms.size());
1110    json.addProperty("structures-count", profiles.size());
1111  }
1112
1113  public void cacheResource(Resource r) throws Exception {
1114    if (r instanceof ValueSet) {
1115      seeValueSet(((ValueSet) r).getUrl(), (ValueSet) r);
1116    } else if (r instanceof CodeSystem) {
1117      seeCodeSystem(((CodeSystem) r).getUrl(), (CodeSystem) r);
1118    } else if (r instanceof StructureDefinition) {
1119      StructureDefinition sd = (StructureDefinition) r;
1120      if ("http://hl7.org/fhir/StructureDefinition/Extension".equals(sd.getBaseDefinition())) {
1121        seeExtensionDefinition(sd.getUrl(), sd);
1122      } else if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) {
1123        seeProfile(sd.getUrl(), sd);
1124      }
1125    }
1126  }
1127
1128  public void dropResource(String type, String id) throws FHIRException {
1129    if (type.equals("ValueSet")) {
1130      dropValueSet(id);
1131    }
1132    if (type.equals("CodeSystem")) {
1133      dropCodeSystem(id);
1134    }
1135    if (type.equals("StructureDefinition")) {
1136      dropProfile(id);
1137      dropExtensionDefinition(id);
1138    }
1139  }
1140
1141  public boolean isAllowLoadingDuplicates() {
1142    return allowLoadingDuplicates;
1143  }
1144
1145  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
1146    this.allowLoadingDuplicates = allowLoadingDuplicates;
1147  }
1148
1149  @Override
1150  public StructureDefinition fetchTypeDefinition(String typeName) {
1151    return fetchResource(StructureDefinition.class,
1152      "http://hl7.org/fhir/StructureDefinition/" + typeName);
1153  }
1154}