001package org.hl7.fhir.r4.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032import java.io.File;
033import java.io.FileNotFoundException;
034import java.io.FileOutputStream;
035import java.io.IOException;
036import java.io.OutputStreamWriter;
037import java.util.ArrayList;
038import java.util.HashMap;
039import java.util.List;
040import java.util.Map;
041
042import org.apache.commons.lang3.StringUtils;
043import org.hl7.fhir.exceptions.FHIRException;
044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
045import org.hl7.fhir.r4.formats.IParser.OutputStyle;
046import org.hl7.fhir.r4.formats.JsonParser;
047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
048import org.hl7.fhir.r4.model.CodeableConcept;
049import org.hl7.fhir.r4.model.Coding;
050import org.hl7.fhir.r4.model.UriType;
051import org.hl7.fhir.r4.model.ValueSet;
052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
058import org.hl7.fhir.utilities.FileUtilities;
059import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
060import org.hl7.fhir.utilities.Utilities;
061import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
062import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
063import org.hl7.fhir.utilities.validation.ValidationOptions;
064
065import com.google.gson.JsonElement;
066import com.google.gson.JsonNull;
067import com.google.gson.JsonObject;
068import com.google.gson.JsonPrimitive;
069
070/**
071 * This implements a two level cache. - a temporary cache for remmbering
072 * previous local operations - a persistent cache for rembering tx server
073 * operations
074 * 
075 * the cache is a series of pairs: a map, and a list. the map is the loaded
076 * cache, the list is the persiistent cache, carefully maintained in order for
077 * version control consistency
078 * 
079 * @author graha
080 *
081 */
082@MarkedToMoveToAdjunctPackage
083public class TerminologyCache {
084  public static final boolean TRANSIENT = false;
085  public static final boolean PERMANENT = true;
086  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
087  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
088  private static final String BREAK = "####";
089
090  private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator();
091
092  protected SystemNameKeyGenerator getSystemNameKeyGenerator() {
093    return systemNameKeyGenerator;
094  }
095
096  public class SystemNameKeyGenerator {
097    public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct";
098    public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm";
099    public static final String LOINC_CODESYSTEM_URL = "http://loinc.org";
100    public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org";
101
102    public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/";
103    public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/";
104    public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/";
105
106    public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:";
107    public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47";
108    public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13";
109
110    public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101";
111    public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM";
112
113    public String getNameForSystem(String system) {
114      final int lastPipe = system.lastIndexOf('|');
115      final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe);
116      final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1);
117
118      if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL))
119        return getVersionedSystem("snomed", systemVersion);
120      if (systemBaseName.equals(RXNORM_CODESYSTEM_URL))
121        return getVersionedSystem("rxnorm", systemVersion);
122      if (systemBaseName.equals(LOINC_CODESYSTEM_URL))
123        return getVersionedSystem("loinc", systemVersion);
124      if (systemBaseName.equals(UCUM_CODESYSTEM_URL))
125        return getVersionedSystem("ucum", systemVersion);
126      if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL))
127        return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
128      if (systemBaseName.equals(_11073_CODESYSTEM_URN))
129        return getVersionedSystem("11073", systemVersion);
130      if (systemBaseName.startsWith(ISO_CODESYSTEM_URN))
131        return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""),
132            systemVersion);
133      if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL))
134        return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
135      if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL))
136        return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
137      if (systemBaseName.equals(LANG_CODESYSTEM_URN))
138        return getVersionedSystem("lang", systemVersion);
139      if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN))
140        return getVersionedSystem("mimetypes", systemVersion);
141      if (systemBaseName.equals(DICOM_CODESYSTEM_URL))
142        return getVersionedSystem("dicom", systemVersion);
143      return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"),
144          systemVersion);
145    }
146
147    public String normalizeBaseURL(String baseUrl, String fullUrl) {
148      return fullUrl.substring(baseUrl.length()).replace("/", "");
149    }
150
151    public String getVersionedSystem(String baseSystem, String version) {
152      if (version != null) {
153        return baseSystem + "_" + version;
154      }
155      return baseSystem;
156    }
157  }
158
159  public class CacheToken {
160    private String name;
161    private String key;
162    private String request;
163
164    public void setName(String n) {
165      String systemName = getSystemNameKeyGenerator().getNameForSystem(n);
166      if (name == null)
167        name = systemName;
168      else if (!systemName.equals(name))
169        name = NAME_FOR_NO_SYSTEM;
170    }
171
172    public String getName() {
173      return name;
174    }
175  }
176
177  private class CacheEntry {
178    private String request;
179    private boolean persistent;
180    private ValidationResult v;
181    private ValueSetExpansionOutcome e;
182  }
183
184  private class NamedCache {
185    private String name;
186    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
187    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
188  }
189
190  private Object lock;
191  private String folder;
192  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
193
194  // use lock from the context
195  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
196    super();
197    this.lock = lock;
198    this.folder = folder;
199    if (folder != null)
200      load();
201  }
202
203  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
204    CacheToken ct = new CacheToken();
205    if (code.hasSystem())
206      ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem());
207    else
208      ct.name = NAME_FOR_NO_SYSTEM;
209    JsonParser json = new JsonParser();
210    json.setOutputStyle(OutputStyle.PRETTY);
211    ValueSet vsc = getVSEssense(vs);
212    try {
213      ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :"
214          + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}";
215    } catch (IOException e) {
216      throw new Error(e);
217    }
218    ct.key = String.valueOf(hashNWS(ct.request));
219    return ct;
220  }
221
222  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
223    CacheToken ct = new CacheToken();
224    for (Coding c : code.getCoding()) {
225      if (c.hasSystem())
226        ct.setName(c.getSystem());
227    }
228    JsonParser json = new JsonParser();
229    json.setOutputStyle(OutputStyle.PRETTY);
230    ValueSet vsc = getVSEssense(vs);
231    try {
232      ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :"
233          + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}";
234    } catch (IOException e) {
235      throw new Error(e);
236    }
237    ct.key = String.valueOf(hashNWS(ct.request));
238    return ct;
239  }
240
241  public ValueSet getVSEssense(ValueSet vs) {
242    if (vs == null)
243      return null;
244    ValueSet vsc = new ValueSet();
245    vsc.setCompose(vs.getCompose());
246    if (vs.hasExpansion()) {
247      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
248      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
249    }
250    return vsc;
251  }
252
253  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
254    CacheToken ct = new CacheToken();
255    ValueSet vsc = getVSEssense(vs);
256    for (ConceptSetComponent inc : vs.getCompose().getInclude())
257      if (inc.hasSystem())
258        ct.setName(inc.getSystem());
259    for (ConceptSetComponent inc : vs.getCompose().getExclude())
260      if (inc.hasSystem())
261        ct.setName(inc.getSystem());
262    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
263      if (inc.hasSystem())
264        ct.setName(inc.getSystem());
265    JsonParser json = new JsonParser();
266    json.setOutputStyle(OutputStyle.PRETTY);
267    try {
268      ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :"
269          + json.composeString(vsc) + "}\r\n";
270    } catch (IOException e) {
271      throw new Error(e);
272    }
273    ct.key = String.valueOf(hashNWS(ct.request));
274    return ct;
275  }
276
277  public NamedCache getNamedCache(CacheToken cacheToken) {
278    NamedCache nc = caches.get(cacheToken.name);
279    if (nc == null) {
280      nc = new NamedCache();
281      nc.name = cacheToken.name;
282      caches.put(nc.name, nc);
283    }
284    return nc;
285  }
286
287  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
288    synchronized (lock) {
289      NamedCache nc = getNamedCache(cacheToken);
290      CacheEntry e = nc.map.get(cacheToken.key);
291      if (e == null)
292        return null;
293      else
294        return e.e;
295    }
296  }
297
298  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
299    synchronized (lock) {
300      NamedCache nc = getNamedCache(cacheToken);
301      CacheEntry e = new CacheEntry();
302      e.request = cacheToken.request;
303      e.persistent = persistent;
304      e.e = res;
305      store(cacheToken, persistent, nc, e);
306    }
307  }
308
309  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
310    boolean n = nc.map.containsKey(cacheToken.key);
311    nc.map.put(cacheToken.key, e);
312    if (persistent) {
313      if (n) {
314        for (int i = nc.list.size() - 1; i >= 0; i--) {
315          if (nc.list.get(i).request.equals(e.request)) {
316            nc.list.remove(i);
317          }
318        }
319      }
320      nc.list.add(e);
321      save(nc);
322    }
323  }
324
325  public ValidationResult getValidation(CacheToken cacheToken) {
326    synchronized (lock) {
327      NamedCache nc = getNamedCache(cacheToken);
328      CacheEntry e = nc.map.get(cacheToken.key);
329      if (e == null)
330        return null;
331      else
332        return e.v;
333    }
334  }
335
336  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
337    synchronized (lock) {
338      NamedCache nc = getNamedCache(cacheToken);
339      CacheEntry e = new CacheEntry();
340      e.request = cacheToken.request;
341      e.persistent = persistent;
342      e.v = res;
343      store(cacheToken, persistent, nc, e);
344    }
345  }
346
347  // persistence
348
349  public void save() {
350
351  }
352
353  private void save(NamedCache nc) {
354    if (folder == null)
355      return;
356
357    try {
358      OutputStreamWriter sw = new OutputStreamWriter(ManagedFileAccess.outStream(Utilities.path(folder, nc.name + ".cache")),
359          "UTF-8");
360      sw.write(ENTRY_MARKER + "\r\n");
361      JsonParser json = new JsonParser();
362      json.setOutputStyle(OutputStyle.PRETTY);
363      for (CacheEntry ce : nc.list) {
364        sw.write(ce.request.trim());
365        sw.write(BREAK + "\r\n");
366        if (ce.e != null) {
367          sw.write("e: {\r\n");
368          if (ce.e.getValueset() != null)
369            sw.write("  \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n");
370          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n");
371        } else {
372          sw.write("v: {\r\n");
373          sw.write("  \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n");
374          sw.write("  \"severity\" : "
375              + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n");
376          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n");
377        }
378        sw.write(ENTRY_MARKER + "\r\n");
379      }
380      sw.close();
381    } catch (Exception e) {
382      System.out.println("error saving " + nc.name + ": " + e.getMessage());
383    }
384  }
385
386  private void load() throws FHIRException, IOException {
387    for (String fn : ManagedFileAccess.file(folder).list()) {
388      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
389        try {
390          // System.out.println("Load "+fn);
391          String title = fn.substring(0, fn.lastIndexOf("."));
392          NamedCache nc = new NamedCache();
393          nc.name = title;
394          caches.put(title, nc);
395          System.out.print(" - load " + title + ".cache");
396          String src = FileUtilities.fileToString(Utilities.path(folder, fn));
397          if (src.startsWith("?"))
398            src = src.substring(1);
399          int i = src.indexOf(ENTRY_MARKER);
400          while (i > -1) {
401            String s = src.substring(0, i);
402            System.out.print(".");
403            src = src.substring(i + ENTRY_MARKER.length() + 1);
404            i = src.indexOf(ENTRY_MARKER);
405            if (!Utilities.noString(s)) {
406              int j = s.indexOf(BREAK);
407              String q = s.substring(0, j);
408              String p = s.substring(j + BREAK.length() + 1).trim();
409              CacheEntry ce = new CacheEntry();
410              ce.persistent = true;
411              ce.request = q;
412              boolean e = p.charAt(0) == 'e';
413              p = p.substring(3);
414              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
415              String error = loadJS(o.get("error"));
416              if (e) {
417                if (o.has("valueSet"))
418                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")),
419                      error, TerminologyServiceErrorClass.UNKNOWN);
420                else
421                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
422              } else {
423                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null
424                    : IssueSeverity.fromCode(o.get("severity").getAsString());
425                String display = loadJS(o.get("display"));
426                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
427              }
428              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
429              nc.list.add(ce);
430            }
431          }
432          System.out.println("done");
433        } catch (Exception e) {
434          throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e);
435        }
436      }
437    }
438  }
439
440  private String loadJS(JsonElement e) {
441    if (e == null)
442      return null;
443    if (!(e instanceof JsonPrimitive))
444      return null;
445    String s = e.getAsString();
446    if ("".equals(s))
447      return null;
448    return s;
449  }
450
451  private String hashNWS(String s) {
452    s = StringUtils.remove(s, ' ');
453    s = StringUtils.remove(s, '\n');
454    s = StringUtils.remove(s, '\r');
455    return String.valueOf(s.hashCode());
456  }
457
458  // management
459
460  public TerminologyCache copy() {
461    // TODO Auto-generated method stub
462    return null;
463  }
464
465  public String summary(ValueSet vs) {
466    if (vs == null)
467      return "null";
468
469    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
470    for (ConceptSetComponent cc : vs.getCompose().getInclude())
471      b.append("Include " + getIncSummary(cc));
472    for (ConceptSetComponent cc : vs.getCompose().getExclude())
473      b.append("Exclude " + getIncSummary(cc));
474    return b.toString();
475  }
476
477  private String getIncSummary(ConceptSetComponent cc) {
478    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
479    for (UriType vs : cc.getValueSet())
480      b.append(vs.asStringValue());
481    String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
482    String system = cc.getSystem();
483    if (cc.hasConcept())
484      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
485    if (cc.hasFilter()) {
486      String s = "";
487      for (ConceptSetFilterComponent f : cc.getFilter()) {
488        if (!Utilities.noString(s))
489          s = s + " & ";
490        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
491      }
492      return "from " + system + " where " + s + vsd;
493    }
494    return "All codes from " + system + vsd;
495  }
496
497  public String summary(Coding code) {
498    return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\"";
499  }
500
501  public String summary(CodeableConcept code) {
502    StringBuilder b = new StringBuilder();
503    b.append("{");
504    boolean first = true;
505    for (Coding c : code.getCoding()) {
506      if (first)
507        first = false;
508      else
509        b.append(",");
510      b.append(summary(c));
511    }
512    b.append("}: \"");
513    b.append(code.getText());
514    b.append("\"");
515    return b.toString();
516  }
517
518}