Source code

001package org.hl7.fhir.r4.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032import java.io.File;
033import java.io.FileNotFoundException;
034import java.io.FileOutputStream;
035import java.io.IOException;
036import java.io.OutputStreamWriter;
037import java.util.ArrayList;
038import java.util.HashMap;
039import java.util.List;
040import java.util.Map;
041
042import org.apache.commons.lang3.StringUtils;
043import org.hl7.fhir.exceptions.FHIRException;
044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
045import org.hl7.fhir.r4.formats.IParser.OutputStyle;
046import org.hl7.fhir.r4.formats.JsonParser;
047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
048import org.hl7.fhir.r4.model.CodeableConcept;
049import org.hl7.fhir.r4.model.Coding;
050import org.hl7.fhir.r4.model.UriType;
051import org.hl7.fhir.r4.model.ValueSet;
052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
058import org.hl7.fhir.utilities.TextFile;
059import org.hl7.fhir.utilities.Utilities;
060import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
061import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
062import org.hl7.fhir.utilities.validation.ValidationOptions;
063
064import com.google.gson.JsonElement;
065import com.google.gson.JsonNull;
066import com.google.gson.JsonObject;
067import com.google.gson.JsonPrimitive;
068
069/**
070 * This implements a two level cache. - a temporary cache for remmbering
071 * previous local operations - a persistent cache for rembering tx server
072 * operations
073 * 
074 * the cache is a series of pairs: a map, and a list. the map is the loaded
075 * cache, the list is the persiistent cache, carefully maintained in order for
076 * version control consistency
077 * 
078 * @author graha
079 *
080 */
081public class TerminologyCache {
082  public static final boolean TRANSIENT = false;
083  public static final boolean PERMANENT = true;
084  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
085  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
086  private static final String BREAK = "####";
087
088  private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator();
089
090  protected SystemNameKeyGenerator getSystemNameKeyGenerator() {
091    return systemNameKeyGenerator;
092  }
093
094  public class SystemNameKeyGenerator {
095    public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct";
096    public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm";
097    public static final String LOINC_CODESYSTEM_URL = "http://loinc.org";
098    public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org";
099
100    public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/";
101    public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/";
102    public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/";
103
104    public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:";
105    public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47";
106    public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13";
107
108    public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101";
109    public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM";
110
111    public String getNameForSystem(String system) {
112      final int lastPipe = system.lastIndexOf('|');
113      final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe);
114      final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1);
115
116      if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL))
117        return getVersionedSystem("snomed", systemVersion);
118      if (systemBaseName.equals(RXNORM_CODESYSTEM_URL))
119        return getVersionedSystem("rxnorm", systemVersion);
120      if (systemBaseName.equals(LOINC_CODESYSTEM_URL))
121        return getVersionedSystem("loinc", systemVersion);
122      if (systemBaseName.equals(UCUM_CODESYSTEM_URL))
123        return getVersionedSystem("ucum", systemVersion);
124      if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL))
125        return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
126      if (systemBaseName.equals(_11073_CODESYSTEM_URN))
127        return getVersionedSystem("11073", systemVersion);
128      if (systemBaseName.startsWith(ISO_CODESYSTEM_URN))
129        return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""),
130            systemVersion);
131      if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL))
132        return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
133      if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL))
134        return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
135      if (systemBaseName.equals(LANG_CODESYSTEM_URN))
136        return getVersionedSystem("lang", systemVersion);
137      if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN))
138        return getVersionedSystem("mimetypes", systemVersion);
139      if (systemBaseName.equals(DICOM_CODESYSTEM_URL))
140        return getVersionedSystem("dicom", systemVersion);
141      return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"),
142          systemVersion);
143    }
144
145    public String normalizeBaseURL(String baseUrl, String fullUrl) {
146      return fullUrl.substring(baseUrl.length()).replace("/", "");
147    }
148
149    public String getVersionedSystem(String baseSystem, String version) {
150      if (version != null) {
151        return baseSystem + "_" + version;
152      }
153      return baseSystem;
154    }
155  }
156
157  public class CacheToken {
158    private String name;
159    private String key;
160    private String request;
161
162    public void setName(String n) {
163      String systemName = getSystemNameKeyGenerator().getNameForSystem(n);
164      if (name == null)
165        name = systemName;
166      else if (!systemName.equals(name))
167        name = NAME_FOR_NO_SYSTEM;
168    }
169
170    public String getName() {
171      return name;
172    }
173  }
174
175  private class CacheEntry {
176    private String request;
177    private boolean persistent;
178    private ValidationResult v;
179    private ValueSetExpansionOutcome e;
180  }
181
182  private class NamedCache {
183    private String name;
184    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
185    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
186  }
187
188  private Object lock;
189  private String folder;
190  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
191
192  // use lock from the context
193  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
194    super();
195    this.lock = lock;
196    this.folder = folder;
197    if (folder != null)
198      load();
199  }
200
201  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
202    CacheToken ct = new CacheToken();
203    if (code.hasSystem())
204      ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem());
205    else
206      ct.name = NAME_FOR_NO_SYSTEM;
207    JsonParser json = new JsonParser();
208    json.setOutputStyle(OutputStyle.PRETTY);
209    ValueSet vsc = getVSEssense(vs);
210    try {
211      ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :"
212          + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}";
213    } catch (IOException e) {
214      throw new Error(e);
215    }
216    ct.key = String.valueOf(hashNWS(ct.request));
217    return ct;
218  }
219
220  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
221    CacheToken ct = new CacheToken();
222    for (Coding c : code.getCoding()) {
223      if (c.hasSystem())
224        ct.setName(c.getSystem());
225    }
226    JsonParser json = new JsonParser();
227    json.setOutputStyle(OutputStyle.PRETTY);
228    ValueSet vsc = getVSEssense(vs);
229    try {
230      ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :"
231          + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}";
232    } catch (IOException e) {
233      throw new Error(e);
234    }
235    ct.key = String.valueOf(hashNWS(ct.request));
236    return ct;
237  }
238
239  public ValueSet getVSEssense(ValueSet vs) {
240    if (vs == null)
241      return null;
242    ValueSet vsc = new ValueSet();
243    vsc.setCompose(vs.getCompose());
244    if (vs.hasExpansion()) {
245      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
246      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
247    }
248    return vsc;
249  }
250
251  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
252    CacheToken ct = new CacheToken();
253    ValueSet vsc = getVSEssense(vs);
254    for (ConceptSetComponent inc : vs.getCompose().getInclude())
255      if (inc.hasSystem())
256        ct.setName(inc.getSystem());
257    for (ConceptSetComponent inc : vs.getCompose().getExclude())
258      if (inc.hasSystem())
259        ct.setName(inc.getSystem());
260    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
261      if (inc.hasSystem())
262        ct.setName(inc.getSystem());
263    JsonParser json = new JsonParser();
264    json.setOutputStyle(OutputStyle.PRETTY);
265    try {
266      ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :"
267          + json.composeString(vsc) + "}\r\n";
268    } catch (IOException e) {
269      throw new Error(e);
270    }
271    ct.key = String.valueOf(hashNWS(ct.request));
272    return ct;
273  }
274
275  public NamedCache getNamedCache(CacheToken cacheToken) {
276    NamedCache nc = caches.get(cacheToken.name);
277    if (nc == null) {
278      nc = new NamedCache();
279      nc.name = cacheToken.name;
280      caches.put(nc.name, nc);
281    }
282    return nc;
283  }
284
285  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
286    synchronized (lock) {
287      NamedCache nc = getNamedCache(cacheToken);
288      CacheEntry e = nc.map.get(cacheToken.key);
289      if (e == null)
290        return null;
291      else
292        return e.e;
293    }
294  }
295
296  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
297    synchronized (lock) {
298      NamedCache nc = getNamedCache(cacheToken);
299      CacheEntry e = new CacheEntry();
300      e.request = cacheToken.request;
301      e.persistent = persistent;
302      e.e = res;
303      store(cacheToken, persistent, nc, e);
304    }
305  }
306
307  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
308    boolean n = nc.map.containsKey(cacheToken.key);
309    nc.map.put(cacheToken.key, e);
310    if (persistent) {
311      if (n) {
312        for (int i = nc.list.size() - 1; i >= 0; i--) {
313          if (nc.list.get(i).request.equals(e.request)) {
314            nc.list.remove(i);
315          }
316        }
317      }
318      nc.list.add(e);
319      save(nc);
320    }
321  }
322
323  public ValidationResult getValidation(CacheToken cacheToken) {
324    synchronized (lock) {
325      NamedCache nc = getNamedCache(cacheToken);
326      CacheEntry e = nc.map.get(cacheToken.key);
327      if (e == null)
328        return null;
329      else
330        return e.v;
331    }
332  }
333
334  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
335    synchronized (lock) {
336      NamedCache nc = getNamedCache(cacheToken);
337      CacheEntry e = new CacheEntry();
338      e.request = cacheToken.request;
339      e.persistent = persistent;
340      e.v = res;
341      store(cacheToken, persistent, nc, e);
342    }
343  }
344
345  // persistence
346
347  public void save() {
348
349  }
350
351  private void save(NamedCache nc) {
352    if (folder == null)
353      return;
354
355    try {
356      OutputStreamWriter sw = new OutputStreamWriter(ManagedFileAccess.outStream(Utilities.path(folder, nc.name + ".cache")),
357          "UTF-8");
358      sw.write(ENTRY_MARKER + "\r\n");
359      JsonParser json = new JsonParser();
360      json.setOutputStyle(OutputStyle.PRETTY);
361      for (CacheEntry ce : nc.list) {
362        sw.write(ce.request.trim());
363        sw.write(BREAK + "\r\n");
364        if (ce.e != null) {
365          sw.write("e: {\r\n");
366          if (ce.e.getValueset() != null)
367            sw.write("  \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n");
368          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n");
369        } else {
370          sw.write("v: {\r\n");
371          sw.write("  \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n");
372          sw.write("  \"severity\" : "
373              + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n");
374          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n");
375        }
376        sw.write(ENTRY_MARKER + "\r\n");
377      }
378      sw.close();
379    } catch (Exception e) {
380      System.out.println("error saving " + nc.name + ": " + e.getMessage());
381    }
382  }
383
384  private void load() throws FHIRException, IOException {
385    for (String fn : ManagedFileAccess.file(folder).list()) {
386      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
387        try {
388          // System.out.println("Load "+fn);
389          String title = fn.substring(0, fn.lastIndexOf("."));
390          NamedCache nc = new NamedCache();
391          nc.name = title;
392          caches.put(title, nc);
393          System.out.print(" - load " + title + ".cache");
394          String src = TextFile.fileToString(Utilities.path(folder, fn));
395          if (src.startsWith("?"))
396            src = src.substring(1);
397          int i = src.indexOf(ENTRY_MARKER);
398          while (i > -1) {
399            String s = src.substring(0, i);
400            System.out.print(".");
401            src = src.substring(i + ENTRY_MARKER.length() + 1);
402            i = src.indexOf(ENTRY_MARKER);
403            if (!Utilities.noString(s)) {
404              int j = s.indexOf(BREAK);
405              String q = s.substring(0, j);
406              String p = s.substring(j + BREAK.length() + 1).trim();
407              CacheEntry ce = new CacheEntry();
408              ce.persistent = true;
409              ce.request = q;
410              boolean e = p.charAt(0) == 'e';
411              p = p.substring(3);
412              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
413              String error = loadJS(o.get("error"));
414              if (e) {
415                if (o.has("valueSet"))
416                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")),
417                      error, TerminologyServiceErrorClass.UNKNOWN);
418                else
419                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
420              } else {
421                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null
422                    : IssueSeverity.fromCode(o.get("severity").getAsString());
423                String display = loadJS(o.get("display"));
424                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
425              }
426              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
427              nc.list.add(ce);
428            }
429          }
430          System.out.println("done");
431        } catch (Exception e) {
432          throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e);
433        }
434      }
435    }
436  }
437
438  private String loadJS(JsonElement e) {
439    if (e == null)
440      return null;
441    if (!(e instanceof JsonPrimitive))
442      return null;
443    String s = e.getAsString();
444    if ("".equals(s))
445      return null;
446    return s;
447  }
448
449  private String hashNWS(String s) {
450    s = StringUtils.remove(s, ' ');
451    s = StringUtils.remove(s, '\n');
452    s = StringUtils.remove(s, '\r');
453    return String.valueOf(s.hashCode());
454  }
455
456  // management
457
458  public TerminologyCache copy() {
459    // TODO Auto-generated method stub
460    return null;
461  }
462
463  public String summary(ValueSet vs) {
464    if (vs == null)
465      return "null";
466
467    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
468    for (ConceptSetComponent cc : vs.getCompose().getInclude())
469      b.append("Include " + getIncSummary(cc));
470    for (ConceptSetComponent cc : vs.getCompose().getExclude())
471      b.append("Exclude " + getIncSummary(cc));
472    return b.toString();
473  }
474
475  private String getIncSummary(ConceptSetComponent cc) {
476    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
477    for (UriType vs : cc.getValueSet())
478      b.append(vs.asStringValue());
479    String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
480    String system = cc.getSystem();
481    if (cc.hasConcept())
482      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
483    if (cc.hasFilter()) {
484      String s = "";
485      for (ConceptSetFilterComponent f : cc.getFilter()) {
486        if (!Utilities.noString(s))
487          s = s + " & ";
488        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
489      }
490      return "from " + system + " where " + s + vsd;
491    }
492    return "All codes from " + system + vsd;
493  }
494
495  public String summary(Coding code) {
496    return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\"";
497  }
498
499  public String summary(CodeableConcept code) {
500    StringBuilder b = new StringBuilder();
501    b.append("{");
502    boolean first = true;
503    for (Coding c : code.getCoding()) {
504      if (first)
505        first = false;
506      else
507        b.append(",");
508      b.append(summary(c));
509    }
510    b.append("}: \"");
511    b.append(code.getText());
512    b.append("\"");
513    return b.toString();
514  }
515
516}