001package org.hl7.fhir.r4.context; 002 003/* 004 Copyright (c) 2011+, HL7, Inc. 005 All rights reserved. 006 007 Redistribution and use in source and binary forms, with or without modification, 008 are permitted provided that the following conditions are met: 009 010 * Redistributions of source code must retain the above copyright notice, this 011 list of conditions and the following disclaimer. 012 * Redistributions in binary form must reproduce the above copyright notice, 013 this list of conditions and the following disclaimer in the documentation 014 and/or other materials provided with the distribution. 015 * Neither the name of HL7 nor the names of its contributors may be used to 016 endorse or promote products derived from this software without specific 017 prior written permission. 018 019 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 022 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 023 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 024 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 025 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 026 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 027 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 028 POSSIBILITY OF SUCH DAMAGE. 029 030 */ 031 032import java.io.File; 033import java.io.FileNotFoundException; 034import java.io.FileOutputStream; 035import java.io.IOException; 036import java.io.OutputStreamWriter; 037import java.util.ArrayList; 038import java.util.HashMap; 039import java.util.List; 040import java.util.Map; 041 042import org.apache.commons.lang3.StringUtils; 043import org.hl7.fhir.exceptions.FHIRException; 044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult; 045import org.hl7.fhir.r4.formats.IParser.OutputStyle; 046import org.hl7.fhir.r4.formats.JsonParser; 047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; 048import org.hl7.fhir.r4.model.CodeableConcept; 049import org.hl7.fhir.r4.model.Coding; 050import org.hl7.fhir.r4.model.UriType; 051import org.hl7.fhir.r4.model.ValueSet; 052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; 053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent; 054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent; 055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass; 056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; 057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; 058import org.hl7.fhir.utilities.TextFile; 059import org.hl7.fhir.utilities.Utilities; 060import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 061import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; 062import org.hl7.fhir.utilities.validation.ValidationOptions; 063 064import com.google.gson.JsonElement; 065import com.google.gson.JsonNull; 066import com.google.gson.JsonObject; 067import com.google.gson.JsonPrimitive; 068 069/** 070 * This implements a two level cache. - a temporary cache for remmbering 071 * previous local operations - a persistent cache for rembering tx server 072 * operations 073 * 074 * the cache is a series of pairs: a map, and a list. the map is the loaded 075 * cache, the list is the persiistent cache, carefully maintained in order for 076 * version control consistency 077 * 078 * @author graha 079 * 080 */ 081public class TerminologyCache { 082 public static final boolean TRANSIENT = false; 083 public static final boolean PERMANENT = true; 084 private static final String NAME_FOR_NO_SYSTEM = "all-systems"; 085 private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------"; 086 private static final String BREAK = "####"; 087 088 private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator(); 089 090 protected SystemNameKeyGenerator getSystemNameKeyGenerator() { 091 return systemNameKeyGenerator; 092 } 093 094 public class SystemNameKeyGenerator { 095 public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct"; 096 public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm"; 097 public static final String LOINC_CODESYSTEM_URL = "http://loinc.org"; 098 public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org"; 099 100 public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/"; 101 public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/"; 102 public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/"; 103 104 public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:"; 105 public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47"; 106 public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13"; 107 108 public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101"; 109 public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM"; 110 111 public String getNameForSystem(String system) { 112 final int lastPipe = system.lastIndexOf('|'); 113 final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe); 114 final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1); 115 116 if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL)) 117 return getVersionedSystem("snomed", systemVersion); 118 if (systemBaseName.equals(RXNORM_CODESYSTEM_URL)) 119 return getVersionedSystem("rxnorm", systemVersion); 120 if (systemBaseName.equals(LOINC_CODESYSTEM_URL)) 121 return getVersionedSystem("loinc", systemVersion); 122 if (systemBaseName.equals(UCUM_CODESYSTEM_URL)) 123 return getVersionedSystem("ucum", systemVersion); 124 if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL)) 125 return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 126 if (systemBaseName.equals(_11073_CODESYSTEM_URN)) 127 return getVersionedSystem("11073", systemVersion); 128 if (systemBaseName.startsWith(ISO_CODESYSTEM_URN)) 129 return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""), 130 systemVersion); 131 if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL)) 132 return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 133 if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL)) 134 return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 135 if (systemBaseName.equals(LANG_CODESYSTEM_URN)) 136 return getVersionedSystem("lang", systemVersion); 137 if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN)) 138 return getVersionedSystem("mimetypes", systemVersion); 139 if (systemBaseName.equals(DICOM_CODESYSTEM_URL)) 140 return getVersionedSystem("dicom", systemVersion); 141 return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"), 142 systemVersion); 143 } 144 145 public String normalizeBaseURL(String baseUrl, String fullUrl) { 146 return fullUrl.substring(baseUrl.length()).replace("/", ""); 147 } 148 149 public String getVersionedSystem(String baseSystem, String version) { 150 if (version != null) { 151 return baseSystem + "_" + version; 152 } 153 return baseSystem; 154 } 155 } 156 157 public class CacheToken { 158 private String name; 159 private String key; 160 private String request; 161 162 public void setName(String n) { 163 String systemName = getSystemNameKeyGenerator().getNameForSystem(n); 164 if (name == null) 165 name = systemName; 166 else if (!systemName.equals(name)) 167 name = NAME_FOR_NO_SYSTEM; 168 } 169 170 public String getName() { 171 return name; 172 } 173 } 174 175 private class CacheEntry { 176 private String request; 177 private boolean persistent; 178 private ValidationResult v; 179 private ValueSetExpansionOutcome e; 180 } 181 182 private class NamedCache { 183 private String name; 184 private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries 185 private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>(); 186 } 187 188 private Object lock; 189 private String folder; 190 private Map<String, NamedCache> caches = new HashMap<String, NamedCache>(); 191 192 // use lock from the context 193 public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException { 194 super(); 195 this.lock = lock; 196 this.folder = folder; 197 if (folder != null) 198 load(); 199 } 200 201 public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) { 202 CacheToken ct = new CacheToken(); 203 if (code.hasSystem()) 204 ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem()); 205 else 206 ct.name = NAME_FOR_NO_SYSTEM; 207 JsonParser json = new JsonParser(); 208 json.setOutputStyle(OutputStyle.PRETTY); 209 ValueSet vsc = getVSEssense(vs); 210 try { 211 ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :" 212 + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}"; 213 } catch (IOException e) { 214 throw new Error(e); 215 } 216 ct.key = String.valueOf(hashNWS(ct.request)); 217 return ct; 218 } 219 220 public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) { 221 CacheToken ct = new CacheToken(); 222 for (Coding c : code.getCoding()) { 223 if (c.hasSystem()) 224 ct.setName(c.getSystem()); 225 } 226 JsonParser json = new JsonParser(); 227 json.setOutputStyle(OutputStyle.PRETTY); 228 ValueSet vsc = getVSEssense(vs); 229 try { 230 ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :" 231 + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}"; 232 } catch (IOException e) { 233 throw new Error(e); 234 } 235 ct.key = String.valueOf(hashNWS(ct.request)); 236 return ct; 237 } 238 239 public ValueSet getVSEssense(ValueSet vs) { 240 if (vs == null) 241 return null; 242 ValueSet vsc = new ValueSet(); 243 vsc.setCompose(vs.getCompose()); 244 if (vs.hasExpansion()) { 245 vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter()); 246 vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains()); 247 } 248 return vsc; 249 } 250 251 public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) { 252 CacheToken ct = new CacheToken(); 253 ValueSet vsc = getVSEssense(vs); 254 for (ConceptSetComponent inc : vs.getCompose().getInclude()) 255 if (inc.hasSystem()) 256 ct.setName(inc.getSystem()); 257 for (ConceptSetComponent inc : vs.getCompose().getExclude()) 258 if (inc.hasSystem()) 259 ct.setName(inc.getSystem()); 260 for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains()) 261 if (inc.hasSystem()) 262 ct.setName(inc.getSystem()); 263 JsonParser json = new JsonParser(); 264 json.setOutputStyle(OutputStyle.PRETTY); 265 try { 266 ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :" 267 + json.composeString(vsc) + "}\r\n"; 268 } catch (IOException e) { 269 throw new Error(e); 270 } 271 ct.key = String.valueOf(hashNWS(ct.request)); 272 return ct; 273 } 274 275 public NamedCache getNamedCache(CacheToken cacheToken) { 276 NamedCache nc = caches.get(cacheToken.name); 277 if (nc == null) { 278 nc = new NamedCache(); 279 nc.name = cacheToken.name; 280 caches.put(nc.name, nc); 281 } 282 return nc; 283 } 284 285 public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) { 286 synchronized (lock) { 287 NamedCache nc = getNamedCache(cacheToken); 288 CacheEntry e = nc.map.get(cacheToken.key); 289 if (e == null) 290 return null; 291 else 292 return e.e; 293 } 294 } 295 296 public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) { 297 synchronized (lock) { 298 NamedCache nc = getNamedCache(cacheToken); 299 CacheEntry e = new CacheEntry(); 300 e.request = cacheToken.request; 301 e.persistent = persistent; 302 e.e = res; 303 store(cacheToken, persistent, nc, e); 304 } 305 } 306 307 public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) { 308 boolean n = nc.map.containsKey(cacheToken.key); 309 nc.map.put(cacheToken.key, e); 310 if (persistent) { 311 if (n) { 312 for (int i = nc.list.size() - 1; i >= 0; i--) { 313 if (nc.list.get(i).request.equals(e.request)) { 314 nc.list.remove(i); 315 } 316 } 317 } 318 nc.list.add(e); 319 save(nc); 320 } 321 } 322 323 public ValidationResult getValidation(CacheToken cacheToken) { 324 synchronized (lock) { 325 NamedCache nc = getNamedCache(cacheToken); 326 CacheEntry e = nc.map.get(cacheToken.key); 327 if (e == null) 328 return null; 329 else 330 return e.v; 331 } 332 } 333 334 public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) { 335 synchronized (lock) { 336 NamedCache nc = getNamedCache(cacheToken); 337 CacheEntry e = new CacheEntry(); 338 e.request = cacheToken.request; 339 e.persistent = persistent; 340 e.v = res; 341 store(cacheToken, persistent, nc, e); 342 } 343 } 344 345 // persistence 346 347 public void save() { 348 349 } 350 351 private void save(NamedCache nc) { 352 if (folder == null) 353 return; 354 355 try { 356 OutputStreamWriter sw = new OutputStreamWriter(ManagedFileAccess.outStream(Utilities.path(folder, nc.name + ".cache")), 357 "UTF-8"); 358 sw.write(ENTRY_MARKER + "\r\n"); 359 JsonParser json = new JsonParser(); 360 json.setOutputStyle(OutputStyle.PRETTY); 361 for (CacheEntry ce : nc.list) { 362 sw.write(ce.request.trim()); 363 sw.write(BREAK + "\r\n"); 364 if (ce.e != null) { 365 sw.write("e: {\r\n"); 366 if (ce.e.getValueset() != null) 367 sw.write(" \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n"); 368 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n"); 369 } else { 370 sw.write("v: {\r\n"); 371 sw.write(" \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n"); 372 sw.write(" \"severity\" : " 373 + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n"); 374 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n"); 375 } 376 sw.write(ENTRY_MARKER + "\r\n"); 377 } 378 sw.close(); 379 } catch (Exception e) { 380 System.out.println("error saving " + nc.name + ": " + e.getMessage()); 381 } 382 } 383 384 private void load() throws FHIRException, IOException { 385 for (String fn : ManagedFileAccess.file(folder).list()) { 386 if (fn.endsWith(".cache") && !fn.equals("validation.cache")) { 387 try { 388 // System.out.println("Load "+fn); 389 String title = fn.substring(0, fn.lastIndexOf(".")); 390 NamedCache nc = new NamedCache(); 391 nc.name = title; 392 caches.put(title, nc); 393 System.out.print(" - load " + title + ".cache"); 394 String src = TextFile.fileToString(Utilities.path(folder, fn)); 395 if (src.startsWith("?")) 396 src = src.substring(1); 397 int i = src.indexOf(ENTRY_MARKER); 398 while (i > -1) { 399 String s = src.substring(0, i); 400 System.out.print("."); 401 src = src.substring(i + ENTRY_MARKER.length() + 1); 402 i = src.indexOf(ENTRY_MARKER); 403 if (!Utilities.noString(s)) { 404 int j = s.indexOf(BREAK); 405 String q = s.substring(0, j); 406 String p = s.substring(j + BREAK.length() + 1).trim(); 407 CacheEntry ce = new CacheEntry(); 408 ce.persistent = true; 409 ce.request = q; 410 boolean e = p.charAt(0) == 'e'; 411 p = p.substring(3); 412 JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p); 413 String error = loadJS(o.get("error")); 414 if (e) { 415 if (o.has("valueSet")) 416 ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), 417 error, TerminologyServiceErrorClass.UNKNOWN); 418 else 419 ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN); 420 } else { 421 IssueSeverity severity = o.get("severity") instanceof JsonNull ? null 422 : IssueSeverity.fromCode(o.get("severity").getAsString()); 423 String display = loadJS(o.get("display")); 424 ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display)); 425 } 426 nc.map.put(String.valueOf(hashNWS(ce.request)), ce); 427 nc.list.add(ce); 428 } 429 } 430 System.out.println("done"); 431 } catch (Exception e) { 432 throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e); 433 } 434 } 435 } 436 } 437 438 private String loadJS(JsonElement e) { 439 if (e == null) 440 return null; 441 if (!(e instanceof JsonPrimitive)) 442 return null; 443 String s = e.getAsString(); 444 if ("".equals(s)) 445 return null; 446 return s; 447 } 448 449 private String hashNWS(String s) { 450 s = StringUtils.remove(s, ' '); 451 s = StringUtils.remove(s, '\n'); 452 s = StringUtils.remove(s, '\r'); 453 return String.valueOf(s.hashCode()); 454 } 455 456 // management 457 458 public TerminologyCache copy() { 459 // TODO Auto-generated method stub 460 return null; 461 } 462 463 public String summary(ValueSet vs) { 464 if (vs == null) 465 return "null"; 466 467 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 468 for (ConceptSetComponent cc : vs.getCompose().getInclude()) 469 b.append("Include " + getIncSummary(cc)); 470 for (ConceptSetComponent cc : vs.getCompose().getExclude()) 471 b.append("Exclude " + getIncSummary(cc)); 472 return b.toString(); 473 } 474 475 private String getIncSummary(ConceptSetComponent cc) { 476 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 477 for (UriType vs : cc.getValueSet()) 478 b.append(vs.asStringValue()); 479 String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : ""; 480 String system = cc.getSystem(); 481 if (cc.hasConcept()) 482 return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd; 483 if (cc.hasFilter()) { 484 String s = ""; 485 for (ConceptSetFilterComponent f : cc.getFilter()) { 486 if (!Utilities.noString(s)) 487 s = s + " & "; 488 s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue(); 489 } 490 return "from " + system + " where " + s + vsd; 491 } 492 return "All codes from " + system + vsd; 493 } 494 495 public String summary(Coding code) { 496 return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\""; 497 } 498 499 public String summary(CodeableConcept code) { 500 StringBuilder b = new StringBuilder(); 501 b.append("{"); 502 boolean first = true; 503 for (Coding c : code.getCoding()) { 504 if (first) 505 first = false; 506 else 507 b.append(","); 508 b.append(summary(c)); 509 } 510 b.append("}: \""); 511 b.append(code.getText()); 512 b.append("\""); 513 return b.toString(); 514 } 515 516}