Source code

001package org.hl7.fhir.convertors.misc;
002
003import java.io.File;
004import java.io.FileNotFoundException;
005import java.io.IOException;
006import java.net.URISyntaxException;
007import java.text.ParseException;
008import java.util.ArrayList;
009import java.util.Collections;
010import java.util.HashMap;
011import java.util.List;
012import java.util.Map;
013
014import org.hl7.fhir.exceptions.FHIRException;
015import org.hl7.fhir.r4.formats.IParser.OutputStyle;
016import org.hl7.fhir.r4.formats.JsonParser;
017import org.hl7.fhir.r4.model.CapabilityStatement;
018import org.hl7.fhir.r4.model.CodeSystem;
019import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
020import org.hl7.fhir.r4.model.IntegerType;
021import org.hl7.fhir.r4.model.OperationOutcome;
022import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
023import org.hl7.fhir.r4.model.OperationOutcome.IssueType;
024import org.hl7.fhir.r4.model.Parameters;
025import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent;
026import org.hl7.fhir.r4.model.UriType;
027import org.hl7.fhir.r4.model.ValueSet;
028import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent;
029import org.hl7.fhir.r4.terminologies.JurisdictionUtilities;
030import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
031import org.hl7.fhir.utilities.CSVReader;
032import org.hl7.fhir.utilities.IniFile;
033import org.hl7.fhir.utilities.Utilities;
034import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
035import org.hl7.fhir.utilities.http.ManagedWebAccess;
036
037@SuppressWarnings("checkstyle:systemout")
038public class VSACImporter extends OIDBasedValueSetImporter {
039
040  public VSACImporter() throws FHIRException, IOException {
041    super();
042    init();
043  }
044
045  public static void main(String[] args) throws FHIRException, IOException, ParseException, URISyntaxException {
046    VSACImporter self = new VSACImporter();
047    self.process(args[0], args[1], "true".equals(args[2]), "true".equals(args[3]));
048  }
049
050  private void process(String source, String dest, boolean onlyNew, boolean onlyActive) throws FHIRException, IOException, URISyntaxException {
051    CSVReader csv = new CSVReader(ManagedFileAccess.inStream(source));
052    csv.readHeaders();
053    Map<String, String> errs = new HashMap<>();
054
055    ManagedWebAccess.loadFromFHIRSettings();
056    FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
057    fhirToolingClient.setTimeoutNormal(30000);
058    fhirToolingClient.setTimeoutExpand(30000);
059
060    CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement();
061    JsonParser json = new JsonParser();
062    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path("[tmp]", "vsac-capability-statement.json")), cs);
063
064    System.out.println("CodeSystems");
065    CodeSystem css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCREC");
066    checkHierarchy(fhirToolingClient, css);
067    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCREC.json")), css);
068    css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCNHSN");
069    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCNHSN.json")), css);
070    css = fhirToolingClient.fetchResource(CodeSystem.class, "HSLOC");
071    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-HSLOC.json")), css);
072    css = fhirToolingClient.fetchResource(CodeSystem.class, "SOP");
073    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-SOP.json")), css);
074
075    System.out.println("Loading");
076    List<String> oids = new ArrayList<>();
077    List<String> allOids = new ArrayList<>();
078    while (csv.line()) {
079      String status = csv.cell("Expansion Status");
080      if (!onlyActive || "Active".equals(status)) {
081        String oid = csv.cell("OID");
082        allOids.add(oid);
083        if (!onlyNew || !(ManagedFileAccess.file(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
084          oids.add(oid);
085        }
086      }
087    }
088    Collections.sort(oids);
089    System.out.println("Cleaning");
090    cleanValueSets(allOids, dest);
091    System.out.println("Go: "+oids.size()+" oids");
092    int i = 0;
093    int j = 0;
094    long t = System.currentTimeMillis();
095    long tt = System.currentTimeMillis();
096    for (String oid : oids) {
097      try {
098        long t3 = System.currentTimeMillis();
099        if (processOid(dest, onlyNew, errs, fhirToolingClient, oid.trim())) {
100          j++;
101        }
102        i++;
103        System.out.print(":"+((System.currentTimeMillis() - t3) / 1000));
104        if (i % 100 == 0) {
105          long elapsed = System.currentTimeMillis() - t;
106          System.out.println("");
107          System.out.println(i+": "+j+" ("+((j * 100) / i)+"%) @ "+Utilities.describeDuration(elapsed)
108          +", "+(elapsed/100000)+"sec/vs, estimated "+Utilities.describeDuration(estimate(i, oids.size(), tt))+" remaining");
109          t = System.currentTimeMillis();
110        }
111      } catch (Exception e) {
112        e.printStackTrace();
113        System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage());
114        errs.put(oid, e.getMessage());
115      }
116    }
117
118    OperationOutcome oo = new OperationOutcome();
119    for (String oid : errs.keySet()) {
120      oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid);
121    }
122    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
123    System.out.println();
124    System.out.println("Done. " + i + " ValueSets in "+Utilities.describeDuration(System.currentTimeMillis() - tt));
125  }
126
127  private void checkHierarchy(FHIRToolingClient client, CodeSystem css) {
128    IniFile ini = new IniFile("/Users/grahamegrieve/temp/vsac-cdc-rec.ini");
129    System.out.println("hierarchy:");
130    List<ConceptDefinitionComponent> codes = new ArrayList<>();
131    int c = 0;
132    List<ConceptDefinitionComponent> list = new ArrayList<>();
133    Map<String,ConceptDefinitionComponent> map = new HashMap<>();
134    for (ConceptDefinitionComponent cc : css.getConcept()) {
135      list.add(cc);
136      map.put(cc.getCode(), cc);
137    }
138    css.getConcept().clear();
139    for (ConceptDefinitionComponent cc : list) {
140      String code = cc.getCode();
141      ConceptDefinitionComponent parent = map.get(ini.getStringProperty("parents-"+css.getVersion(), code));
142      if (parent == null) {
143        parent = findParent(client, css, css.getConcept(), code);
144        if (parent == null) {
145          ini.setStringProperty("parents-"+css.getVersion(), code, "null", null);
146        } else {
147          ini.setStringProperty("parents-"+css.getVersion(), code, parent.getCode(), null);
148        }
149        ini.save();
150      }
151      if (parent == null) {
152        css.getConcept().add(cc);
153      } else {
154        parent.getConcept().add(cc);        
155      }
156
157      codes.add(0, cc);
158      c++;
159      if (c % 20 == 0) {
160        System.out.print("."+c);
161      }
162    }    
163    System.out.println("Done. "+c+" concepts");
164  }
165
166  private ConceptDefinitionComponent findParent(FHIRToolingClient client, CodeSystem css, List<ConceptDefinitionComponent> concepts, String code) {
167    if (concepts.size() == 0) {
168      return null;
169    }
170    ConceptDefinitionComponent last = concepts.get(concepts.size() -1);
171    if (last.hasConcept()) {
172      ConceptDefinitionComponent parent = findParent(client, css, last.getConcept(), code);
173      if (parent != null) {
174        return parent;
175      }      
176    } 
177
178    String reln = getRelationship(client, css, code, last.getCode());
179    if ("subsumes".equals(reln)) {
180      return last;
181    } else {
182      for (ConceptDefinitionComponent cc : concepts) {
183        reln = getRelationship(client, css, code, cc.getCode());
184        if ("subsumes".equals(reln)) {
185          return cc;        
186        }
187      }
188      return null;
189    }
190  }
191
192      
193  private String getRelationship(FHIRToolingClient client, CodeSystem css, String codeA, String codeB) {
194    Map<String, String> params = new HashMap<>();
195    params.put("system", css.getUrl());
196    params.put("codeA", codeB);
197    params.put("codeB", codeA);    
198    Parameters p = client.subsumes(params);
199    for (ParametersParameterComponent pp : p.getParameter()) {
200      if ("outcome".equals(pp.getName())) {
201        return pp.getValue().primitiveValue();
202      }
203    }
204    return null;
205  }
206
207  private void cleanValueSets(List<String> allOids, String dest) throws IOException {
208    cleanValueSets(allOids, ManagedFileAccess.file(Utilities.path(dest)));
209  }
210
211  private void cleanValueSets(List<String> allOids, File file) {
212    for (File f : file.listFiles()) {
213      if (f.getName().startsWith("ValueSet-")) {
214        String oid = f.getName().substring(9).replace(".json", "");
215        if (!allOids.contains(oid)) {
216          f.delete();
217        }
218      }
219    }
220
221  }
222
223  private long estimate(int i, int size, long tt) {
224    long elapsed = System.currentTimeMillis() - tt;
225    long average = elapsed / i;
226    return (size - i) * average;
227  }
228
229  private boolean processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, String oid)
230      throws IOException, InterruptedException, FileNotFoundException {
231
232    while (true) {
233      boolean ok = true;
234      long t = System.currentTimeMillis();
235      ValueSet vs = null;
236      try {
237        vs = fhirToolingClient.read(ValueSet.class, oid);
238      } catch (Exception e) {
239        if (e.getMessage().contains("timed out")) {
240          ok = false;
241        } else {
242          errs.put(oid, "Read: " +e.getMessage());
243          System.out.println("Read "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
244          return false;
245        }
246      }
247      if (ok) {
248        t = System.currentTimeMillis();
249        try {
250          Parameters p = new Parameters();
251          p.addParameter("url", new UriType(vs.getUrl()));
252          ValueSet vse = fhirToolingClient.expandValueset(null, p);
253          vs.setExpansion(vse.getExpansion());
254        } catch (Exception e) {
255          if (e.getMessage().contains("timed out")) {
256            ok = false;
257          } else {
258            errs.put(oid, "Expansion: " +e.getMessage());
259            System.out.println("Expand "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
260            return false;
261          }
262        }
263      }
264      if (ok) {
265        while (isIncomplete(vs.getExpansion())) {
266          Parameters p = new Parameters();
267          int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue();
268          p.addParameter("offset", offset);
269          p.addParameter("url", new UriType(vs.getUrl()));
270          t = System.currentTimeMillis();
271          try {
272            ValueSet vse = fhirToolingClient.expandValueset(null, p);    
273            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
274            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
275          } catch (Exception e2) {
276            if (e2.getMessage().contains("timed out")) {
277              ok = false;
278              break;
279            } else {
280              errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset);
281              System.out.println("Expand "+oid+" @ "+offset+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e2.getMessage());
282              return false;
283            }
284          } 
285        }
286      }
287      if (ok) {
288        vs.getExpansion().setOffsetElement(null);
289        vs.getExpansion().getParameter().clear();
290
291        if (vs.hasTitle()) {
292          if (vs.getTitle().equals(vs.getDescription())) {
293            vs.setTitle(vs.getName());              
294          } else {
295            //              System.out.println(oid);
296            //              System.out.println("  name: "+vs.getName());
297            //              System.out.println("  title: "+vs.getTitle());
298            //              System.out.println("  desc: "+vs.getDescription());
299          }
300        } else {
301          vs.setTitle(vs.getName());
302        }
303        if (vs.getUrl().startsWith("https://")) {
304          System.out.println("URL is https: "+vs.getUrl());
305        }
306        vs.setName(makeValidName(vs.getName()));
307        JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
308        new JsonParser().setOutputStyle(OutputStyle.NORMAL).compose(ManagedFileAccess.outStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
309        return true;
310      }
311    }
312  }
313
314  private boolean isIncomplete(ValueSetExpansionComponent expansion) {
315    IntegerType c = expansion.getParameter("count") != null ? expansion.getParameter("count").getValueIntegerType() : new IntegerType(0);
316    IntegerType offset = expansion.getParameter("offset") != null ? expansion.getParameter("offset").getValueIntegerType() : new IntegerType(0);
317    return c.getValue() + offset.getValue() < expansion.getTotal();
318  }
319
320  private String makeValidName(String name) {
321    StringBuilder b = new StringBuilder();
322    boolean upper = true;
323    for (char ch : name.toCharArray()) {
324      if (ch == ' ') {
325        upper = true;
326      } else if (Character.isAlphabetic(ch)) {
327        if (upper) {
328          b.append(Character.toUpperCase(ch));
329        } else {
330          b.append(ch);
331        }
332        upper = false;
333      } else if (Character.isDigit(ch)) {
334        if (b.length() == 0) {
335          b.append('N');
336        }
337        b.append(ch);
338      } else if (ch == '_' && b.length() != 0) {
339        b.append(ch);
340      } else {
341        upper = true;
342      }
343    }
344    //    System.out.println(b.toString()+" from "+name);
345    return b.toString();
346  }
347}