Source code

001package org.hl7.fhir.convertors.misc;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.FileNotFoundException;
006import java.io.FileOutputStream;
007import java.io.IOException;
008import java.net.URISyntaxException;
009import java.text.ParseException;
010import java.time.Duration;
011import java.util.ArrayList;
012import java.util.Collections;
013import java.util.HashMap;
014import java.util.List;
015import java.util.Map;
016
017import org.hl7.fhir.exceptions.FHIRException;
018import org.hl7.fhir.r4.formats.IParser.OutputStyle;
019import org.hl7.fhir.r4.formats.JsonParser;
020import org.hl7.fhir.r4.model.CapabilityStatement;
021import org.hl7.fhir.r4.model.CodeSystem;
022import org.hl7.fhir.r4.model.IntegerType;
023import org.hl7.fhir.r4.model.OperationOutcome;
024import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
025import org.hl7.fhir.r4.model.OperationOutcome.IssueType;
026import org.hl7.fhir.r4.model.Parameters;
027import org.hl7.fhir.r4.model.UriType;
028import org.hl7.fhir.r4.model.ValueSet;
029import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent;
030import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
031import org.hl7.fhir.r4.terminologies.JurisdictionUtilities;
032import org.hl7.fhir.utilities.CSVReader;
033import org.hl7.fhir.utilities.Utilities;
034import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
035import org.hl7.fhir.utilities.http.ManagedWebAccess;
036import org.hl7.fhir.utilities.json.model.JsonArray;
037import org.hl7.fhir.utilities.json.model.JsonObject;
038import org.hl7.fhir.utilities.json.model.JsonProperty;
039
040public class VSACImporter extends OIDBasedValueSetImporter {
041  
042  public VSACImporter() throws FHIRException, IOException {
043    super();
044    init();
045  }
046
047  public static void main(String[] args) throws FHIRException, IOException, ParseException, URISyntaxException {
048    VSACImporter self = new VSACImporter();
049    self.process(args[0], args[1], "true".equals(args[2]), "true".equals(args[3]));
050  }
051
052  private void process(String source, String dest, boolean onlyNew, boolean onlyActive) throws FHIRException, IOException, URISyntaxException {
053    CSVReader csv = new CSVReader(ManagedFileAccess.inStream(source));
054    csv.readHeaders();
055    Map<String, String> errs = new HashMap<>();
056
057    ManagedWebAccess.loadFromFHIRSettings();
058    FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
059    fhirToolingClient.setTimeoutNormal(30000);
060    fhirToolingClient.setTimeoutExpand(30000);
061
062    CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement();
063    JsonParser json = new JsonParser();
064    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path("[tmp]", "vsac-capability-statement.json")), cs);
065
066    System.out.println("CodeSystems");
067    CodeSystem css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCNHSN");
068    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCNHSN.json")), css);
069    css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCREC");
070    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCREC.json")), css);
071    css = fhirToolingClient.fetchResource(CodeSystem.class, "HSLOC");
072    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-HSLOC.json")), css);
073    css = fhirToolingClient.fetchResource(CodeSystem.class, "SOP");
074    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-SOP.json")), css);
075    
076    System.out.println("Loading");
077    List<String> oids = new ArrayList<>();
078    List<String> allOids = new ArrayList<>();
079    while (csv.line()) {
080      String status = csv.cell("Expansion Status");
081      if (!onlyActive || "Active".equals(status)) {
082        String oid = csv.cell("OID");
083        allOids.add(oid);
084        if (!onlyNew || !(ManagedFileAccess.file(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
085          oids.add(oid);
086        }
087      }
088    }
089    Collections.sort(oids);
090    System.out.println("Cleaning");
091    cleanValueSets(allOids, dest);
092    System.out.println("Go: "+oids.size()+" oids");
093    int i = 0;
094    int j = 0;
095    long t = System.currentTimeMillis();
096    long tt = System.currentTimeMillis();
097    for (String oid : oids) {
098      try {
099        long t3 = System.currentTimeMillis();
100        if (processOid(dest, onlyNew, errs, fhirToolingClient, oid.trim())) {
101          j++;
102        }
103        i++;
104        System.out.print(":"+((System.currentTimeMillis() - t3) / 1000));
105        if (i % 100 == 0) {
106          long elapsed = System.currentTimeMillis() - t;
107          System.out.println("");
108          System.out.println(i+": "+j+" ("+((j * 100) / i)+"%) @ "+Utilities.describeDuration(elapsed)
109              +", "+(elapsed/100000)+"sec/vs, estimated "+Utilities.describeDuration(estimate(i, oids.size(), tt))+" remaining");
110          t = System.currentTimeMillis();
111        }
112      } catch (Exception e) {
113        e.printStackTrace();
114        System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage());
115        errs.put(oid, e.getMessage());
116      }
117    }
118    
119    OperationOutcome oo = new OperationOutcome();
120    for (String oid : errs.keySet()) {
121      oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid);
122    }
123    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
124    System.out.println();
125    System.out.println("Done. " + i + " ValueSets in "+Utilities.describeDuration(System.currentTimeMillis() - tt));
126  }
127
128  private void cleanValueSets(List<String> allOids, String dest) throws IOException {
129    cleanValueSets(allOids, ManagedFileAccess.file(Utilities.path(dest)));
130  }
131
132  private void cleanValueSets(List<String> allOids, File file) {
133    for (File f : file.listFiles()) {
134      if (f.getName().startsWith("ValueSet-")) {
135        String oid = f.getName().substring(9).replace(".json", "");
136        if (!allOids.contains(oid)) {
137          f.delete();
138        }
139      }
140    }
141    
142  }
143
144  private long estimate(int i, int size, long tt) {
145    long elapsed = System.currentTimeMillis() - tt;
146    long average = elapsed / i;
147    return (size - i) * average;
148  }
149
150  private boolean processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, String oid)
151      throws IOException, InterruptedException, FileNotFoundException {
152    
153    while (true) {
154      boolean ok = true;
155      long t = System.currentTimeMillis();
156      ValueSet vs = null;
157      try {
158        vs = fhirToolingClient.read(ValueSet.class, oid);
159      } catch (Exception e) {
160        if (e.getMessage().contains("timed out")) {
161          ok = false;
162        } else {
163          errs.put(oid, "Read: " +e.getMessage());
164          System.out.println("Read "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
165          return false;
166        }
167      }
168      if (ok) {
169        t = System.currentTimeMillis();
170        try {
171          Parameters p = new Parameters();
172          p.addParameter("url", new UriType(vs.getUrl()));
173          ValueSet vse = fhirToolingClient.expandValueset(null, p);
174          vs.setExpansion(vse.getExpansion());
175        } catch (Exception e) {
176          if (e.getMessage().contains("timed out")) {
177            ok = false;
178          } else {
179            errs.put(oid, "Expansion: " +e.getMessage());
180            System.out.println("Expand "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
181            return false;
182          }
183        }
184      }
185      if (ok) {
186        while (isIncomplete(vs.getExpansion())) {
187          Parameters p = new Parameters();
188          int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue();
189          p.addParameter("offset", offset);
190          p.addParameter("url", new UriType(vs.getUrl()));
191          t = System.currentTimeMillis();
192          try {
193            ValueSet vse = fhirToolingClient.expandValueset(null, p);    
194            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
195            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
196          } catch (Exception e2) {
197            if (e2.getMessage().contains("timed out")) {
198              ok = false;
199              break;
200            } else {
201              errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset);
202              System.out.println("Expand "+oid+" @ "+offset+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e2.getMessage());
203              return false;
204            }
205          } 
206        }
207      }
208      if (ok) {
209        vs.getExpansion().setOffsetElement(null);
210        vs.getExpansion().getParameter().clear();
211
212        if (vs.hasTitle()) {
213          if (vs.getTitle().equals(vs.getDescription())) {
214            vs.setTitle(vs.getName());              
215          } else {
216            //              System.out.println(oid);
217            //              System.out.println("  name: "+vs.getName());
218            //              System.out.println("  title: "+vs.getTitle());
219            //              System.out.println("  desc: "+vs.getDescription());
220          }
221        } else {
222          vs.setTitle(vs.getName());
223        }
224        if (vs.getUrl().startsWith("https://")) {
225          System.out.println("URL is https: "+vs.getUrl());
226        }
227        vs.setName(makeValidName(vs.getName()));
228        JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
229        new JsonParser().setOutputStyle(OutputStyle.NORMAL).compose(ManagedFileAccess.outStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
230        return true;
231      }
232    }
233  }
234
235  private boolean isIncomplete(ValueSetExpansionComponent expansion) {
236    IntegerType c = expansion.getParameter("count") != null ? expansion.getParameter("count").getValueIntegerType() : new IntegerType(0);
237    IntegerType offset = expansion.getParameter("offset") != null ? expansion.getParameter("offset").getValueIntegerType() : new IntegerType(0);
238    return c.getValue() + offset.getValue() < expansion.getTotal();
239  }
240
241  private String makeValidName(String name) {
242    StringBuilder b = new StringBuilder();
243    boolean upper = true;
244    for (char ch : name.toCharArray()) {
245      if (ch == ' ') {
246        upper = true;
247      } else if (Character.isAlphabetic(ch)) {
248        if (upper) {
249          b.append(Character.toUpperCase(ch));
250        } else {
251          b.append(ch);
252        }
253        upper = false;
254      } else if (Character.isDigit(ch)) {
255        if (b.length() == 0) {
256          b.append('N');
257        }
258        b.append(ch);
259      } else if (ch == '_' && b.length() != 0) {
260        b.append(ch);
261      } else {
262        upper = true;
263      }
264    }
265//    System.out.println(b.toString()+" from "+name);
266    return b.toString();
267  }
268}