Source code

001package org.hl7.fhir.convertors.misc;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.FileNotFoundException;
006import java.io.FileOutputStream;
007import java.io.IOException;
008import java.net.URISyntaxException;
009import java.text.ParseException;
010import java.time.Duration;
011import java.util.ArrayList;
012import java.util.Collections;
013import java.util.HashMap;
014import java.util.List;
015import java.util.Map;
016
017import org.hl7.fhir.exceptions.FHIRException;
018import org.hl7.fhir.r4.formats.IParser.OutputStyle;
019import org.hl7.fhir.r4.formats.JsonParser;
020import org.hl7.fhir.r4.model.CapabilityStatement;
021import org.hl7.fhir.r4.model.IntegerType;
022import org.hl7.fhir.r4.model.OperationOutcome;
023import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
024import org.hl7.fhir.r4.model.OperationOutcome.IssueType;
025import org.hl7.fhir.r4.model.Parameters;
026import org.hl7.fhir.r4.model.UriType;
027import org.hl7.fhir.r4.model.ValueSet;
028import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent;
029import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
030import org.hl7.fhir.r4.terminologies.JurisdictionUtilities;
031import org.hl7.fhir.utilities.CSVReader;
032import org.hl7.fhir.utilities.Utilities;
033import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
034import org.hl7.fhir.utilities.json.model.JsonArray;
035import org.hl7.fhir.utilities.json.model.JsonObject;
036import org.hl7.fhir.utilities.json.model.JsonProperty;
037
038public class VSACImporter extends OIDBasedValueSetImporter {
039  
040  public VSACImporter() throws FHIRException, IOException {
041    super();
042    init();
043  }
044
045  public static void main(String[] args) throws FHIRException, IOException, ParseException, URISyntaxException {
046    VSACImporter self = new VSACImporter();
047    self.process(args[0], args[1], args[2], "true".equals(args[3]));
048  }
049
050  private void process(String source, String dest, String apiKey, boolean onlyNew) throws FHIRException, IOException, URISyntaxException {
051    CSVReader csv = new CSVReader(ManagedFileAccess.inStream(source));
052    csv.readHeaders();
053    Map<String, String> errs = new HashMap<>();
054
055    FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
056    fhirToolingClient.setUsername("apikey");
057    fhirToolingClient.setPassword(apiKey);
058    fhirToolingClient.setTimeoutNormal(30000);
059    fhirToolingClient.setTimeoutExpand(30000);
060
061    CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement();
062    JsonParser json = new JsonParser();
063    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path("[tmp]", "vsac-capability-statmenet.json")), cs);
064
065    System.out.println("Loading");
066    List<String> oids = new ArrayList<>();
067    while (csv.line()) {
068      String oid = csv.cell("OID");
069      if (!onlyNew || !(ManagedFileAccess.file(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
070        oids.add(oid);
071      }
072    }
073    Collections.sort(oids);
074    System.out.println("Go: "+oids.size()+" oids");
075    int i = 0;
076    int j = 0;
077    long t = System.currentTimeMillis();
078    long tt = System.currentTimeMillis();
079    for (String oid : oids) {
080      try {
081        long t3 = System.currentTimeMillis();
082        if (processOid(dest, onlyNew, errs, fhirToolingClient, oid.trim())) {
083          j++;
084        }
085        i++;
086        System.out.print(":"+((System.currentTimeMillis() - t3) / 1000));
087        if (i % 100 == 0) {
088          long elapsed = System.currentTimeMillis() - t;
089          System.out.println("");
090          System.out.println(i+": "+j+" ("+((j * 100) / i)+"%) @ "+Utilities.describeDuration(elapsed)
091              +", "+(elapsed/100000)+"sec/vs, estimated "+Utilities.describeDuration(estimate(i, oids.size(), tt))+" remaining");
092          t = System.currentTimeMillis();
093        }
094      } catch (Exception e) {
095        e.printStackTrace();
096        System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage());
097        errs.put(oid, e.getMessage());
098      }
099    }
100    OperationOutcome oo = new OperationOutcome();
101    for (String oid : errs.keySet()) {
102      oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid);
103    }
104    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
105    System.out.println("Done. " + i + " ValueSets in "+Utilities.describeDuration(System.currentTimeMillis() - tt));
106  }
107
108  private long estimate(int i, int size, long tt) {
109    long elapsed = System.currentTimeMillis() - tt;
110    long average = elapsed / i;
111    return (size - i) * average;
112  }
113
114  private boolean processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, String oid)
115      throws IOException, InterruptedException, FileNotFoundException {
116    
117    while (true) {
118      boolean ok = true;
119      long t = System.currentTimeMillis();
120      ValueSet vs = null;
121      try {
122        vs = fhirToolingClient.read(ValueSet.class, oid);
123      } catch (Exception e) {
124        if (e.getMessage().contains("timed out")) {
125          ok = false;
126        } else {
127          errs.put(oid, "Read: " +e.getMessage());
128          System.out.println("Read "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
129          return false;
130        }
131      }
132      if (ok) {
133        t = System.currentTimeMillis();
134        try {
135          Parameters p = new Parameters();
136          p.addParameter("url", new UriType(vs.getUrl()));
137          ValueSet vse = fhirToolingClient.expandValueset(null, p);
138          vs.setExpansion(vse.getExpansion());
139        } catch (Exception e) {
140          if (e.getMessage().contains("timed out")) {
141            ok = false;
142          } else {
143            errs.put(oid, "Expansion: " +e.getMessage());
144            System.out.println("Expand "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
145            return false;
146          }
147        }
148      }
149      if (ok) {
150        while (isIncomplete(vs.getExpansion())) {
151          Parameters p = new Parameters();
152          int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue();
153          p.addParameter("offset", offset);
154          p.addParameter("url", new UriType(vs.getUrl()));
155          t = System.currentTimeMillis();
156          try {
157            ValueSet vse = fhirToolingClient.expandValueset(null, p);    
158            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
159            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
160          } catch (Exception e2) {
161            if (e2.getMessage().contains("timed out")) {
162              ok = false;
163              break;
164            } else {
165              errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset);
166              System.out.println("Expand "+oid+" @ "+offset+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e2.getMessage());
167              return false;
168            }
169          } 
170        }
171      }
172      if (ok) {
173        vs.getExpansion().setOffsetElement(null);
174        vs.getExpansion().getParameter().clear();
175
176        if (vs.hasTitle()) {
177          if (vs.getTitle().equals(vs.getDescription())) {
178            vs.setTitle(vs.getName());              
179          } else {
180            //              System.out.println(oid);
181            //              System.out.println("  name: "+vs.getName());
182            //              System.out.println("  title: "+vs.getTitle());
183            //              System.out.println("  desc: "+vs.getDescription());
184          }
185        } else {
186          vs.setTitle(vs.getName());
187        }
188        if (vs.getUrl().startsWith("https://")) {
189          System.out.println("URL is https: "+vs.getUrl());
190        }
191        vs.setName(makeValidName(vs.getName()));
192        JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
193        new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
194        return true;
195      }
196    }
197  }
198
199  private boolean isIncomplete(ValueSetExpansionComponent expansion) {
200    IntegerType c = expansion.getParameter("count") != null ? expansion.getParameter("count").getValueIntegerType() : new IntegerType(0);
201    IntegerType offset = expansion.getParameter("offset") != null ? expansion.getParameter("offset").getValueIntegerType() : new IntegerType(0);
202    return c.getValue() + offset.getValue() < expansion.getTotal();
203  }
204
205  private String makeValidName(String name) {
206    StringBuilder b = new StringBuilder();
207    boolean upper = true;
208    for (char ch : name.toCharArray()) {
209      if (ch == ' ') {
210        upper = true;
211      } else if (Character.isAlphabetic(ch)) {
212        if (upper) {
213          b.append(Character.toUpperCase(ch));
214        } else {
215          b.append(ch);
216        }
217        upper = false;
218      } else if (Character.isDigit(ch)) {
219        if (b.length() == 0) {
220          b.append('N');
221        }
222        b.append(ch);
223      } else if (ch == '_' && b.length() != 0) {
224        b.append(ch);
225      } else {
226        upper = true;
227      }
228    }
229//    System.out.println(b.toString()+" from "+name);
230    return b.toString();
231  }
232}