Source code

001package org.hl7.fhir.convertors.analytics;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.IOException;
006import java.util.ArrayList;
007import java.util.HashMap;
008import java.util.HashSet;
009import java.util.List;
010import java.util.Map;
011import java.util.Set;
012
013import javax.xml.parsers.ParserConfigurationException;
014
015import org.hl7.fhir.exceptions.FHIRException;
016import org.hl7.fhir.r5.utils.EOperationOutcome;
017import org.hl7.fhir.utilities.TextFile;
018import org.hl7.fhir.utilities.Utilities;
019import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
020import org.hl7.fhir.utilities.http.HTTPResult;
021import org.hl7.fhir.utilities.http.ManagedWebAccess;
022import org.hl7.fhir.utilities.json.model.JsonArray;
023import org.hl7.fhir.utilities.json.model.JsonObject;
024import org.hl7.fhir.utilities.json.parser.JsonParser;
025import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
026import org.hl7.fhir.utilities.npm.NpmPackage;
027import org.hl7.fhir.utilities.npm.PackageClient;
028import org.hl7.fhir.utilities.npm.PackageInfo;
029import org.hl7.fhir.utilities.npm.PackageServer;
030import org.hl7.fhir.utilities.xml.XMLUtil;
031import org.w3c.dom.Document;
032import org.w3c.dom.Element;
033import org.xml.sax.SAXException;
034
035public class PackageVisitor {
036
037  private PackageServer clientPackageServer = null;
038
039  public void setClientPackageServer(PackageServer packageServer) {
040    this.clientPackageServer = packageServer;
041  }
042  private List<PackageServer> cachePackageServers = null;
043  public void setCachePackageServers(List<PackageServer> packageServers) {
044    this.cachePackageServers = packageServers;
045  }
046
047  public static class PackageContext {
048    private String pid;
049    private NpmPackage npm;
050    private String version;
051    protected PackageContext(String pid, NpmPackage npm, String version) {
052      super();
053      this.pid = pid;
054      this.npm = npm;
055      this.version = version;
056    }
057    public String getPid() {
058      return pid;
059    }
060    public NpmPackage getNpm() {
061      return npm;
062    }
063    public String getVersion() {
064      return version;
065    }
066  }
067  
068  public interface IPackageVisitorProcessor {
069    public Object startPackage(PackageContext context) throws FHIRException, IOException, EOperationOutcome;
070    public void processResource(PackageContext context, Object clientContext, String type, String id, byte[] content) throws FHIRException, IOException, EOperationOutcome;
071    public void finishPackage(PackageContext context) throws FHIRException, IOException, EOperationOutcome;
072
073    public void alreadyVisited(String pid) throws FHIRException, IOException, EOperationOutcome;
074  }
075
076  private List<String> resourceTypes = new ArrayList<>();
077  private List<String> versions = new ArrayList<>();
078  private boolean corePackages;
079  private boolean oldVersions;
080  private boolean current;
081  private IPackageVisitorProcessor processor;
082  private FilesystemPackageCacheManager pcm;
083  private PackageClient pc;
084  private String cache;  
085  private int step;
086
087  public List<String> getResourceTypes() {
088    return resourceTypes;
089  }
090
091  public void setResourceTypes(List<String> resourceTypes) {
092    this.resourceTypes = resourceTypes;
093  }
094
095  public List<String> getVersions() {
096    return versions;
097  }
098
099  public void setVersions(List<String> versions) {
100    this.versions = versions;
101  }
102
103
104  public boolean isCurrent() {
105    return current;
106  }
107
108  public void setCurrent(boolean current) {
109    this.current = current;
110  }
111
112  public boolean isCorePackages() {
113    return corePackages;
114  }
115
116
117
118
119  public String getCache() {
120    return cache;
121  }
122
123  public void setCache(String cache) {
124    this.cache = cache;
125  }
126
127  public void setCorePackages(boolean corePackages) {
128    this.corePackages = corePackages;
129  }
130
131
132
133
134  public boolean isOldVersions() {
135    return oldVersions;
136  }
137
138
139
140
141  public void setOldVersions(boolean oldVersions) {
142    this.oldVersions = oldVersions;
143  }
144
145
146
147
148  public IPackageVisitorProcessor getProcessor() {
149    return processor;
150  }
151
152  public void setProcessor(IPackageVisitorProcessor processor) {
153    this.processor = processor;
154  }
155
156  public void visitPackages() throws IOException, ParserConfigurationException, SAXException, FHIRException, EOperationOutcome {
157    System.out.println("Finding packages");
158    pc = clientPackageServer == null
159      ? new PackageClient(PackageServer.primaryServer())
160      : new PackageClient(clientPackageServer);
161
162    pcm = cachePackageServers == null
163      ? new FilesystemPackageCacheManager.Builder().build()
164      : new FilesystemPackageCacheManager.Builder().withPackageServers(cachePackageServers).build();
165
166    Set<String> pidList = getAllPackages();
167
168    Map<String, String> cpidMap = getAllCIPackages();
169    Set<String> cpidSet = new HashSet<>();
170    System.out.println("Go: "+cpidMap.size()+" current packages");
171    int i = 0;
172    for (String s : cpidMap.keySet()) {
173      processCurrentPackage(cpidMap.get(s), s, cpidSet, i, cpidMap.size()); 
174      i++;
175    }
176
177    System.out.println("Go: "+pidList.size()+" published packages");
178    i = 0;
179    for (String pid : pidList) {  
180      if (pid != null) {
181        if (!cpidSet.contains(pid)) {
182          cpidSet.add(pid);
183          if (step == 0 || step == 3) {
184            List<String> vList = listVersions(pid);
185            if (oldVersions) {
186              for (String v : vList) {
187                processPackage(pid, v, i, pidList.size());          
188              }
189            } else if (vList.isEmpty()) {
190              System.out.println("No Packages for "+pid);
191            } else {
192              processPackage(pid, vList.get(vList.size() - 1), i, pidList.size());
193            }
194          }
195        } else {
196          processor.alreadyVisited(pid);
197        }
198        i++;
199      }    
200    }
201
202    if (step == 0 || step == 3) {
203      JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json");
204      i = 0;
205      List<JsonObject> objects = json.getJsonObjects("guides");
206      for (JsonObject o : objects) {
207        String pid = o.asString("npm-name");
208        if (pid != null && !cpidSet.contains(pid)) {
209          cpidSet.add(pid);
210          List<String> vList = listVersions(pid);
211          if (oldVersions) {
212            for (String v : vList) {
213              processPackage(pid, v, i, objects.size());          
214            }
215          } else if (vList.isEmpty()) {
216            System.out.println("No Packages for "+pid);
217          } else {
218            processPackage(pid, vList.get(vList.size() - 1), i, objects.size());
219          }
220        }
221        i++;
222      }
223    }
224  }
225
226  private void processCurrentPackage(String url, String pid, Set<String> cpidSet, int i, int t) {
227    try {
228      cpidSet.add(pid);
229      if (step == 0 || (step == 1 && i < t/2) || (step == 2 && i >= t/2)) {
230        long ms1 = System.currentTimeMillis();
231        String[] p = url.split("\\/");
232        String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1];
233        JsonObject manifest = JsonParser.parseObjectFromUrl(repo+"/package.manifest.json");
234        File co = ManagedFileAccess.file(Utilities.path(cache, pid+"."+manifest.asString("date")+".tgz"));
235        if (!co.exists()) {
236
237          HTTPResult res = ManagedWebAccess.get(repo+"/package.tgz?nocache=" + System.currentTimeMillis());
238          res.checkThrowException();
239          TextFile.bytesToFile(res.getContent(), co);
240        }
241        NpmPackage npm = NpmPackage.fromPackage(ManagedFileAccess.inStream(co));          
242        String fv = npm.fhirVersion();
243        long ms2 = System.currentTimeMillis();
244
245        if (corePackages || !corePackage(npm)) {
246          if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
247            PackageContext ctxt = new PackageContext(pid+"#current", npm, fv);
248            boolean ok = false;
249            Object context = null;
250            try {
251              context = processor.startPackage(ctxt);
252              ok = true;
253            } catch (Exception e) {
254              System.out.println("####### Error loading "+pid+"#current["+fv+"]: ####### "+e.getMessage());
255              //                e.printStackTrace();
256            }
257            if (ok) {
258              int c = 0;
259              for (String type : resourceTypes) {
260                for (String s : npm.listResources(type)) {
261                  c++;
262                  try {
263                    processor.processResource(ctxt, context, type, s, TextFile.streamToBytes(npm.load("package", s)));
264                  } catch (Exception e) {
265                    System.out.println("####### Error loading "+pid+"#current["+fv+"]/"+type+" ####### "+e.getMessage());
266                    //                e.printStackTrace();
267                  }
268                }
269              }
270              processor.finishPackage(ctxt);
271              System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+", "+(ms2-ms1)+"/"+(System.currentTimeMillis()-ms2)+"ms)");
272            }
273          } else {
274            System.out.println("Ignored: "+pid+"#current: no version");            
275          }
276        }
277      }
278    } catch (Exception e) {      
279      System.out.println("Unable to process: "+pid+"#current: "+e.getMessage());      
280    }
281  }
282
283  private Map<String, String> getAllCIPackages() throws IOException {
284    System.out.println("Fetch https://build.fhir.org/ig/qas.json");
285    Map<String, String> res = new HashMap<>();
286    if (current) {
287      JsonArray json = (JsonArray) JsonParser.parseFromUrl("https://build.fhir.org/ig/qas.json");
288      for (JsonObject o  : json.asJsonObjects()) {
289        String url = o.asString("repo");
290        String pid = o.asString("package-id");
291        if (url.contains("/branches/master") || url.contains("/branches/main") ) {
292          if (!res.containsKey(pid)) {
293            res.put(pid, url);
294          } else if (!url.equals(res.get(pid))) {
295            System.out.println("Ignore "+url+" already encountered "+pid +" @ "+res.get(pid));
296          }
297        }
298      }
299    }
300    return res;
301  }
302
303  private List<String> listVersions(String pid) throws IOException {
304    List<String> list = new ArrayList<>();
305    if (pid !=null) {
306      for (PackageInfo i : pc.getVersions(pid)) {
307        list.add(i.getVersion());
308      }    
309    }
310    return list;
311  }
312
313  private Set<String> getAllPackages() throws IOException, ParserConfigurationException, SAXException {
314    Set<String> list = new HashSet<>();
315    for (PackageInfo i : pc.search(null, null, null, false)) {
316      list.add(i.getId());
317    }    
318    JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json");
319    for (JsonObject ig : json.getJsonObjects("guides")) {
320      list.add(ig.asString("npm-name"));
321    }
322    json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/package-feeds.json");
323    for (JsonObject feed : json.getJsonObjects("feeds")) {
324      processFeed(list, feed.asString("url"));
325    }
326
327    return list;
328  }
329
330  private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException {
331    System.out.println("Feed "+str);
332    try {
333
334      HTTPResult res = ManagedWebAccess.get(str+"?nocache=" + System.currentTimeMillis());
335      res.checkThrowException();
336      Document xml = XMLUtil.parseToDom(res.getContent());
337      for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) {
338        for (Element item : XMLUtil.getNamedChildren(channel, "item")) {
339          String pid = XMLUtil.getNamedChildText(item, "title");
340          if (pid != null && pid.contains("#")) {
341            list.add(pid.substring(0, pid.indexOf("#")));
342          }
343        }
344      }
345    } catch (Exception e) {
346      System.out.println("   "+e.getMessage());
347    }
348  }
349
350
351  private void processPackage(String pid, String v, int i, int t) throws IOException, FHIRException, EOperationOutcome {
352    NpmPackage npm = null;
353    String fv = null;
354    try {
355      npm = pcm.loadPackage(pid, v);
356    } catch (Throwable e) {
357      System.out.println("Unable to load package: "+pid+"#"+v+": "+e.getMessage());
358    }
359
360    try {
361      fv = npm.fhirVersion();
362    } catch (Throwable e) {
363      System.out.println("Unable to identify package FHIR version:: "+pid+"#"+v+": "+e.getMessage());
364    }
365    if (corePackages || !corePackage(npm)) {
366      PackageContext ctxt = new PackageContext(pid+"#"+v, npm, fv);
367      boolean ok = false;
368      Object context = null;
369      try {
370        context = processor.startPackage(ctxt);
371        ok = true;
372      } catch (Exception e) {
373        System.out.println("####### Error loading package  "+pid+"#"+v +"["+fv+"]: "+e.getMessage());
374        e.printStackTrace();
375      }
376      if (ok) {
377        int c = 0;
378        if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
379          for (String type : resourceTypes) {
380            for (String s : npm.listResources(type)) {
381              c++;
382              try {
383                processor.processResource(ctxt, context, type, s, TextFile.streamToBytes(npm.load("package", s)));
384              } catch (Exception e) {
385                System.out.println("####### Error loading "+pid+"#"+v +"["+fv+"]/"+type+" ####### "+e.getMessage());
386                e.printStackTrace();
387              }
388            }
389          }
390        }    
391        processor.finishPackage(ctxt);
392        System.out.println("Processed: "+pid+"#"+v+": "+c+" resources ("+i+" of "+t+")");  
393      }
394    }
395  }
396
397  private boolean corePackage(NpmPackage npm) {
398    return npm != null && !Utilities.noString(npm.name()) && (
399        npm.name().startsWith("hl7.terminology") || 
400        npm.name().startsWith("hl7.fhir.core") || 
401        npm.name().startsWith("hl7.fhir.r2.") || 
402        npm.name().startsWith("hl7.fhir.r2b.") || 
403        npm.name().startsWith("hl7.fhir.r3.") || 
404        npm.name().startsWith("hl7.fhir.r4.") || 
405        npm.name().startsWith("hl7.fhir.r4b.") || 
406        npm.name().startsWith("hl7.fhir.r5."));
407  }
408
409  public int getStep() {
410    return step;
411  }
412
413  public void setStep(int step) {
414    this.step = step;
415  }
416
417}