001package org.hl7.fhir.convertors.analytics; 002 003import java.io.IOException; 004import java.util.ArrayList; 005import java.util.HashMap; 006import java.util.HashSet; 007import java.util.List; 008import java.util.Map; 009import java.util.Set; 010 011import javax.xml.parsers.ParserConfigurationException; 012 013import org.hl7.fhir.exceptions.FHIRException; 014import org.hl7.fhir.r5.utils.EOperationOutcome; 015import org.hl7.fhir.utilities.SimpleHTTPClient; 016import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult; 017import org.hl7.fhir.utilities.TextFile; 018import org.hl7.fhir.utilities.Utilities; 019import org.hl7.fhir.utilities.json.model.JsonArray; 020import org.hl7.fhir.utilities.json.model.JsonObject; 021import org.hl7.fhir.utilities.json.parser.JsonParser; 022import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager; 023import org.hl7.fhir.utilities.npm.NpmPackage; 024import org.hl7.fhir.utilities.npm.PackageClient; 025import org.hl7.fhir.utilities.npm.PackageInfo; 026import org.hl7.fhir.utilities.npm.PackageServer; 027import org.hl7.fhir.utilities.xml.XMLUtil; 028import org.w3c.dom.Document; 029import org.w3c.dom.Element; 030import org.xml.sax.SAXException; 031 032public class PackageVisitor { 033 034 public interface IPackageVisitorProcessor { 035 public void processResource(String pid, NpmPackage npm, String version, String type, String id, byte[] content) throws FHIRException, IOException, EOperationOutcome; 036 } 037 038 private List<String> resourceTypes = new ArrayList<>(); 039 private List<String> versions = new ArrayList<>(); 040 private boolean corePackages; 041 private boolean oldVersions; 042 private boolean current; 043 private IPackageVisitorProcessor processor; 044 private FilesystemPackageCacheManager pcm; 045 private PackageClient pc; 046 047 public List<String> getResourceTypes() { 048 return resourceTypes; 049 } 050 051 public void setResourceTypes(List<String> resourceTypes) { 052 this.resourceTypes = resourceTypes; 053 } 054 055 public List<String> getVersions() { 056 return versions; 057 } 058 059 public void setVersions(List<String> versions) { 060 this.versions = versions; 061 } 062 063 064 public boolean isCurrent() { 065 return current; 066 } 067 068 public void setCurrent(boolean current) { 069 this.current = current; 070 } 071 072 public boolean isCorePackages() { 073 return corePackages; 074 } 075 076 077 078 079 public void setCorePackages(boolean corePackages) { 080 this.corePackages = corePackages; 081 } 082 083 084 085 086 public boolean isOldVersions() { 087 return oldVersions; 088 } 089 090 091 092 093 public void setOldVersions(boolean oldVersions) { 094 this.oldVersions = oldVersions; 095 } 096 097 098 099 100 public IPackageVisitorProcessor getProcessor() { 101 return processor; 102 } 103 104 public void setProcessor(IPackageVisitorProcessor processor) { 105 this.processor = processor; 106 } 107 108 public void visitPackages() throws IOException, ParserConfigurationException, SAXException { 109 System.out.println("Finding packages"); 110 pc = new PackageClient(PackageServer.primaryServer()); 111 pcm = new FilesystemPackageCacheManager(org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode.USER); 112 113 Map<String, String> cpidMap = getAllCIPackages(); 114 Set<String> cpidSet = new HashSet<>(); 115 System.out.println("Go: "+cpidMap.size()+" current packages"); 116 int i = 0; 117 for (String s : cpidMap.keySet()) { 118 processCurrentPackage(s, cpidMap.get(s), cpidSet, i, cpidMap.size()); 119 i++; 120 } 121 Set<String> pidList = getAllPackages(); 122 System.out.println("Go: "+pidList.size()+" published packages"); 123 i = 0; 124 for (String pid : pidList) { 125 if (!cpidSet.contains(pid)) { 126 cpidSet.add(pid); 127 List<String> vList = listVersions(pid); 128 if (oldVersions) { 129 for (String v : vList) { 130 processPackage(pid, v, i, pidList.size()); 131 } 132 } else if (vList.isEmpty()) { 133 System.out.println("No Packages for "+pid); 134 } else { 135 processPackage(pid, vList.get(vList.size() - 1), i, pidList.size()); 136 } 137 } 138 i++; 139 } 140 JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json"); 141 i = 0; 142 List<JsonObject> objects = json.getJsonObjects("guides"); 143 for (JsonObject o : objects) { 144 String pid = o.asString("npm-name"); 145 if (pid != null && !cpidSet.contains(pid)) { 146 cpidSet.add(pid); 147 List<String> vList = listVersions(pid); 148 if (oldVersions) { 149 for (String v : vList) { 150 processPackage(pid, v, i, objects.size()); 151 } 152 } else if (vList.isEmpty()) { 153 System.out.println("No Packages for "+pid); 154 } else { 155 processPackage(pid, vList.get(vList.size() - 1), i, objects.size()); 156 } 157 } 158 i++; 159 } 160 } 161 162 private void processCurrentPackage(String url, String pid, Set<String> cpidSet, int i, int t) { 163 try { 164 String[] p = url.split("\\/"); 165 String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1]; 166 NpmPackage npm = NpmPackage.fromUrl(repo+"/package.tgz"); 167 String fv = npm.fhirVersion(); 168 cpidSet.add(pid); 169 170 if (corePackages || !corePackage(npm)) { 171 int c = 0; 172 if (fv != null && (versions.isEmpty() || versions.contains(fv))) { 173 for (String type : resourceTypes) { 174 for (String s : npm.listResources(type)) { 175 c++; 176 try { 177 processor.processResource(pid+"#current", npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s))); 178 } catch (Exception e) { 179 System.out.println("####### Error loading "+pid+"#current["+fv+"]/"+type+" ####### "+e.getMessage()); 180// e.printStackTrace(); 181 } 182 } 183 } 184 } 185 System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+")"); 186 } 187 } catch (Exception e) { 188 System.out.println("Unable to process: "+pid+"#current: "+e.getMessage()); 189 } 190 } 191 192 private Map<String, String> getAllCIPackages() throws IOException { 193 Map<String, String> res = new HashMap<>(); 194 if (current) { 195 JsonArray json = (JsonArray) JsonParser.parseFromUrl("https://build.fhir.org/ig/qas.json"); 196 for (JsonObject o : json.asJsonObjects()) { 197 String url = o.asString("repo"); 198 res.put(url, o.asString("package-id")); 199 } 200 } 201 return res; 202 } 203 204 private List<String> listVersions(String pid) throws IOException { 205 List<String> list = new ArrayList<>(); 206 if (pid !=null) { 207 for (PackageInfo i : pc.getVersions(pid)) { 208 list.add(i.getVersion()); 209 } 210 } 211 return list; 212 } 213 214 private Set<String> getAllPackages() throws IOException, ParserConfigurationException, SAXException { 215 Set<String> list = new HashSet<>(); 216 for (PackageInfo i : pc.search(null, null, null, false)) { 217 list.add(i.getId()); 218 } 219 JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json"); 220 for (JsonObject ig : json.getJsonObjects("guides")) { 221 list.add(ig.asString("npm-name")); 222 } 223 json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/package-feeds.json"); 224 for (JsonObject feed : json.getJsonObjects("feeds")) { 225 processFeed(list, feed.asString("url")); 226 } 227 228 return list; 229 } 230 231 private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException { 232 System.out.println("Feed "+str); 233 try { 234 SimpleHTTPClient fetcher = new SimpleHTTPClient(); 235 HTTPResult res = fetcher.get(str+"?nocache=" + System.currentTimeMillis()); 236 res.checkThrowException(); 237 Document xml = XMLUtil.parseToDom(res.getContent()); 238 for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) { 239 for (Element item : XMLUtil.getNamedChildren(channel, "item")) { 240 String pid = XMLUtil.getNamedChildText(item, "title"); 241 if (pid.contains("#")) { 242 list.add(pid.substring(0, pid.indexOf("#"))); 243 } 244 } 245 } 246 } catch (Exception e) { 247 System.out.println(" "+e.getMessage()); 248 } 249 } 250 251 252 private void processPackage(String pid, String v, int i, int t) throws IOException { 253 NpmPackage npm = null; 254 String fv = null; 255 try { 256 npm = pcm.loadPackage(pid, v); 257 fv = npm.fhirVersion(); 258 } catch (Throwable e) { 259 System.out.println("Unable to process: "+pid+"#"+v+": "+e.getMessage()); 260 } 261 if (corePackages || !corePackage(npm)) { 262 int c = 0; 263 if (fv != null && (versions.isEmpty() || versions.contains(fv))) { 264 for (String type : resourceTypes) { 265 for (String s : npm.listResources(type)) { 266 c++; 267 try { 268 processor.processResource(pid+"#"+v, npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s))); 269 } catch (Exception e) { 270 System.out.println("####### Error loading "+pid+"#"+v +"["+fv+"]/"+type+" ####### "+e.getMessage()); 271 e.printStackTrace(); 272 } 273 } 274 } 275 } 276 System.out.println("Processed: "+pid+"#"+v+": "+c+" resources ("+i+" of "+t+")"); 277 } 278 } 279 280 private boolean corePackage(NpmPackage npm) { 281 return npm != null && !Utilities.noString(npm.name()) && ( 282 npm.name().startsWith("hl7.terminology") || 283 npm.name().startsWith("hl7.fhir.core") || 284 npm.name().startsWith("hl7.fhir.r2.") || 285 npm.name().startsWith("hl7.fhir.r2b.") || 286 npm.name().startsWith("hl7.fhir.r3.") || 287 npm.name().startsWith("hl7.fhir.r4.") || 288 npm.name().startsWith("hl7.fhir.r4b.") || 289 npm.name().startsWith("hl7.fhir.r5.")); 290 } 291 292}