001package org.hl7.fhir.r5.terminologies.client;
002
003import java.io.File;
004import java.io.IOException;
005import java.net.MalformedURLException;
006import java.net.URISyntaxException;
007import java.net.URL;
008import java.util.ArrayList;
009import java.util.Collections;
010import java.util.HashMap;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Map;
014import java.util.Set;
015
016import org.hl7.fhir.exceptions.TerminologyServiceException;
017import org.hl7.fhir.r5.context.CanonicalResourceManager;
018import org.hl7.fhir.r5.model.Bundle;
019import org.hl7.fhir.r5.model.Bundle.BundleEntryComponent;
020import org.hl7.fhir.r5.model.CodeSystem;
021import org.hl7.fhir.r5.model.Parameters;
022import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
023import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesCodeSystemComponent;
024import org.hl7.fhir.r5.model.TerminologyCapabilities;
025import org.hl7.fhir.r5.model.ValueSet;
026import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
027import org.hl7.fhir.r5.terminologies.ValueSetUtilities;
028import org.hl7.fhir.r5.terminologies.client.TerminologyClientContext.TerminologyClientContextUseType;
029import org.hl7.fhir.r5.terminologies.client.TerminologyClientManager.ServerOptionList;
030import org.hl7.fhir.r5.terminologies.utilities.TerminologyCache;
031import org.hl7.fhir.r5.terminologies.utilities.TerminologyCache.SourcedCodeSystem;
032import org.hl7.fhir.r5.terminologies.utilities.TerminologyCache.SourcedValueSet;
033import org.hl7.fhir.r5.utils.ToolingExtensions;
034import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
035import org.hl7.fhir.utilities.ToolingClientLogger;
036import org.hl7.fhir.utilities.Utilities;
037import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
038import org.hl7.fhir.utilities.json.model.JsonObject;
039import org.hl7.fhir.utilities.json.parser.JsonParser;
040
041public class TerminologyClientManager {
042  public class ServerOptionList {
043    private List<String> authoritative = new ArrayList<String>();
044    private List<String> candidates = new ArrayList<String>();
045    
046    public ServerOptionList(String address) {
047      candidates.add(address);
048    }
049    
050    public ServerOptionList() {
051    }
052
053    public ServerOptionList(List<String> auth, List<String> cand) {
054      authoritative.addAll(auth);
055      candidates.addAll(cand);
056    }
057
058    public void replace(String src, String dst) {
059      for (int i = 0; i < candidates.size(); i++) {
060        if (candidates.get(i).contains("://"+src)) {
061          candidates.set(i, candidates.get(i).replace("://"+src, "://"+dst));
062        }
063      }
064      for (int i = 0; i < authoritative.size(); i++) {
065        if (authoritative.get(i).contains("://"+src)) {
066          authoritative.set(i, authoritative.get(i).replace("://"+src, "://"+dst));
067        }
068      }      
069    }
070
071    @Override
072    public String toString() {
073      return "auth = " + CommaSeparatedStringBuilder.join("|", authoritative)+ ", candidates=" + CommaSeparatedStringBuilder.join("|", candidates);
074    }    
075    
076  }
077
078  public ITerminologyClientFactory getFactory() {
079    return factory;
080  }
081
082  public interface ITerminologyClientFactory {
083    ITerminologyClient makeClient(String id, String url, String userAgent, ToolingClientLogger logger) throws URISyntaxException;
084    String getVersion();
085  }
086
087  public static final String UNRESOLVED_VALUESET = "--unknown--";
088
089  private static final boolean IGNORE_TX_REGISTRY = false;
090  
091  private ITerminologyClientFactory factory;
092  private String cacheId;
093  private List<TerminologyClientContext> serverList = new ArrayList<>(); // clients by server address
094  private Map<String, TerminologyClientContext> serverMap = new HashMap<>(); // clients by server address
095  private Map<String, ServerOptionList> resMap = new HashMap<>(); // client resolution list
096  private List<String> internalLog = new ArrayList<>();
097  protected Parameters expParameters;
098
099  private TerminologyCache cache;
100
101  private File cacheFile;
102  private String usage;
103
104  private String monitorServiceURL;
105
106  private boolean useEcosystem;
107
108  public TerminologyClientManager(ITerminologyClientFactory factory, String cacheId) {
109    super();
110    this.factory = factory;
111    this.cacheId = cacheId;
112  }
113  
114  public String getCacheId() {
115    return cacheId; 
116  }
117  
118  public void copy(TerminologyClientManager other) {
119    cacheId = other.cacheId;  
120    serverList.addAll(other.serverList);
121    serverMap.putAll(other.serverMap);
122    resMap.putAll(other.resMap);
123    useEcosystem = other.useEcosystem;
124    monitorServiceURL = other.monitorServiceURL;
125    factory = other.factory;
126    usage = other.usage;
127  }
128
129
130  public TerminologyClientContext chooseServer(ValueSet vs, Set<String> systems, boolean expand) throws TerminologyServiceException {
131    if (serverList.isEmpty()) {
132      return null;
133    }
134    if (systems.contains(UNRESOLVED_VALUESET) || systems.isEmpty()) {
135      return serverList.get(0);
136    }
137    
138    List<ServerOptionList> choices = new ArrayList<>();
139    for (String s : systems) {
140      choices.add(findServerForSystem(s, expand));
141    }    
142    
143    // first we look for a server that's authoritative for all of them
144    for (ServerOptionList ol : choices) {
145      for (String s : ol.authoritative) {
146        boolean ok = true;
147        for (ServerOptionList t : choices) {
148          if (!t.authoritative.contains(s)) {
149            ok = false;
150          }
151        }
152        if (ok) {
153          return findClient(s, systems, expand);
154        }
155      }
156    }
157    
158    // now we look for a server that's authoritative for one of them and a candidate for the others
159    for (ServerOptionList ol : choices) {
160      for (String s : ol.authoritative) {
161        boolean ok = true;
162        for (ServerOptionList t : choices) {
163          if (!t.authoritative.contains(s) && !t.candidates.contains(s)) {
164            ok = false;
165          }
166        }
167        if (ok) {
168          return findClient(s, systems, expand);
169        }
170      }
171    }
172
173    // now we look for a server that's a candidate for all of them
174    for (ServerOptionList ol : choices) {
175      for (String s : ol.candidates) {
176        boolean ok = true;
177        for (ServerOptionList t : choices) {
178          if (!t.candidates.contains(s)) {
179            ok = false;
180          }
181        }
182        if (ok) {
183          return findClient(s, systems, expand);
184        }
185      }
186    }
187    
188    for (String sys : systems) {
189      String uri = sys.contains("|") ? sys.substring(0, sys.indexOf("|")) : sys;
190      // this list is the list of code systems that have special handling on tx.fhir.org, and might not be resolved above.
191      // we don't want them to go to secondary servers (e.g. VSAC) by accident (they might go deliberately above)
192      if (Utilities.existsInList(uri, "http://unitsofmeasure.org", "http://loinc.org", "http://snomed.info/sct",
193          "http://www.nlm.nih.gov/research/umls/rxnorm", "http://hl7.org/fhir/sid/cvx", "urn:ietf:bcp:13", "urn:ietf:bcp:47",
194          "urn:ietf:rfc:3986", "http://www.ama-assn.org/go/cpt", "urn:oid:1.2.36.1.2001.1005.17", "urn:iso:std:iso:3166", 
195          "http://varnomen.hgvs.org", "http://unstats.un.org/unsd/methods/m49/m49.htm", "urn:iso:std:iso:4217", 
196          "http://hl7.org/fhir/sid/ndc", "http://fhir.ohdsi.org/CodeSystem/concepts", "http://fdasis.nlm.nih.gov", "https://www.usps.com/")) {
197        return serverList.get(0);
198      }
199    }
200
201    // no agreement? Then what we do depends     
202    if (vs != null) {
203      if (vs.hasUserData("External.Link")) {
204        if (systems.size() == 1) {
205          internalLog.add(vs.getVersionedUrl()+" uses the system "+systems.toString()+" not handled by any servers. Using source @ '"+vs.getUserString("External.Link")+"'");
206        } else {
207          internalLog.add(vs.getVersionedUrl()+" includes multiple systems "+systems.toString()+" best handled by multiple servers: "+choices.toString()+". Using source @ '"+vs.getUserString("External.Link")+"'");
208        }
209        return findClient(vs.getUserString("External.Link"), systems, expand);
210      } else {
211        if (systems.size() == 1) {
212          internalLog.add(vs.getVersionedUrl()+" uses the system "+systems.toString()+" not handled by any servers. Using master @ '"+serverList.get(0)+"'");
213        } else {
214          internalLog.add(vs.getVersionedUrl()+" includes multiple systems "+systems.toString()+" best handled by multiple servers: "+choices.toString()+". Using master @ '"+serverList.get(0)+"'");
215        }
216        return findClient(serverList.get(0).getAddress(), systems, expand);
217      }      
218    } else {
219      if (systems.size() == 1) {
220        internalLog.add("Request for system "+systems.toString()+" not handled by any servers. Using master @ '"+serverList.get(0)+"'");
221      } else {
222        internalLog.add("Request for multiple systems "+systems.toString()+" best handled by multiple servers: "+choices.toString()+". Using master @ '"+serverList.get(0)+"'");
223      }
224      return findClient(serverList.get(0).getAddress(), systems, expand);
225    }
226  }
227
228  private TerminologyClientContext findClient(String server, Set<String> systems, boolean expand) {
229    TerminologyClientContext client = serverMap.get(server);
230    if (client == null) {
231      try {
232        client = new TerminologyClientContext(factory.makeClient("id"+(serverList.size()+1), server, getMasterClient().getUserAgent(), getMasterClient().getLogger()), cacheId, false);
233      } catch (URISyntaxException e) {
234        throw new TerminologyServiceException(e);
235      }
236      client.setTxCache(cache);
237      serverList.add(client);
238      serverMap.put(server, client);
239    }
240    client.seeUse(systems, expand ? TerminologyClientContextUseType.expand : TerminologyClientContextUseType.validate);
241    return client;
242  }
243
244  private ServerOptionList findServerForSystem(String s, boolean expand) throws TerminologyServiceException {
245    ServerOptionList serverList = resMap.get(s);
246    if (serverList == null) {
247      serverList = decideWhichServer(s);
248      // testing support
249      try {
250        serverList.replace("tx.fhir.org", new URL(getMasterClient().getAddress()).getHost());
251      } catch (MalformedURLException e) {
252      }
253      resMap.put(s, serverList);
254      save();
255    }
256    return serverList;
257  }
258
259  private ServerOptionList decideWhichServer(String url) {
260    if (IGNORE_TX_REGISTRY || !useEcosystem) {
261      return new ServerOptionList(getMasterClient().getAddress());
262    }
263    if (expParameters != null) {
264      if (!url.contains("|")) {
265        // the client hasn't specified an explicit version, but the expansion parameters might
266        for (ParametersParameterComponent p : expParameters.getParameter()) {
267          if (Utilities.existsInList(p.getName(), "system-version", "force-system-version") && p.hasValuePrimitive() && p.getValue().primitiveValue().startsWith(url+"|")) {
268            url = p.getValue().primitiveValue();
269          }
270        }
271      } else {
272        // the expansion parameters might override the version
273        for (ParametersParameterComponent p : expParameters.getParameter()) {
274          if (Utilities.existsInList(p.getName(), "force-system-version") && p.hasValueCanonicalType() && p.getValue().primitiveValue().startsWith(url+"|")) {
275            url = p.getValue().primitiveValue();
276          }
277        }
278      }
279    }
280    String request = Utilities.pathURL(monitorServiceURL, "resolve?fhirVersion="+factory.getVersion()+"&url="+Utilities.URLEncode(url));
281    if (usage != null) {
282      request = request + "&usage="+usage;
283    } 
284    try {
285      ServerOptionList ret = new ServerOptionList();
286      JsonObject json = JsonParser.parseObjectFromUrl(request);
287      for (JsonObject item : json.getJsonObjects("authoritative")) {
288        ret.authoritative.add(item.asString("url"));
289      }
290      for (JsonObject item : json.getJsonObjects("candidates")) {
291        ret.candidates.add(item.asString("url"));
292      }
293      return ret;
294    } catch (Exception e) {
295      String msg = "Error resolving system "+url+": "+e.getMessage()+" ("+request+")";
296      if (!internalLog.contains(msg)) {
297        internalLog.add(msg);
298      }
299      if (!monitorServiceURL.contains("tx.fhir.org")) {
300        e.printStackTrace();
301      }
302    }
303    return new ServerOptionList( getMasterClient().getAddress());
304    
305  }
306
307  public List<TerminologyClientContext> serverList() {
308    return serverList;
309  }
310  
311  public boolean hasClient() {
312    return !serverList.isEmpty();
313  }
314
315  public int getRetryCount() {
316    return hasClient() ? getMasterClient().getRetryCount() : 0;
317  }
318
319  public void setRetryCount(int value) {
320    if (hasClient()) {
321      getMasterClient().setRetryCount(value);
322    }
323  }
324
325  public void setUserAgent(String value) {
326    if (hasClient()) {
327      getMasterClient().setUserAgent(value);
328    }
329  }
330
331  public void setLogger(ToolingClientLogger txLog) {
332    if (hasClient()) {
333      getMasterClient().setLogger(txLog);
334    }
335  }
336
337  public TerminologyClientContext setMasterClient(ITerminologyClient client, boolean useEcosystem) {
338    this.useEcosystem = useEcosystem;
339    TerminologyClientContext details = new TerminologyClientContext(client, cacheId, true);
340    details.setTxCache(cache);
341    serverList.clear();
342    serverList.add(details);
343    serverMap.put(client.getAddress(), details);  
344    monitorServiceURL = Utilities.pathURL(Utilities.getDirectoryForURL(client.getAddress()), "tx-reg");
345    return details;
346  }
347  
348  public TerminologyClientContext getMaster() {
349    return serverList.isEmpty() ? null : serverList.get(0);
350  }
351
352  public ITerminologyClient getMasterClient() {
353    return serverList.isEmpty() ? null : serverList.get(0).getClient();
354  }
355
356  public Map<String, TerminologyClientContext> serverMap() {
357    Map<String, TerminologyClientContext> map = new HashMap<>();
358    for (TerminologyClientContext t : serverList) {
359      map.put(t.getClient().getAddress(), t);
360    }
361    return map;
362  }
363
364
365  public void setFactory(ITerminologyClientFactory factory) {
366    this.factory = factory;    
367  }
368
369  public void setCache(TerminologyCache cache) {
370    this.cache = cache;
371    this.cacheFile = null;
372
373    if (cache != null && cache.getFolder() != null) {
374      try {
375        cacheFile = ManagedFileAccess.file(Utilities.path(cache.getFolder(), "system-map.json"));
376        if (cacheFile.exists()) {
377          JsonObject json = JsonParser.parseObject(cacheFile);
378          for (JsonObject pair : json.getJsonObjects("systems")) {
379            if (pair.has("server")) {
380              resMap.put(pair.asString("system"), new ServerOptionList(pair.asString("server")));
381            } else {
382              resMap.put(pair.asString("system"), new ServerOptionList(pair.getStrings("authoritative"), pair.getStrings("candidates")));
383            }
384          }
385        }
386      } catch (Exception e) {
387        e.printStackTrace();
388      }
389    }
390  }
391
392  private void save() {
393    if (cacheFile != null && cache.getFolder() != null) {
394      JsonObject json = new JsonObject();
395      for (String s : Utilities.sorted(resMap.keySet())) {
396        JsonObject si = new JsonObject();
397        json.forceArray("systems").add(si);
398        si.add("system", s);
399        si.add("authoritative", resMap.get(s).authoritative);
400        si.add("candidates", resMap.get(s).candidates);
401      }
402      try {
403        JsonParser.compose(json, cacheFile, true);
404      } catch (IOException e) {
405      }
406    }
407  }
408
409  public List<String> getInternalLog() {
410    return internalLog;
411  }
412
413  public List<TerminologyClientContext> getServerList() {
414    return serverList;
415  }
416
417  public Map<String, TerminologyClientContext> getServerMap() {
418    return serverMap;
419  }
420
421  public String getMonitorServiceURL() {
422    return monitorServiceURL;
423  }
424
425  public Parameters getExpansionParameters() {
426    return expParameters;
427  }
428
429  public void setExpansionParameters(Parameters expParameters) {
430    this.expParameters = expParameters;
431  }
432
433  public String getUsage() {
434    return usage;
435  }
436
437  public void setUsage(String usage) {
438    this.usage = usage;
439  }
440
441  public SourcedValueSet findValueSetOnServer(String canonical) {
442    if (IGNORE_TX_REGISTRY || getMasterClient() == null) {
443      return null;
444    }
445    String request = Utilities.pathURL(monitorServiceURL, "resolve?fhirVersion="+factory.getVersion()+"&valueSet="+Utilities.URLEncode(canonical));
446    String server = null;
447    try {
448      if (!useEcosystem) {
449        server = getMasterClient().getAddress();
450      } else {
451        if (usage != null) {
452          request = request + "&usage="+usage;
453        }
454        JsonObject json = JsonParser.parseObjectFromUrl(request);
455        for (JsonObject item : json.getJsonObjects("authoritative")) {
456          if (server == null) {
457            server = item.asString("url");
458          }
459        }
460        for (JsonObject item : json.getJsonObjects("candidates")) {
461          if (server == null) {
462            server = item.asString("url");
463          }
464        }
465        if (server == null) {
466          return null;
467        }
468        if (server.contains("://tx.fhir.org")) {
469          try {
470            server = server.replace("tx.fhir.org", new URL(getMasterClient().getAddress()).getHost());
471          } catch (MalformedURLException e) {
472          }
473        }
474      }
475      TerminologyClientContext client = serverMap.get(server);
476      if (client == null) {
477        try {
478          client = new TerminologyClientContext(factory.makeClient("id"+(serverList.size()+1), server, getMasterClient().getUserAgent(), getMasterClient().getLogger()), cacheId, false);
479        } catch (URISyntaxException e) {
480          throw new TerminologyServiceException(e);
481        }
482        client.setTxCache(cache);
483        serverList.add(client);
484        serverMap.put(server, client);
485      }
486      client.seeUse(canonical, TerminologyClientContextUseType.readVS);
487      String criteria = canonical.contains("|") ? 
488          "?_format=json&url="+Utilities.URLEncode(canonical.substring(0, canonical.lastIndexOf("|")))+"&version="+Utilities.URLEncode(canonical.substring(canonical.lastIndexOf("|")+1)): 
489            "?_format=json&url="+Utilities.URLEncode(canonical);
490      request = Utilities.pathURL(client.getAddress(), "ValueSet"+ criteria);
491      Bundle bnd = client.getClient().search("ValueSet", criteria);
492      String rid = null;
493      if (bnd.getEntry().size() == 0) {
494        return null;
495      } else if (bnd.getEntry().size() > 1) {
496        List<ValueSet> vslist = new ArrayList<>();
497        for (BundleEntryComponent be : bnd.getEntry()) {
498          if (be.hasResource() && be.getResource() instanceof ValueSet) {
499            vslist.add((ValueSet) be.getResource());
500          }
501        }
502        Collections.sort(vslist, new ValueSetUtilities.ValueSetSorter());
503        rid = vslist.get(vslist.size()-1).getIdBase();
504      } else {
505        if (bnd.getEntryFirstRep().hasResource() && bnd.getEntryFirstRep().getResource() instanceof ValueSet) {
506          rid = bnd.getEntryFirstRep().getResource().getIdBase();
507        }
508      }
509      if (rid == null) {
510        return null;
511      }
512      ValueSet vs = (ValueSet) client.getClient().read("ValueSet", rid);
513      return new SourcedValueSet(server, vs);
514    } catch (Exception e) {
515      e.printStackTrace();
516      String msg = "Error resolving valueSet "+canonical+": "+e.getMessage()+" ("+request+")";
517      if (!internalLog.contains(msg)) {
518        internalLog.add(msg);
519      }
520      e.printStackTrace();
521      return null;
522    }
523  }
524
525  public SourcedCodeSystem findCodeSystemOnServer(String canonical) {
526    if (IGNORE_TX_REGISTRY || getMasterClient() == null || !useEcosystem) {
527      return null;
528    }
529    String request = Utilities.pathURL(monitorServiceURL, "resolve?fhirVersion="+factory.getVersion()+"&url="+Utilities.URLEncode(canonical));
530    if (usage != null) {
531      request = request + "&usage="+usage;
532    }
533    String server = null;
534    try {
535      JsonObject json = JsonParser.parseObjectFromUrl(request);
536      for (JsonObject item : json.getJsonObjects("authoritative")) {
537        if (server == null) {
538          server = item.asString("url");
539        }
540      }
541      for (JsonObject item : json.getJsonObjects("candidates")) {
542        if (server == null) {
543          server = item.asString("url");
544        }
545      }
546      if (server == null) {
547        return null;
548      }
549      if (server.contains("://tx.fhir.org")) {
550        try {
551          server = server.replace("tx.fhir.org", new URL(getMasterClient().getAddress()).getHost());
552        } catch (MalformedURLException e) {
553        }
554      }
555      TerminologyClientContext client = serverMap.get(server);
556      if (client == null) {
557        try {
558          client = new TerminologyClientContext(factory.makeClient("id"+(serverList.size()+1), server, getMasterClient().getUserAgent(), getMasterClient().getLogger()), cacheId, false);
559        } catch (URISyntaxException e) {
560          throw new TerminologyServiceException(e);
561        }
562        client.setTxCache(cache);
563        serverList.add(client);
564        serverMap.put(server, client);
565      }
566      client.seeUse(canonical, TerminologyClientContextUseType.readCS);
567      String criteria = canonical.contains("|") ? 
568          "?_format=json&url="+Utilities.URLEncode(canonical.substring(0, canonical.lastIndexOf("|")))+"&version="+Utilities.URLEncode(canonical.substring(canonical.lastIndexOf("|")+1)): 
569            "?_format=json&url="+Utilities.URLEncode(canonical);
570      request = Utilities.pathURL(client.getAddress(), "CodeSystem"+ criteria);
571      Bundle bnd = client.getClient().search("CodeSystem", criteria);
572      String rid = null;
573      if (bnd.getEntry().size() == 0) {
574        return null;
575      } else if (bnd.getEntry().size() > 1) {
576        List<CodeSystem> cslist = new ArrayList<>();
577        for (BundleEntryComponent be : bnd.getEntry()) {
578          if (be.hasResource() && be.getResource() instanceof CodeSystem) {
579            cslist.add((CodeSystem) be.getResource());
580          }
581        }
582        Collections.sort(cslist, new CodeSystemUtilities.CodeSystemSorter());
583        rid = cslist.get(cslist.size()-1).getIdBase();
584      } else {
585        if (bnd.getEntryFirstRep().hasResource() && bnd.getEntryFirstRep().getResource() instanceof CodeSystem) {
586          rid = bnd.getEntryFirstRep().getResource().getIdBase();
587        }
588      }
589      if (rid == null) {
590        return null;
591      }
592      CodeSystem vs = (CodeSystem) client.getClient().read("CodeSystem", rid);
593      return new SourcedCodeSystem(server, vs);
594    } catch (Exception e) {
595      e.printStackTrace();
596      String msg = "Error resolving valueSet "+canonical+": "+e.getMessage()+" ("+request+")";
597      if (!internalLog.contains(msg)) {
598        internalLog.add(msg);
599      }
600      e.printStackTrace();
601      return null;
602    }
603  }
604
605  public boolean supportsSystem(String system) throws IOException {
606    for (TerminologyClientContext client : serverList) {
607      if (client.supportsSystem(system)) {
608        return true;
609      }
610    }
611    return false;
612  }
613  
614}