Source code

001package org.hl7.fhir.r5.elementmodel;
002
003import java.io.IOException;
004import java.util.ArrayList;
005import java.util.Arrays;
006import java.util.HashMap;
007import java.util.HashSet;
008import java.util.List;
009import java.util.Locale;
010import java.util.Map;
011import java.util.Set;
012
013import lombok.extern.slf4j.Slf4j;
014import org.hl7.fhir.r5.context.ContextUtilities;
015import org.hl7.fhir.r5.context.IWorkerContext;
016import org.hl7.fhir.r5.elementmodel.Element.SpecialElement;
017import org.hl7.fhir.r5.extensions.ExtensionDefinitions;
018import org.hl7.fhir.r5.extensions.ExtensionUtilities;
019import org.hl7.fhir.r5.model.Base;
020import org.hl7.fhir.r5.model.CodeSystem;
021import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
022import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
023import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
024import org.hl7.fhir.r5.model.ContactDetail;
025import org.hl7.fhir.r5.model.DataType;
026import org.hl7.fhir.r5.model.DomainResource;
027import org.hl7.fhir.r5.model.ElementDefinition;
028import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingAdditionalComponent;
029import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionConstraintComponent;
030import org.hl7.fhir.r5.model.Extension;
031import org.hl7.fhir.r5.model.MarkdownType;
032import org.hl7.fhir.r5.model.PrimitiveType;
033import org.hl7.fhir.r5.model.Property;
034import org.hl7.fhir.r5.model.Resource;
035import org.hl7.fhir.r5.model.StringType;
036import org.hl7.fhir.r5.model.StructureDefinition;
037import org.hl7.fhir.r5.renderers.utils.RenderingContext;
038import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
039import org.hl7.fhir.r5.utils.UserDataNames;
040import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
041import org.hl7.fhir.utilities.Utilities;
042import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader;
043import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader.LanguagePreference;
044import org.hl7.fhir.utilities.i18n.I18nConstants;
045import org.hl7.fhir.utilities.i18n.LanguageFileProducer.LanguageProducerLanguageSession;
046import org.hl7.fhir.utilities.i18n.LanguageFileProducer.TextUnit;
047import org.hl7.fhir.utilities.i18n.LanguageFileProducer.TranslationUnit;
048import org.hl7.fhir.utilities.i18n.RenderingI18nContext;
049import org.hl7.fhir.utilities.validation.ValidationMessage;
050import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
051import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
052import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
053import org.hl7.fhir.utilities.xhtml.XhtmlNode;
054
055/**
056 * in here:
057 *   * generateTranslations
058 *   * importFromTranslations
059 *   * stripTranslations
060 *   * switchLanguage
061 *   
062 *  in the validator
063 *  
064 * @author grahamegrieve
065 *  generateTranslations = -langTransform export -src {src} -tgt {tgt} -dest {dest}
066 *  importFromTranslations =  -langTransform import -src {src} -tgt {tgt} -dest {dest}
067 */
068@MarkedToMoveToAdjunctPackage
069@Slf4j
070public class LanguageUtils {
071  
072  public static final List<String> TRANSLATION_SUPPLEMENT_RESOURCE_TYPES = Arrays.asList("CodeSystem", "StructureDefinition", "Questionnaire");
073
074  public static class TranslationUnitCollection {
075    List<TranslationUnit> list= new ArrayList<>();
076    Map<String, TranslationUnit> map = new HashMap<>();
077    public void add(TranslationUnit tu) {
078      String key = tu.getId()+"||"+tu.getSrcText();
079      if (!map.containsKey(key)) {
080        map.put(key, tu);
081        list.add(tu);
082      }
083      
084    }
085  }
086  IWorkerContext context;
087  private List<String> crlist;
088  
089  
090  public LanguageUtils(IWorkerContext context) {
091    super();
092    this.context = context;
093  }
094
095  public void generateTranslations(Element resource, LanguageProducerLanguageSession session) {
096    translate(null, resource, session, resource.fhirType());
097  }
098  
099  
100  private void translate(Element parent, Element element, LanguageProducerLanguageSession langSession, String path) {
101    String npath = pathForElement(path, element);
102    if (element.isPrimitive() && isTranslatable(element)) {
103      String base = element.primitiveValue();
104      if (base != null) {
105        String translation = getSpecialTranslation(path, parent, element, langSession.getTargetLang());
106        if (translation == null) {
107          translation = element.getTranslation(langSession.getTargetLang());
108        }
109        langSession.entry(new TextUnit(npath, contextForElement(element), base, translation));
110      }
111    }
112    for (Element c: element.getChildren()) {
113      if (!c.getName().equals("designation")) {
114        translate(element, c, langSession, npath);
115      }
116    }
117  }
118
119  private String contextForElement(Element element) {
120    throw new Error("Not done yet");
121  }
122
123  private String getSpecialTranslation(String path, Element parent, Element element, String targetLang) {
124    if (parent == null) {
125      return null;
126    }
127    String npath = parent.getBasePath();
128    if (Utilities.existsInList(npath, "CodeSystem.concept", "CodeSystem.concept.concept") && "CodeSystem.concept.display".equals(element.getBasePath())) {
129      return getDesignationTranslation(parent, targetLang);
130    }
131    if (Utilities.existsInList(npath, "ValueSet.compose.include.concept") && "ValueSet.compose.include.concept.display".equals(element.getBasePath())) {
132      return getDesignationTranslation(parent, targetLang);
133    }
134    if (Utilities.existsInList(npath, "ValueSet.expansion.contains", "ValueSet.expansion.contains.contains") && "ValueSet.expansion.contains.display".equals(element.getBasePath())) {
135      return getDesignationTranslation(parent, targetLang);
136    }
137    return null;
138  }
139
140  private String getDesignationTranslation(Element parent, String targetLang) {
141    for (Element e : parent.getChildren("designation")) {
142      String lang = e.getNamedChildValue("language");
143      if (langsMatch(targetLang, lang)) {
144        return e.getNamedChildValue("value");
145      }
146    }
147    return null;
148  }
149
150  private boolean isTranslatable(Element element) {    
151    return element.getProperty().isTranslatable();
152  }
153
154  private String pathForElement(String path, Element element) {
155    if (element.getSpecial() != null) {
156      String bp = element.getBasePath();
157      return pathForElement(bp, element.getProperty().getStructure().getType());
158    } else {
159      return (path == null ? element.getName() : path+"."+element.getName());
160    }
161  }
162  
163  private String pathForElement(String path, String type) {
164    // special case support for metadata elements prior to R5:
165    if (crlist == null) {
166      crlist = new ContextUtilities(context).getCanonicalResourceNames();
167    }
168    if (crlist.contains(type)) {
169      String fp = path.replace(type+".", "CanonicalResource.");
170      if (Utilities.existsInList(fp,
171         "CanonicalResource.url", "CanonicalResource.identifier", "CanonicalResource.version", "CanonicalResource.name", 
172         "CanonicalResource.title", "CanonicalResource.status", "CanonicalResource.experimental", "CanonicalResource.date",
173         "CanonicalResource.publisher", "CanonicalResource.contact", "CanonicalResource.description", "CanonicalResource.useContext", 
174         "CanonicalResource.jurisdiction"))  {
175        return fp;
176      }
177    }
178    return path; 
179  }
180  
181  
182  public int importFromTranslations(Element resource, List<TranslationUnit> translations) {
183    return importFromTranslations(resource.fhirType(), null, resource, translations, new HashSet<>());
184  }
185  
186  public int importFromTranslations(Element resource, List<TranslationUnit> translations, List<ValidationMessage> messages) {
187    Set<TranslationUnit> usedUnits = new HashSet<>();
188    int r = 0;
189    if (resource.fhirType().equals("StructureDefinition")) {
190      r = importFromTranslationsForSD(null, resource, translations, usedUnits);
191    } else {
192     r = importFromTranslations(null, null, resource, translations, usedUnits);
193    }
194    for (TranslationUnit t : translations) {
195      if (!usedUnits.contains(t)) {
196        if (messages != null) {
197          messages.add(new ValidationMessage(Source.Publisher, IssueType.INFORMATIONAL, t.getId(), "Unused '"+t.getLanguage()+"' translation '"+t.getSrcText()+"' -> '"+t.getTgtText()+"'", IssueSeverity.INFORMATION));
198        }
199      }
200    }
201    return r;
202  }
203  
204  public int importFromTranslations(Resource resource, List<TranslationUnit> translations, List<ValidationMessage> messages) {
205    Set<TranslationUnit> usedUnits = new HashSet<>();
206    int r = 0;
207    if (resource.fhirType().equals("StructureDefinition")) {
208      // todo... r = importFromTranslationsForSD(null, resource, translations, usedUnits);
209    } else {
210     r = importResourceFromTranslations(null, resource, translations, usedUnits, resource.fhirType());
211    }
212    for (TranslationUnit t : translations) {
213      if (!usedUnits.contains(t)) {
214        if (messages != null) {
215          messages.add(new ValidationMessage(Source.Publisher, IssueType.INFORMATIONAL, t.getId(), "Unused '"+t.getLanguage()+"' translation '"+t.getSrcText()+"' -> '"+t.getTgtText()+"'", IssueSeverity.INFORMATION));
216        }
217      }
218    }
219    return r;
220  }
221  
222
223  /*
224   * */
225  private int importFromTranslationsForSD(Object object, Element resource, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits) {
226    int r = 0;
227    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.name", "name");
228    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.title", "title");
229    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.publisher", "publisher");
230    for (Element cd : resource.getChildrenByName("contact")) {
231      r = r + checkForTranslations(translations, usedUnits, cd, "StructureDefinition.contact.name", "name");
232    }
233    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.purpose", "purpose");
234    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.copyright", "copyright");
235    Element diff = resource.getNamedChild("differential");
236    if (diff != null) {
237      for (Element ed : diff.getChildrenByName("element")) {
238        String id = ed.getNamedChildValue("id");
239        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/label", "label");
240        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/short", "short");
241        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/definition", "definition");
242        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/comment", "comment");
243        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/requirements", "requirements");
244        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/meaningWhenMissing", "meaningWhenMissing");
245        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/orderMeaning", "orderMeaning");
246        //      for (ElementDefinitionConstraintComponent con : ed.getConstraint()) {
247        //        addToList(list, lang, con, ed.getId()+"/constraint", "human", con.getHumanElement());
248        //      }
249        //      if (ed.hasBinding()) {
250        //        addToList(list, lang, ed.getBinding(), ed.getId()+"/b/desc", "description", ed.getBinding().getDescriptionElement());
251        //        for (ElementDefinitionBindingAdditionalComponent ab : ed.getBinding().getAdditional()) {
252        //          addToList(list, lang, ab, ed.getId()+"/ab/doco", "documentation", ab.getDocumentationElement());
253        //          addToList(list, lang, ab, ed.getId()+"/ab/short", "shortDoco", ab.getShortDocoElement());
254        //        }
255        //      }
256      }
257    }
258    return r;
259  }
260
261  private int checkForTranslations(List<TranslationUnit> translations, Set<TranslationUnit> usedUnits, Element context, String tname, String pname) {
262    int r = 0;
263    Element child = context.getNamedChild(pname);
264    if (child != null) {
265      String v = child.primitiveValue();
266      if (v != null) {
267        for (TranslationUnit tu : translations) {
268          if (tname.equals(tu.getId()) && v.equals(tu.getSrcText())) {
269            usedUnits.add(tu);
270            child.setTranslation(tu.getLanguage(), tu.getTgtText());
271            r++;
272          }
273        }
274      }
275    }
276    return r;
277  }
278
279  private int importResourceFromTranslations(Base parent, Base element, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits, String path) {
280    int t = 0;
281    if (element.isPrimitive() && isTranslatable(element, path) && element instanceof org.hl7.fhir.r5.model.Element) {
282      org.hl7.fhir.r5.model.Element e = (org.hl7.fhir.r5.model.Element) element;
283      String base = element.primitiveValue();
284      if (base != null) {
285        String epath = pathForElement(path, element.fhirType());
286        Set<TranslationUnit> tlist = findTranslations(epath, base, translations);
287        for (TranslationUnit translation : tlist) {
288          t++;
289          if (!handleAsSpecial(parent, element, translation)) {
290            usedUnits.add(translation);
291            if (translation.getTgtText() != null) {
292              ExtensionUtilities.setLanguageTranslation(e, translation.getLanguage(), translation.getTgtText());
293            } else {
294              log.warn("?");
295            }
296          }
297        }
298      }
299    }
300    for (Property c : element.children()) {
301      for (Base v : c.getValues()) {
302        if (!c.getName().equals("designation") && !isTranslation(v)) {
303          t = t + importResourceFromTranslations(element, v, translations, usedUnits, genPath(c, v, path, c.getName()));
304        }
305      }
306    }
307    return t;
308  }
309
310  private String genPath(Property c, Base v, String path, String name) {
311    // special cases: recursion
312    if ("ImplementationGuide.definition.page".equals(path) && "page".equals(name)) {
313      return path;
314    }
315    if ("ValueSet.expansion.contains".equals(path) && "contains".equals(name)) {
316      return path;
317    }
318    if ("ValueSet.expansion.contains".equals(path) && "contains".equals(name)) {
319      return path;
320    }
321    if (v.isResource() && !"contained".equals(name)) {
322      return v.fhirType();
323    } else {
324      return path+"."+name;
325    }
326  }
327
328  private boolean isTranslation(Base element) {
329    return "Extension".equals(element.fhirType()) && element.getChildByName("url").hasValues()  && ExtensionDefinitions.EXT_TRANSLATION.equals(element.getChildByName("url").getValues().get(0).primitiveValue());
330  }
331
332  private boolean handleAsSpecial(Base parent, Base element, TranslationUnit translation) {
333    return false;
334  }
335
336  private boolean isTranslatable(Base element, String path) {
337    return (Utilities.existsInList(element.fhirType(), "string", "markdown") 
338        || isTranslatable(path)) && !isExemptFromTranslations(path);
339  }
340
341  private int importFromTranslations(String path, Element parent, Element element, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits) {
342    String npath = pathForElement(path, element);
343    int t = 0;
344    if (element.isPrimitive() && isTranslatable(element) && !isExemptFromTranslations(npath)) {
345      String base = element.primitiveValue();
346      if (base != null) {
347        Set<TranslationUnit> tlist = findTranslations(npath, base, translations);
348        for (TranslationUnit translation : tlist) {
349          t++;
350          if (!handleAsSpecial(parent, element, translation)) {
351            element.setTranslation(translation.getLanguage(), translation.getTgtText());
352            usedUnits.add(translation);
353          }
354        }
355      }
356    }
357    // Create a copy of the children collection before iterating
358    List<Element> childrenCopy = List.copyOf(element.getChildren());
359    for (Element c : childrenCopy) {
360      if (!c.getName().equals("designation")) {
361        t = t + importFromTranslations(npath, element, c, translations, usedUnits);
362      }
363    }
364    return t;
365  }
366
367  private boolean handleAsSpecial(Element parent, Element element, TranslationUnit translation) {
368    if (parent == null) {
369      return false;
370    }
371    if (Utilities.existsInList(parent.getBasePath(), "CodeSystem.concept", "CodeSystem.concept.concept") && "CodeSystem.concept.display".equals(element.getBasePath())) {
372      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
373    }
374    if (Utilities.existsInList(parent.getBasePath(), "ValueSet.compose.include.concept") && "ValueSet.compose.include.concept.display".equals(element.getBasePath())) {
375      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
376    }
377    if (Utilities.existsInList(parent.getBasePath(), "ValueSet.expansion.contains", "ValueSet.expansion.contains.contains") && "ValueSet.expansion.contains.display".equals(element.getBasePath())) {
378      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
379    }
380    return false;
381  }
382
383  private boolean setDesignationTranslation(Element parent, String targetLang, String translation) {
384    for (Element e : parent.getChildren("designation")) {
385      String lang = e.getNamedChildValue("language");
386      if (langsMatch(targetLang, lang)) {
387        Element value = e.getNamedChild("value");
388        if (value != null) {
389          value.setValue(translation);
390        } else {
391          e.addElement("value").setValue(translation);
392        }
393        return true;
394      }
395    }
396    Element d = parent.addElement("designation");
397    d.addElement("language").setValue(targetLang);
398    d.addElement("value").setValue(translation);
399    return true;
400  }
401
402  private Set<TranslationUnit> findTranslations(String path, String src, List<TranslationUnit> translations) {
403    Set<TranslationUnit> res = new HashSet<>();
404    for (TranslationUnit translation : translations) {
405      if (path.equals(translation.getId()) && src.equals(translation.getSrcText())) {
406        res.add(translation);
407      }
408    }
409    return res;
410  }
411
412  public static boolean langsMatchExact(AcceptLanguageHeader langs, String srcLang) {
413    if (langs == null) {
414      return false;
415    }
416    for (LanguagePreference lang : langs.getLangs()) {
417      if (lang.getValue() > 0) {
418        if ("*".equals(lang.getLang())) {
419          return true;
420        } else {
421          return langsMatch(lang.getLang(), srcLang);
422        }
423      }
424    }
425    return false;
426  }
427
428  public static boolean langsMatch(AcceptLanguageHeader langs, String srcLang) {
429    if (langs == null) {
430      return false;
431    }
432    for (LanguagePreference lang : langs.getLangs()) {
433      if (lang.getValue() > 0) {
434        if ("*".equals(lang.getLang())) {
435          return true;
436        } else {
437          boolean ok = langsMatch(lang.getLang(), srcLang);
438          if (ok) {
439            return true;
440          }
441        }
442      }
443    }
444    return false;
445  }
446
447  public static boolean langsMatchExact(String dstLang, String srcLang) {
448    return dstLang == null ? false : dstLang.equals(srcLang);
449  }
450
451  public static boolean langsMatch(String dstLang, String srcLang) {
452    if (dstLang == null && srcLang == null) {
453      return true;
454    } if (dstLang == null) {
455      return srcLang.equals("en") || srcLang.startsWith("en-");
456    } else if (srcLang == null) {
457      return dstLang.equals("en") || dstLang.startsWith("en-");
458    } else {
459      return dstLang.startsWith(srcLang) || "*".equals(srcLang);
460    }
461  }
462
463  public void fillSupplement(CodeSystem csSrc, CodeSystem csDst, List<TranslationUnit> list) {
464    csDst.setUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT, csSrc);
465    csDst.setUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS, list);
466    for (TranslationUnit tu : list) {
467      String code = tu.getId();
468      String subCode = null;
469      if (code.contains("@")) {
470        subCode = code.substring(code.indexOf("@")+1);
471        code = code.substring(0, code.indexOf("@"));
472      }
473      ConceptDefinitionComponent cdSrc = CodeSystemUtilities.getCode(csSrc, tu.getId());
474      if (cdSrc == null) {
475        addOrphanTranslation(csSrc, tu);
476      } else {
477        ConceptDefinitionComponent cdDst = CodeSystemUtilities.getCode(csDst, cdSrc.getCode());
478        if (cdDst == null) {
479          cdDst = csDst.addConcept().setCode(cdSrc.getCode());
480        }
481        String tt = tu.getTgtText();
482        if (tt.startsWith("!!")) {
483          tt = tt.substring(3);
484        }
485        if (subCode == null) {
486          cdDst.setDisplay(tt);
487        } else if ("definition".equals(subCode)) {
488          cdDst.setDefinition(tt);
489        } else {
490          boolean found = false;
491          for (ConceptDefinitionDesignationComponent d : cdSrc.getDesignation()) {
492            if (d.hasUse() && subCode.equals(d.getUse().getCode())) {
493              found = true;
494              cdDst.addDesignation().setUse(d.getUse()).setLanguage(tu.getLanguage()).setValue(tt); //.setUserData(SUPPLEMENT_SOURCE, tu);       
495              break;
496            }
497          }
498          if (!found) {
499            for (Extension e : cdSrc.getExtension()) {
500              if (subCode.equals(tail(e.getUrl()))) {
501                found = true;
502                cdDst.addExtension().setUrl(e.getUrl()).setValue(
503                    e.getValue().fhirType().equals("markdown") ? new MarkdownType(tt) : new StringType(tt)); //.setUserData(SUPPLEMENT_SOURCE, tu);          
504                break;
505              }
506            }
507          }
508          if (!found) {
509            addOrphanTranslation(csSrc, tu);            
510          }
511        }
512      }      
513    }    
514  }
515
516  private String tail(String url) {
517    return url.contains("/") ? url.substring(url.lastIndexOf("/")+1) : url;
518  }
519
520  private void addOrphanTranslation(CodeSystem cs, TranslationUnit tu) {
521    List<TranslationUnit> list = (List<TranslationUnit>) cs.getUserData(UserDataNames.LANGUTILS_ORPHAN);
522    if (list == null) {
523      list = new ArrayList<>();
524      cs.setUserData(UserDataNames.LANGUTILS_ORPHAN, list);
525    }
526    list.add(tu);
527  }
528
529  public String nameForLang(String lang) {
530    // todo: replace with structures from loading languages properly
531    switch (lang) {
532    case "en" : return "English";
533    case "de" : return "German";
534    case "es" : return "Spanish";
535    case "nl" : return "Dutch";
536    }
537    return Utilities.capitalize(lang);
538  }
539
540  public String titleForLang(String lang) {
541    // todo: replace with structures from loading languages properly
542    switch (lang) {
543    case "en" : return "English";
544    case "de" : return "German";
545    case "es" : return "Spanish";
546    case "nl" : return "Dutch";
547    }
548    return Utilities.capitalize(lang);
549  }
550
551  public boolean handlesAsResource(Resource resource) {
552    return (resource instanceof CodeSystem && resource.hasUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT)) || (resource instanceof StructureDefinition);
553  }
554
555  public boolean handlesAsElement(Element element) {
556    return true; // for now...
557  }
558
559  public List<TranslationUnit> generateTranslations(Resource res, String lang) {
560    List<TranslationUnit> list = new ArrayList<>();
561    if (res instanceof StructureDefinition) {
562      StructureDefinition sd = (StructureDefinition) res;
563      generateTranslations(list, sd, lang);
564      if (res.hasUserData(UserDataNames.LANGUTILS_ORPHAN)) {
565        List<TranslationUnit> orphans = (List<TranslationUnit>) res.getUserData(UserDataNames.LANGUTILS_ORPHAN);
566        for (TranslationUnit t : orphans) {
567          if (!hasInList(list, t.getId(), t.getSrcText())) {
568            list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext(), t.getSrcText(), t.getTgtText()));
569          }
570        }
571      }
572    } else {
573      CodeSystem cs = (CodeSystem) res.getUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT);
574      List<TranslationUnit> inputs = res.hasUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS) ? (List<TranslationUnit>) res.getUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS) : new ArrayList<>();
575      for (ConceptDefinitionComponent cd : cs.getConcept()) {
576        generateTranslations(list, cd, lang, inputs);
577      }
578      if (cs.hasUserData(UserDataNames.LANGUTILS_ORPHAN)) {
579        List<TranslationUnit> orphans = (List<TranslationUnit>) cs.getUserData(UserDataNames.LANGUTILS_ORPHAN);
580        for (TranslationUnit t : orphans) {
581          if (!hasInList(list, t.getId(), t.getSrcText())) {
582            list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext(), t.getSrcText(), t.getTgtText()));
583          }
584        }
585      }
586    }
587    return list;
588  }
589
590  private void generateTranslations(List<TranslationUnit> list, StructureDefinition sd, String lang) {
591    addToList(list, lang, sd, "StructureDefinition.name", "name", sd.getNameElement());
592    addToList(list, lang, sd, "StructureDefinition.title", "title", sd.getTitleElement());
593    addToList(list, lang, sd, "StructureDefinition.publisher", "publisher", sd.getPublisherElement());
594    for (ContactDetail cd : sd.getContact()) {
595      addToList(list, lang, cd, "StructureDefinition.contact.name", "name", cd.getNameElement());
596    }
597    addToList(list, lang, sd, "StructureDefinition.purpose", "purpose", sd.getPurposeElement());
598    addToList(list, lang, sd, "StructureDefinition.copyright", "copyright", sd.getCopyrightElement());
599    for (ElementDefinition ed : sd.getDifferential().getElement()) {
600      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/label", "label", ed.getLabelElement());
601      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/short", "short", ed.getShortElement());
602      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/definition", "definition", ed.getDefinitionElement());
603      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/comment", "comment", ed.getCommentElement());
604      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/requirements", "requirements", ed.getRequirementsElement());
605      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/meaningWhenMissing", "meaningWhenMissing", ed.getMeaningWhenMissingElement());
606      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/orderMeaning", "orderMeaning", ed.getOrderMeaningElement());
607      for (ElementDefinitionConstraintComponent con : ed.getConstraint()) {
608        addToList(list, lang, con, "StructureDefinition.element."+ed.getId()+"/constraint", "human", con.getHumanElement());
609      }
610      if (ed.hasBinding()) {
611        addToList(list, lang, ed.getBinding(), "StructureDefinition.element."+ed.getId()+"/b/desc", "description", ed.getBinding().getDescriptionElement());
612        for (ElementDefinitionBindingAdditionalComponent ab : ed.getBinding().getAdditional()) {
613          addToList(list, lang, ab, "StructureDefinition.element."+ed.getId()+"/ab/doco", "documentation", ab.getDocumentationElement());
614          addToList(list, lang, ab, "StructureDefinition.element."+ed.getId()+"/ab/short", "shortDoco", ab.getShortDocoElement());
615        }
616      }
617    }
618  }
619
620  private void addToList(List<TranslationUnit> list, String lang, Base ctxt, String name, String propName, DataType value) {
621    if (value != null && value.hasPrimitiveValue()) {
622      if (!hasInList(list, name, value.primitiveValue())) {
623        list.add(new TranslationUnit(lang, name, ctxt.getNamedProperty(propName).getDefinition(), value.primitiveValue(), value.getTranslation(lang)));
624      }
625    }
626    
627  }
628
629  private void generateTranslations(List<TranslationUnit> list, ConceptDefinitionComponent cd, String lang, List<TranslationUnit> inputs) {
630    // we generate translation units for the display, the definition, and any designations and extensions that we find
631    // the id of the designation is the use.code (there will be a use) and for the extension, the tail of the extension URL 
632    // todo: do we need to worry about name clashes? why would we, and more importantly, how would we solve that?
633
634    addTranslationUnit(list, cd.getCode(), cd.getDisplay(), lang, inputs);
635    if (cd.hasDefinition()) {
636      addTranslationUnit(list, cd.getCode()+"@definition", cd.getDefinition(), lang, inputs);        
637    }
638    for (ConceptDefinitionDesignationComponent d : cd.getDesignation()) {
639      addTranslationUnit(list, cd.getCode()+"@"+d.getUse().getCode(), d.getValue(), lang, inputs);              
640    }
641    for (Extension e : cd.getExtension()) {
642      addTranslationUnit(list, cd.getCode()+"@"+tail(e.getUrl()), e.getValue().primitiveValue(), lang, inputs);                    
643    }
644  }
645
646  private void addTranslationUnit(List<TranslationUnit> list, String id, String srcText, String lang, List<TranslationUnit> inputs) {
647    TranslationUnit existing = null;
648    for (TranslationUnit t : inputs) {
649      if (id.equals(t.getId())) {
650        existing = t;
651        break;
652      }
653    }
654    if (!hasInList(list, id, srcText)) {
655      // not sure what to do with context?
656      if (existing == null) {
657        list.add(new TranslationUnit(lang, id, null, srcText, null));
658      } else if (srcText.equals(existing.getSrcText())) {
659        list.add(new TranslationUnit(lang, id, null, srcText, existing.getTgtText()));
660      } else {
661        list.add(new TranslationUnit(lang, id, null, srcText, "!!"+existing.getTgtText()).setOriginal(existing.getSrcText()));
662      }
663    }
664  }
665  
666  private String getDefinition(ConceptDefinitionComponent cd) {
667    ConceptPropertyComponent v = CodeSystemUtilities.getProperty(cd, "translation-context");
668    if (v != null && v.hasValue()) {
669      return v.getValue().primitiveValue();
670    } else {
671      return cd.getDefinition();
672    }
673  }
674
675  public List<TranslationUnit> generateTranslations(Element e, String lang) {
676    TranslationUnitCollection list = new TranslationUnitCollection();
677    generateTranslations(e, lang, list, e.fhirType());
678    return list.list;
679  }
680
681  private void generateTranslations(Element e, String lang, TranslationUnitCollection list, String path) {
682    String npath = pathForElement(path, e);
683    if ((e.getProperty().isTranslatable() || isTranslatable(e.getProperty().getDefinition().getBase().getPath())) 
684        && !isExemptFromTranslations(e.getProperty().getDefinition().getBase().getPath())) {
685      String id = e.getProperty().getDefinition().getBase().getPath(); // .getProperty().getDefinition().getPath();
686      String context = e.getProperty().getDefinition().getDefinition();
687      String src = e.primitiveValue();
688      String tgt = getTranslation(e, lang);
689      if (!hasInList(list.list, id, src)) {
690        list.add(new TranslationUnit(lang, id, context, src, tgt));
691      }
692    }
693    if (e.hasChildren()) {
694      for (Element c : e.getChildren()) {
695        generateTranslations(c, lang, list, npath);
696      }
697    }
698  }
699
700  private boolean hasInList(List<TranslationUnit> list, String id, String src) {
701    for (TranslationUnit t : list) {
702      if (t.getId() != null && t.getId().equals(id) && t.getSrcText()!= null && t.getSrcText().equals(src)) {
703        return true;
704      }
705    }
706    return false;
707  }
708
709  /**
710   * override specifications
711   * 
712   * @param path
713   * @return
714   */
715  private boolean isTranslatable(String path) {
716    return Utilities.existsInList(path, "TestCases.publisher",
717        "TestCases.contact.telecom.value",
718        "TestCases.definition",
719        "TestCases.parameter.name",
720        "TestCases.parameter.description",
721        "TestCases.scope.description ",
722        "TestCases.dependency.description",
723        "TestCases.mode.description",
724        "TestCases.suite",
725        "TestCases.suite.name",
726        "TestCases.suite.description",
727        "TestCases.suite.test",
728        "TestCases.suite.test.name",
729        "TestCases.suite.test.description",
730        "TestCases.suite.test.assert.human",
731        "ActorDefinition.title",
732        "ActorDefinition.description",
733        "ActorDefinition.purpose",
734        "ActorDefinition.copyright",
735        "ActorDefinition.copyrightLabel",
736        "ActorDefinition.documentation",
737        "Requirements.title",
738        "Requirements.publisher",
739        "Requirements.description",
740        "Requirements.purpose",
741        "Requirements.copyright",
742        "Requirements.copyrightLabel",
743        "Requirements.statement.label",
744        "Requirements.statement.requirement");    
745  }
746
747  private boolean isExemptFromTranslations(String path) {
748    if (path.endsWith(".reference")) {
749      return true;
750    }
751    return Utilities.existsInList(path, 
752        "ImplementationGuide.definition.parameter.value", "ImplementationGuide.dependsOn.version", "ImplementationGuide.dependsOn.id",
753        "CanonicalResource.name", 
754        "CapabilityStatement.rest.resource.searchRevInclude", "CapabilityStatement.rest.resource.searchInclude", "CapabilityStatement.rest.resource.searchParam.name",
755        "SearchParameter.expression", "SearchParameter.xpath",
756        "ExampleScenario.actor.actorId", "ExampleScenario.instance.resourceId", "ExampleScenario.instance.containedInstance.resourceId", "ExampleScenario.instance.version.versionId", 
757          "ExampleScenario.process.step.operation.number", "ExampleScenario.process.step.operation.initiator", "ExampleScenario.process.step.operation.receiver",
758        "ExampleScenario.process.step.operation.number", "ExampleScenario.process.step.operation.initiator", "ExampleScenario.process.step.operation.receiver",
759        "OperationDefinition.parameter.max", "OperationDefinition.overload.parameterName",
760        "StructureMap.group.rule.source.type", "StructureMap.group.rule.source.element", "StructureMap.group.rule.target.element");
761  }
762
763  private String getTranslation(Element e, String lang) {
764    if (!e.hasChildren()) {
765      return null;
766    }
767    for (Element ext : e.getChildren()) {
768      if ("Extension".equals(ext.fhirType()) && "http://hl7.org/fhir/StructureDefinition/translation".equals(ext.getNamedChildValue("url"))) {
769        String l = null;
770        String v = null;
771        for (Element subExt : ext.getChildren()) {
772          if ("Extension".equals(subExt.fhirType()) && "lang".equals(subExt.getNamedChildValue("url"))) {
773            l = subExt.getNamedChildValue("value");
774          }
775          if ("Extension".equals(subExt.fhirType()) && "content".equals(subExt.getNamedChildValue("url"))) {
776            v = subExt.getNamedChildValue("value");
777          }
778        }
779        if (lang.equals(l)) {
780          return v;
781        }
782      }
783    }
784    return null;
785  }
786 
787  public boolean switchLanguage(Base r, String lang, boolean markLanguage, boolean contained, String resourceLang, String defaultLang, List<ValidationMessage> errors) {
788    boolean changed = false;
789    if (r.isPrimitive()) {
790
791      PrimitiveType<?> dt = (PrimitiveType<?>) r;
792      String cnt = ExtensionUtilities.getLanguageTranslation(dt, lang); 
793      dt.removeExtension(ExtensionDefinitions.EXT_TRANSLATION);
794      if (cnt != null) {
795        dt.setValueAsString(cnt);
796        changed = true;
797      }
798    }
799
800    if (r.fhirType().equals("Narrative")) {
801      Base div = r.getChildValueByName("div");
802      Base status = r.getChildValueByName("status");
803
804      XhtmlNode xhtml = div.getXhtml();
805      xhtml = adjustToLang(xhtml, lang, status == null ? null : status.primitiveValue(), resourceLang, defaultLang, errors);
806      if (xhtml == null) {
807        r.removeChild("div", div);
808      } else {
809        div.setXhtml(xhtml);
810      }
811    }
812    for (Property p : r.children()) {
813      for (Base c : p.getValues()) {
814        changed = switchLanguage(c, lang, markLanguage, p.getName().equals("contained"), resourceLang, defaultLang, errors) || changed;
815      }
816    }
817    if (markLanguage && r.isResource() && !contained) {
818      Resource res = (Resource) r;
819      res.setLanguage(lang);
820      changed = true;
821    }
822    return changed;
823  }
824  
825  public XhtmlNode divForLang(DomainResource r, String lang, String defaultLang, List<ValidationMessage> errors) {
826    if (!r.hasText() || !r.getText().hasDiv())
827      return null;
828    else
829      return divForLang(r.getText().getDiv(), lang, r.getLanguage(), defaultLang, errors);
830  }
831  
832  public XhtmlNode divForLang(XhtmlNode xhtml, String lang, String resourceLang, String defaultLang, List<ValidationMessage> errors) {
833    if (xhtml==null)
834      return null;
835   
836    boolean foundLangDivs = false;
837    for (XhtmlNode div : xhtml.getChildNodes()) {
838      if ("div".equals(div.getName())) {
839        String l = div.hasAttribute("lang") ? div.getAttribute("lang") : div.getAttribute("xml:lang");
840        if (l!=null)
841          foundLangDivs = true;
842        if (lang.equals(l)) {
843          return div;
844        }       
845      }
846    }
847
848    // If the base div declares a language that matches, then the whole div is the content 
849    String l = xhtml.hasAttribute("lang") ? xhtml.getAttribute("lang") : xhtml.getAttribute("xml:lang");
850    if (lang.equals(l)) {
851      return xhtml;
852    }
853    
854    // If there's no language declared at all, then the narrative is presumed to be the language the resource declares
855    // (if there is one), or the default language for the IG if the resource doesn't have a language.
856    if (!foundLangDivs && (resourceLang!=null ? lang.equals(resourceLang) : lang.equals(defaultLang))) {
857      return xhtml;
858    }
859    
860    // If we didn't find a div for the requested language, then we'll either use narrative div for the default language
861    // (if it wasn't generated) or will leave the narrative as null, which will force it to be generated for this language
862    if (!lang.equals(defaultLang)) {
863      XhtmlNode defaultDiv = divForLang(xhtml, defaultLang, resourceLang, defaultLang, null);
864      if (defaultDiv != null) {
865        if (scanForGeneratedNarrative(defaultDiv, defaultLang)) {
866          return null;
867        }
868      }
869      
870      if (errors != null)
871        errors.add(new ValidationMessage(Source.Publisher, IssueType.BUSINESSRULE, "IG", context.formatMessage(I18nConstants.NARRATIVE_NOT_TRANSLATED, defaultLang, lang), IssueSeverity.WARNING));
872      return defaultDiv;
873    }
874    
875    return null;
876  }
877  
878  public static boolean scanForGeneratedNarrative(XhtmlNode x, String lang) {
879    RenderingI18nContext rc = new RenderingI18nContext();
880    rc.setLocale(Locale.forLanguageTag(lang));
881    return scanForGeneratedNarrative(x, lang, rc);
882  }
883    
884  private static boolean scanForGeneratedNarrative(XhtmlNode x, String lang, RenderingI18nContext rc) {
885
886    if (x.getContent() != null && x.getContent().contains( rc.formatPhrase(RenderingContext.PROF_DRIV_GEN_NARR_TECH, "", "").trim())) {
887      return true;
888    }
889    for (XhtmlNode c : x.getChildNodes()) {
890      if (scanForGeneratedNarrative(c, lang)) {
891        return true;
892      }
893    }
894    return false;
895  }
896
897  private XhtmlNode adjustToLang(XhtmlNode xhtml, String lang, String status, String resourceLang, String defaultLang, List<ValidationMessage> errors) {
898    if (xhtml == null) {
899      return null;
900    }
901    xhtml = divForLang(xhtml, lang, resourceLang, defaultLang, errors);
902    
903    if (xhtml!=null)
904      return xhtml;
905    
906    // if the root language is null and the status is not additional
907    // it can be regenerated
908    if (Utilities.existsInList(status, "generated", "extensions")) {
909      return null;
910    }    
911    // well, really, not much we can do...
912    return xhtml;
913  }
914
915  public boolean switchLanguage(Element e, String lang, boolean markLanguage, String resourceLang, String defaultLang, List<ValidationMessage> errors) throws IOException {
916    boolean changed = false;
917    if (e.getProperty().isTranslatable()) {
918      String cnt = getTranslation(e, lang);
919      e.removeExtension(ExtensionDefinitions.EXT_TRANSLATION);
920      if (cnt != null) {
921        e.setValue(cnt);
922        changed = true;
923      }
924    }
925    if (e.fhirType().equals("Narrative") && e.hasChild("div")) {
926      XhtmlNode xhtml = e.getNamedChild("div").getXhtml();
927      xhtml = adjustToLang(xhtml, lang, e.getNamedChildValue("status"), resourceLang, defaultLang, errors);
928      if (xhtml == null) {
929        e.removeChild("div");
930      } else {
931        e.getNamedChild("div").setXhtml(xhtml);
932      }
933    }
934    if (e.hasChildren()) {
935      for (Element c : e.getChildren()) {
936        changed = switchLanguage(c, lang, markLanguage, resourceLang, defaultLang, errors) || changed;
937      }
938    }
939    if (markLanguage && e.isResource() && e.getSpecial() != SpecialElement.CONTAINED) {
940      e.setChildValue("language", lang);
941      changed = true;
942    }
943    return changed;
944  }
945
946  public boolean hasTranslation(org.hl7.fhir.r5.model.Element e, String lang) {
947    return getTranslation(e, lang) != null;
948  }
949
950  public String getTranslation(org.hl7.fhir.r5.model.Element e, String lang) {
951    for (Extension ext : e.getExtensionsByUrl(ExtensionDefinitions.EXT_TRANSLATION)) {
952      String l = ext.getExtensionString("lang");
953      String v =  ext.getExtensionString("content");
954      if (langsMatch(l, lang) && v != null) {
955        return v;
956      }
957    }
958    return null;
959  }
960
961  public String getTranslationOrBase(PrimitiveType<?> e, String lang) {
962    for (Extension ext : e.getExtensionsByUrl(ExtensionDefinitions.EXT_TRANSLATION)) {
963      String l = ext.getExtensionString("lang");
964      String v =  ext.getExtensionString("content");
965      if (langsMatch(l, lang) && v != null) {
966        return v;
967      }
968    }
969    return e.primitiveValue();
970  }
971
972  public Element copyToLanguage(Element element, String lang, boolean markLanguage, String resourceLang, String defaultLang, List<ValidationMessage> errors) throws IOException {
973    Element result = (Element) element.copy();
974    switchLanguage(result, lang, markLanguage, resourceLang, defaultLang, errors);
975    return result;
976  }
977
978  public Resource copyToLanguage(Resource res, String lang, boolean markLanguage, String defaultLang, List<ValidationMessage> errors) {
979    if (res == null) {
980      return null;
981    }
982    Resource r = res.copy();
983    switchLanguage(r, lang, markLanguage, false, res.getLanguage(), defaultLang, errors);    
984    return r;
985  }
986}