001package org.hl7.fhir.r5.elementmodel;
002
003import java.io.IOException;
004import java.util.ArrayList;
005import java.util.Arrays;
006import java.util.HashMap;
007import java.util.HashSet;
008import java.util.List;
009import java.util.Map;
010import java.util.Set;
011
012import lombok.extern.slf4j.Slf4j;
013import org.checkerframework.checker.units.qual.cd;
014import org.hl7.fhir.r5.context.ContextUtilities;
015import org.hl7.fhir.r5.context.IWorkerContext;
016import org.hl7.fhir.r5.elementmodel.Element.SpecialElement;
017import org.hl7.fhir.r5.model.Base;
018import org.hl7.fhir.r5.model.CodeSystem;
019import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
020import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
021import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
022import org.hl7.fhir.r5.model.ContactDetail;
023import org.hl7.fhir.r5.model.DataType;
024import org.hl7.fhir.r5.model.ElementDefinition;
025import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingAdditionalComponent;
026import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionConstraintComponent;
027import org.hl7.fhir.r5.model.Extension;
028import org.hl7.fhir.r5.model.MarkdownType;
029import org.hl7.fhir.r5.model.PrimitiveType;
030import org.hl7.fhir.r5.model.Property;
031import org.hl7.fhir.r5.model.Resource;
032import org.hl7.fhir.r5.model.StringType;
033import org.hl7.fhir.r5.model.StructureDefinition;
034import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
035import org.hl7.fhir.r5.utils.ToolingExtensions;
036import org.hl7.fhir.r5.utils.UserDataNames;
037import org.hl7.fhir.utilities.FileUtilities;
038import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
039import org.hl7.fhir.utilities.Utilities;
040import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader;
041import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader.LanguagePreference;
042import org.hl7.fhir.utilities.i18n.LanguageFileProducer;
043import org.hl7.fhir.utilities.i18n.LanguageFileProducer.LanguageProducerLanguageSession;
044import org.hl7.fhir.utilities.i18n.LanguageFileProducer.TextUnit;
045import org.hl7.fhir.utilities.i18n.LanguageFileProducer.TranslationUnit;
046import org.hl7.fhir.utilities.json.model.JsonElement;
047import org.hl7.fhir.utilities.validation.ValidationMessage;
048import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
049import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
050import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
051import org.hl7.fhir.utilities.xhtml.XhtmlNode;
052
053/**
054 * in here:
055 *   * generateTranslations
056 *   * importFromTranslations
057 *   * stripTranslations
058 *   * switchLanguage
059 *   
060 *  in the validator
061 *  
062 * @author grahamegrieve
063 *  generateTranslations = -langTransform export -src {src} -tgt {tgt} -dest {dest}
064 *  importFromTranslations =  -langTransform import -src {src} -tgt {tgt} -dest {dest}
065 */
066@MarkedToMoveToAdjunctPackage
067@Slf4j
068public class LanguageUtils {
069
070  public static final List<String> TRANSLATION_SUPPLEMENT_RESOURCE_TYPES = Arrays.asList("CodeSystem", "StructureDefinition", "Questionnaire");
071
072  public static class TranslationUnitCollection {
073    List<TranslationUnit> list= new ArrayList<>();
074    Map<String, TranslationUnit> map = new HashMap<>();
075    public void add(TranslationUnit tu) {
076      String key = tu.getId()+"||"+tu.getSrcText();
077      if (!map.containsKey(key)) {
078        map.put(key, tu);
079        list.add(tu);
080      }
081      
082    }
083  }
084  IWorkerContext context;
085  private List<String> crlist;
086  
087  
088  public LanguageUtils(IWorkerContext context) {
089    super();
090    this.context = context;
091  }
092
093  public void generateTranslations(Element resource, LanguageProducerLanguageSession session) {
094    translate(null, resource, session, resource.fhirType());
095  }
096  
097  
098  private void translate(Element parent, Element element, LanguageProducerLanguageSession langSession, String path) {
099    String npath = pathForElement(path, element);
100    if (element.isPrimitive() && isTranslatable(element)) {
101      String base = element.primitiveValue();
102      if (base != null) {
103        String translation = getSpecialTranslation(path, parent, element, langSession.getTargetLang());
104        if (translation == null) {
105          translation = element.getTranslation(langSession.getTargetLang());
106        }
107        langSession.entry(new TextUnit(npath, contextForElement(element), base, translation));
108      }
109    }
110    for (Element c: element.getChildren()) {
111      if (!c.getName().equals("designation")) {
112        translate(element, c, langSession, npath);
113      }
114    }
115  }
116
117  private String contextForElement(Element element) {
118    throw new Error("Not done yet");
119  }
120
121  private String getSpecialTranslation(String path, Element parent, Element element, String targetLang) {
122    if (parent == null) {
123      return null;
124    }
125    String npath = parent.getBasePath();
126    if (Utilities.existsInList(npath, "CodeSystem.concept", "CodeSystem.concept.concept") && "CodeSystem.concept.display".equals(element.getBasePath())) {
127      return getDesignationTranslation(parent, targetLang);
128    }
129    if (Utilities.existsInList(npath, "ValueSet.compose.include.concept") && "ValueSet.compose.include.concept.display".equals(element.getBasePath())) {
130      return getDesignationTranslation(parent, targetLang);
131    }
132    if (Utilities.existsInList(npath, "ValueSet.expansion.contains", "ValueSet.expansion.contains.contains") && "ValueSet.expansion.contains.display".equals(element.getBasePath())) {
133      return getDesignationTranslation(parent, targetLang);
134    }
135    return null;
136  }
137
138  private String getDesignationTranslation(Element parent, String targetLang) {
139    for (Element e : parent.getChildren("designation")) {
140      String lang = e.getNamedChildValue("language");
141      if (langsMatch(targetLang, lang)) {
142        return e.getNamedChildValue("value");
143      }
144    }
145    return null;
146  }
147
148  private boolean isTranslatable(Element element) {    
149    return element.getProperty().isTranslatable();
150  }
151
152  private String pathForElement(String path, Element element) {
153    if (element.getSpecial() != null) {
154      String bp = element.getBasePath();
155      return pathForElement(bp, element.getProperty().getStructure().getType());
156    } else {
157      return (path == null ? element.getName() : path+"."+element.getName());
158    }
159  }
160  
161  private String pathForElement(String path, String type) {
162    // special case support for metadata elements prior to R5:
163    if (crlist == null) {
164      crlist = new ContextUtilities(context).getCanonicalResourceNames();
165    }
166    if (crlist.contains(type)) {
167      String fp = path.replace(type+".", "CanonicalResource.");
168      if (Utilities.existsInList(fp,
169         "CanonicalResource.url", "CanonicalResource.identifier", "CanonicalResource.version", "CanonicalResource.name", 
170         "CanonicalResource.title", "CanonicalResource.status", "CanonicalResource.experimental", "CanonicalResource.date",
171         "CanonicalResource.publisher", "CanonicalResource.contact", "CanonicalResource.description", "CanonicalResource.useContext", 
172         "CanonicalResource.jurisdiction"))  {
173        return fp;
174      }
175    }
176    return path; 
177  }
178  
179  
180  public int importFromTranslations(Element resource, List<TranslationUnit> translations) {
181    return importFromTranslations(resource.fhirType(), null, resource, translations, new HashSet<>());
182  }
183  
184  public int importFromTranslations(Element resource, List<TranslationUnit> translations, List<ValidationMessage> messages) {
185    Set<TranslationUnit> usedUnits = new HashSet<>();
186    int r = 0;
187    if (resource.fhirType().equals("StructureDefinition")) {
188      r = importFromTranslationsForSD(null, resource, translations, usedUnits);
189    } else {
190     r = importFromTranslations(null, null, resource, translations, usedUnits);
191    }
192    for (TranslationUnit t : translations) {
193      if (!usedUnits.contains(t)) {
194        if (messages != null) {
195          messages.add(new ValidationMessage(Source.Publisher, IssueType.INFORMATIONAL, t.getId(), "Unused '"+t.getLanguage()+"' translation '"+t.getSrcText()+"' -> '"+t.getTgtText()+"'", IssueSeverity.INFORMATION));
196        }
197      }
198    }
199    return r;
200  }
201  
202  public int importFromTranslations(Resource resource, List<TranslationUnit> translations, List<ValidationMessage> messages) {
203    Set<TranslationUnit> usedUnits = new HashSet<>();
204    int r = 0;
205    if (resource.fhirType().equals("StructureDefinition")) {
206      // todo... r = importFromTranslationsForSD(null, resource, translations, usedUnits);
207    } else {
208     r = importResourceFromTranslations(null, resource, translations, usedUnits, resource.fhirType());
209    }
210    for (TranslationUnit t : translations) {
211      if (!usedUnits.contains(t)) {
212        if (messages != null) {
213          messages.add(new ValidationMessage(Source.Publisher, IssueType.INFORMATIONAL, t.getId(), "Unused '"+t.getLanguage()+"' translation '"+t.getSrcText()+"' -> '"+t.getTgtText()+"'", IssueSeverity.INFORMATION));
214        }
215      }
216    }
217    return r;
218  }
219  
220
221  /*
222   * */
223  private int importFromTranslationsForSD(Object object, Element resource, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits) {
224    int r = 0;
225    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.name", "name");
226    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.title", "title");
227    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.publisher", "publisher");
228    for (Element cd : resource.getChildrenByName("contact")) {
229      r = r + checkForTranslations(translations, usedUnits, cd, "StructureDefinition.contact.name", "name");
230    }
231    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.purpose", "purpose");
232    r = r + checkForTranslations(translations, usedUnits, resource, "StructureDefinition.copyright", "copyright");
233    Element diff = resource.getNamedChild("differential");
234    if (diff != null) {
235      for (Element ed : diff.getChildrenByName("element")) {
236        String id = ed.getNamedChildValue("id");
237        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/label", "label");
238        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/short", "short");
239        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/definition", "definition");
240        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/comment", "comment");
241        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/requirements", "requirements");
242        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/meaningWhenMissing", "meaningWhenMissing");
243        r = r + checkForTranslations(translations, usedUnits, ed, "StructureDefinition.element."+id+"/orderMeaning", "orderMeaning");
244        //      for (ElementDefinitionConstraintComponent con : ed.getConstraint()) {
245        //        addToList(list, lang, con, ed.getId()+"/constraint", "human", con.getHumanElement());
246        //      }
247        //      if (ed.hasBinding()) {
248        //        addToList(list, lang, ed.getBinding(), ed.getId()+"/b/desc", "description", ed.getBinding().getDescriptionElement());
249        //        for (ElementDefinitionBindingAdditionalComponent ab : ed.getBinding().getAdditional()) {
250        //          addToList(list, lang, ab, ed.getId()+"/ab/doco", "documentation", ab.getDocumentationElement());
251        //          addToList(list, lang, ab, ed.getId()+"/ab/short", "shortDoco", ab.getShortDocoElement());
252        //        }
253        //      }
254      }
255    }
256    return r;
257  }
258
259  private int checkForTranslations(List<TranslationUnit> translations, Set<TranslationUnit> usedUnits, Element context, String tname, String pname) {
260    int r = 0;
261    Element child = context.getNamedChild(pname);
262    if (child != null) {
263      String v = child.primitiveValue();
264      if (v != null) {
265        for (TranslationUnit tu : translations) {
266          if (tname.equals(tu.getId()) && v.equals(tu.getSrcText())) {
267            usedUnits.add(tu);
268            child.setTranslation(tu.getLanguage(), tu.getTgtText());
269            r++;
270          }
271        }
272      }
273    }
274    return r;
275  }
276
277  private int importResourceFromTranslations(Base parent, Base element, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits, String path) {
278    int t = 0;
279    if (element.isPrimitive() && isTranslatable(element, path) && element instanceof org.hl7.fhir.r5.model.Element) {
280      org.hl7.fhir.r5.model.Element e = (org.hl7.fhir.r5.model.Element) element;
281      String base = element.primitiveValue();
282      if (base != null) {
283        String epath = pathForElement(path, element.fhirType());
284        Set<TranslationUnit> tlist = findTranslations(epath, base, translations);
285        for (TranslationUnit translation : tlist) {
286          t++;
287          if (!handleAsSpecial(parent, element, translation)) {
288            usedUnits.add(translation);
289            if (translation.getTgtText() != null) {
290              ToolingExtensions.setLanguageTranslation(e, translation.getLanguage(), translation.getTgtText());
291            } else {
292              log.warn("?");
293            }
294          }
295        }
296      }
297    }
298    for (Property c : element.children()) {
299      for (Base v : c.getValues()) {
300        if (!c.getName().equals("designation") && !isTranslation(v)) {
301          t = t + importResourceFromTranslations(element, v, translations, usedUnits, genPath(c, v, path, c.getName()));
302        }
303      }
304    }
305    return t;
306  }
307
308  private String genPath(Property c, Base v, String path, String name) {
309    // special cases: recursion
310    if ("ImplementationGuide.definition.page".equals(path) && "page".equals(name)) {
311      return path;
312    }
313    if ("ValueSet.expansion.contains".equals(path) && "contains".equals(name)) {
314      return path;
315    }
316    if ("ValueSet.expansion.contains".equals(path) && "contains".equals(name)) {
317      return path;
318    }
319    if (v.isResource() && !"contained".equals(name)) {
320      return v.fhirType();
321    } else {
322      return path+"."+name;
323    }
324  }
325
326  private boolean isTranslation(Base element) {
327    return "Extension".equals(element.fhirType()) && element.getChildByName("url").hasValues()  && ToolingExtensions.EXT_TRANSLATION.equals(element.getChildByName("url").getValues().get(0).primitiveValue());
328  }
329
330  private boolean handleAsSpecial(Base parent, Base element, TranslationUnit translation) {
331    return false;
332  }
333
334  private boolean isTranslatable(Base element, String path) {
335    return (Utilities.existsInList(element.fhirType(), "string", "markdown") 
336        || isTranslatable(path)) && !isExemptFromTranslations(path);
337  }
338
339  private int importFromTranslations(String path, Element parent, Element element, List<TranslationUnit> translations, Set<TranslationUnit> usedUnits) {
340    String npath = pathForElement(path, element);
341    int t = 0;
342    if (element.isPrimitive() && isTranslatable(element) && !isExemptFromTranslations(npath)) {
343      String base = element.primitiveValue();
344      if (base != null) {
345        Set<TranslationUnit> tlist = findTranslations(npath, base, translations);
346        for (TranslationUnit translation : tlist) {
347          t++;
348          if (!handleAsSpecial(parent, element, translation)) {
349            element.setTranslation(translation.getLanguage(), translation.getTgtText());
350            usedUnits.add(translation);
351          }
352        }
353      }
354    }
355    // Create a copy of the children collection before iterating
356    List<Element> childrenCopy = List.copyOf(element.getChildren());
357    for (Element c : childrenCopy) {
358      if (!c.getName().equals("designation")) {
359        t = t + importFromTranslations(npath, element, c, translations, usedUnits);
360      }
361    }
362    return t;
363  }
364
365  private boolean handleAsSpecial(Element parent, Element element, TranslationUnit translation) {
366    if (parent == null) {
367      return false;
368    }
369    if (Utilities.existsInList(parent.getBasePath(), "CodeSystem.concept", "CodeSystem.concept.concept") && "CodeSystem.concept.display".equals(element.getBasePath())) {
370      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
371    }
372    if (Utilities.existsInList(parent.getBasePath(), "ValueSet.compose.include.concept") && "ValueSet.compose.include.concept.display".equals(element.getBasePath())) {
373      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
374    }
375    if (Utilities.existsInList(parent.getBasePath(), "ValueSet.expansion.contains", "ValueSet.expansion.contains.contains") && "ValueSet.expansion.contains.display".equals(element.getBasePath())) {
376      return setDesignationTranslation(parent, translation.getLanguage(), translation.getTgtText());
377    }
378    return false;
379  }
380
381  private boolean setDesignationTranslation(Element parent, String targetLang, String translation) {
382    for (Element e : parent.getChildren("designation")) {
383      String lang = e.getNamedChildValue("language");
384      if (langsMatch(targetLang, lang)) {
385        Element value = e.getNamedChild("value");
386        if (value != null) {
387          value.setValue(translation);
388        } else {
389          e.addElement("value").setValue(translation);
390        }
391        return true;
392      }
393    }
394    Element d = parent.addElement("designation");
395    d.addElement("language").setValue(targetLang);
396    d.addElement("value").setValue(translation);
397    return true;
398  }
399
400  private Set<TranslationUnit> findTranslations(String path, String src, List<TranslationUnit> translations) {
401    Set<TranslationUnit> res = new HashSet<>();
402    for (TranslationUnit translation : translations) {
403      if (path.equals(translation.getId()) && src.equals(translation.getSrcText())) {
404        res.add(translation);
405      }
406    }
407    return res;
408  }
409
410  public static boolean langsMatchExact(AcceptLanguageHeader langs, String srcLang) {
411    if (langs == null) {
412      return false;
413    }
414    for (LanguagePreference lang : langs.getLangs()) {
415      if (lang.getValue() > 0) {
416        if ("*".equals(lang.getLang())) {
417          return true;
418        } else {
419          return langsMatch(lang.getLang(), srcLang);
420        }
421      }
422    }
423    return false;
424  }
425
426  public static boolean langsMatch(AcceptLanguageHeader langs, String srcLang) {
427    if (langs == null) {
428      return false;
429    }
430    for (LanguagePreference lang : langs.getLangs()) {
431      if (lang.getValue() > 0) {
432        if ("*".equals(lang.getLang())) {
433          return true;
434        } else {
435          boolean ok = langsMatch(lang.getLang(), srcLang);
436          if (ok) {
437            return true;
438          }
439        }
440      }
441    }
442    return false;
443  }
444
445  public static boolean langsMatchExact(String dstLang, String srcLang) {
446    return dstLang == null ? false : dstLang.equals(srcLang);
447  }
448
449  public static boolean langsMatch(String dstLang, String srcLang) {
450    return dstLang == null || srcLang == null ? Utilities.existsInList(srcLang, "en", "en-US") : dstLang.startsWith(srcLang) || "*".equals(srcLang);
451  }
452
453  public void fillSupplement(CodeSystem csSrc, CodeSystem csDst, List<TranslationUnit> list) {
454    csDst.setUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT, csSrc);
455    csDst.setUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS, list);
456    for (TranslationUnit tu : list) {
457      String code = tu.getId();
458      String subCode = null;
459      if (code.contains("@")) {
460        subCode = code.substring(code.indexOf("@")+1);
461        code = code.substring(0, code.indexOf("@"));
462      }
463      ConceptDefinitionComponent cdSrc = CodeSystemUtilities.getCode(csSrc, tu.getId());
464      if (cdSrc == null) {
465        addOrphanTranslation(csSrc, tu);
466      } else {
467        ConceptDefinitionComponent cdDst = CodeSystemUtilities.getCode(csDst, cdSrc.getCode());
468        if (cdDst == null) {
469          cdDst = csDst.addConcept().setCode(cdSrc.getCode());
470        }
471        String tt = tu.getTgtText();
472        if (tt.startsWith("!!")) {
473          tt = tt.substring(3);
474        }
475        if (subCode == null) {
476          cdDst.setDisplay(tt);
477        } else if ("definition".equals(subCode)) {
478          cdDst.setDefinition(tt);
479        } else {
480          boolean found = false;
481          for (ConceptDefinitionDesignationComponent d : cdSrc.getDesignation()) {
482            if (d.hasUse() && subCode.equals(d.getUse().getCode())) {
483              found = true;
484              cdDst.addDesignation().setUse(d.getUse()).setLanguage(tu.getLanguage()).setValue(tt); //.setUserData(SUPPLEMENT_SOURCE, tu);       
485              break;
486            }
487          }
488          if (!found) {
489            for (Extension e : cdSrc.getExtension()) {
490              if (subCode.equals(tail(e.getUrl()))) {
491                found = true;
492                cdDst.addExtension().setUrl(e.getUrl()).setValue(
493                    e.getValue().fhirType().equals("markdown") ? new MarkdownType(tt) : new StringType(tt)); //.setUserData(SUPPLEMENT_SOURCE, tu);          
494                break;
495              }
496            }
497          }
498          if (!found) {
499            addOrphanTranslation(csSrc, tu);            
500          }
501        }
502      }      
503    }    
504  }
505
506  private String tail(String url) {
507    return url.contains("/") ? url.substring(url.lastIndexOf("/")+1) : url;
508  }
509
510  private void addOrphanTranslation(CodeSystem cs, TranslationUnit tu) {
511    List<TranslationUnit> list = (List<TranslationUnit>) cs.getUserData(UserDataNames.LANGUTILS_ORPHAN);
512    if (list == null) {
513      list = new ArrayList<>();
514      cs.setUserData(UserDataNames.LANGUTILS_ORPHAN, list);
515    }
516    list.add(tu);
517  }
518
519  public String nameForLang(String lang) {
520    // todo: replace with structures from loading languages properly
521    switch (lang) {
522    case "en" : return "English";
523    case "de" : return "German";
524    case "es" : return "Spanish";
525    case "nl" : return "Dutch";
526    }
527    return Utilities.capitalize(lang);
528  }
529
530  public String titleForLang(String lang) {
531    // todo: replace with structures from loading languages properly
532    switch (lang) {
533    case "en" : return "English";
534    case "de" : return "German";
535    case "es" : return "Spanish";
536    case "nl" : return "Dutch";
537    }
538    return Utilities.capitalize(lang);
539  }
540
541  public boolean handlesAsResource(Resource resource) {
542    return (resource instanceof CodeSystem && resource.hasUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT)) || (resource instanceof StructureDefinition);
543  }
544
545  public boolean handlesAsElement(Element element) {
546    return true; // for now...
547  }
548
549  public List<TranslationUnit> generateTranslations(Resource res, String lang) {
550    List<TranslationUnit> list = new ArrayList<>();
551    if (res instanceof StructureDefinition) {
552      StructureDefinition sd = (StructureDefinition) res;
553      generateTranslations(list, sd, lang);
554      if (res.hasUserData(UserDataNames.LANGUTILS_ORPHAN)) {
555        List<TranslationUnit> orphans = (List<TranslationUnit>) res.getUserData(UserDataNames.LANGUTILS_ORPHAN);
556        for (TranslationUnit t : orphans) {
557          if (!hasInList(list, t.getId(), t.getSrcText())) {
558            list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext(), t.getSrcText(), t.getTgtText()));
559          }
560        }
561      }
562    } else {
563      CodeSystem cs = (CodeSystem) res.getUserData(UserDataNames.LANGUTILS_SOURCE_SUPPLEMENT);
564      List<TranslationUnit> inputs = res.hasUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS) ? (List<TranslationUnit>) res.getUserData(UserDataNames.LANGUTILS_SOURCE_TRANSLATIONS) : new ArrayList<>();
565      for (ConceptDefinitionComponent cd : cs.getConcept()) {
566        generateTranslations(list, cd, lang, inputs);
567      }
568      if (cs.hasUserData(UserDataNames.LANGUTILS_ORPHAN)) {
569        List<TranslationUnit> orphans = (List<TranslationUnit>) cs.getUserData(UserDataNames.LANGUTILS_ORPHAN);
570        for (TranslationUnit t : orphans) {
571          if (!hasInList(list, t.getId(), t.getSrcText())) {
572            list.add(new TranslationUnit(lang, "!!"+t.getId(), t.getContext(), t.getSrcText(), t.getTgtText()));
573          }
574        }
575      }
576    }
577    return list;
578  }
579
580  private void generateTranslations(List<TranslationUnit> list, StructureDefinition sd, String lang) {
581    addToList(list, lang, sd, "StructureDefinition.name", "name", sd.getNameElement());
582    addToList(list, lang, sd, "StructureDefinition.title", "title", sd.getTitleElement());
583    addToList(list, lang, sd, "StructureDefinition.publisher", "publisher", sd.getPublisherElement());
584    for (ContactDetail cd : sd.getContact()) {
585      addToList(list, lang, cd, "StructureDefinition.contact.name", "name", cd.getNameElement());
586    }
587    addToList(list, lang, sd, "StructureDefinition.purpose", "purpose", sd.getPurposeElement());
588    addToList(list, lang, sd, "StructureDefinition.copyright", "copyright", sd.getCopyrightElement());
589    for (ElementDefinition ed : sd.getDifferential().getElement()) {
590      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/label", "label", ed.getLabelElement());
591      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/short", "short", ed.getShortElement());
592      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/definition", "definition", ed.getDefinitionElement());
593      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/comment", "comment", ed.getCommentElement());
594      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/requirements", "requirements", ed.getRequirementsElement());
595      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/meaningWhenMissing", "meaningWhenMissing", ed.getMeaningWhenMissingElement());
596      addToList(list, lang, ed, "StructureDefinition.element."+ed.getId()+"/orderMeaning", "orderMeaning", ed.getOrderMeaningElement());
597      for (ElementDefinitionConstraintComponent con : ed.getConstraint()) {
598        addToList(list, lang, con, "StructureDefinition.element."+ed.getId()+"/constraint", "human", con.getHumanElement());
599      }
600      if (ed.hasBinding()) {
601        addToList(list, lang, ed.getBinding(), "StructureDefinition.element."+ed.getId()+"/b/desc", "description", ed.getBinding().getDescriptionElement());
602        for (ElementDefinitionBindingAdditionalComponent ab : ed.getBinding().getAdditional()) {
603          addToList(list, lang, ab, "StructureDefinition.element."+ed.getId()+"/ab/doco", "documentation", ab.getDocumentationElement());
604          addToList(list, lang, ab, "StructureDefinition.element."+ed.getId()+"/ab/short", "shortDoco", ab.getShortDocoElement());
605        }
606      }
607    }
608  }
609
610  private void addToList(List<TranslationUnit> list, String lang, Base ctxt, String name, String propName, DataType value) {
611    if (value != null && value.hasPrimitiveValue()) {
612      if (!hasInList(list, name, value.primitiveValue())) {
613        list.add(new TranslationUnit(lang, name, ctxt.getNamedProperty(propName).getDefinition(), value.primitiveValue(), value.getTranslation(lang)));
614      }
615    }
616    
617  }
618
619  private void generateTranslations(List<TranslationUnit> list, ConceptDefinitionComponent cd, String lang, List<TranslationUnit> inputs) {
620    // we generate translation units for the display, the definition, and any designations and extensions that we find
621    // the id of the designation is the use.code (there will be a use) and for the extension, the tail of the extension URL 
622    // todo: do we need to worry about name clashes? why would we, and more importantly, how would we solve that?
623
624    addTranslationUnit(list, cd.getCode(), cd.getDisplay(), lang, inputs);
625    if (cd.hasDefinition()) {
626      addTranslationUnit(list, cd.getCode()+"@definition", cd.getDefinition(), lang, inputs);        
627    }
628    for (ConceptDefinitionDesignationComponent d : cd.getDesignation()) {
629      addTranslationUnit(list, cd.getCode()+"@"+d.getUse().getCode(), d.getValue(), lang, inputs);              
630    }
631    for (Extension e : cd.getExtension()) {
632      addTranslationUnit(list, cd.getCode()+"@"+tail(e.getUrl()), e.getValue().primitiveValue(), lang, inputs);                    
633    }
634  }
635
636  private void addTranslationUnit(List<TranslationUnit> list, String id, String srcText, String lang, List<TranslationUnit> inputs) {
637    TranslationUnit existing = null;
638    for (TranslationUnit t : inputs) {
639      if (id.equals(t.getId())) {
640        existing = t;
641        break;
642      }
643    }
644    if (!hasInList(list, id, srcText)) {
645      // not sure what to do with context?
646      if (existing == null) {
647        list.add(new TranslationUnit(lang, id, null, srcText, null));
648      } else if (srcText.equals(existing.getSrcText())) {
649        list.add(new TranslationUnit(lang, id, null, srcText, existing.getTgtText()));
650      } else {
651        list.add(new TranslationUnit(lang, id, null, srcText, "!!"+existing.getTgtText()).setOriginal(existing.getSrcText()));
652      }
653    }
654  }
655  
656  private String getDefinition(ConceptDefinitionComponent cd) {
657    ConceptPropertyComponent v = CodeSystemUtilities.getProperty(cd, "translation-context");
658    if (v != null && v.hasValue()) {
659      return v.getValue().primitiveValue();
660    } else {
661      return cd.getDefinition();
662    }
663  }
664
665  public List<TranslationUnit> generateTranslations(Element e, String lang) {
666    TranslationUnitCollection list = new TranslationUnitCollection();
667    generateTranslations(e, lang, list, e.fhirType());
668    return list.list;
669  }
670
671  private void generateTranslations(Element e, String lang, TranslationUnitCollection list, String path) {
672    String npath = pathForElement(path, e);
673    if ((e.getProperty().isTranslatable() || isTranslatable(e.getProperty().getDefinition().getBase().getPath())) 
674        && !isExemptFromTranslations(e.getProperty().getDefinition().getBase().getPath())) {
675      String id = e.getProperty().getDefinition().getBase().getPath(); // .getProperty().getDefinition().getPath();
676      String context = e.getProperty().getDefinition().getDefinition();
677      String src = e.primitiveValue();
678      String tgt = getTranslation(e, lang);
679      if (!hasInList(list.list, id, src)) {
680        list.add(new TranslationUnit(lang, id, context, src, tgt));
681      }
682    }
683    if (e.hasChildren()) {
684      for (Element c : e.getChildren()) {
685        generateTranslations(c, lang, list, npath);
686      }
687    }
688  }
689
690  private boolean hasInList(List<TranslationUnit> list, String id, String src) {
691    for (TranslationUnit t : list) {
692      if (t.getId() != null && t.getId().equals(id) && t.getSrcText()!= null && t.getSrcText().equals(src)) {
693        return true;
694      }
695    }
696    return false;
697  }
698
699  /**
700   * override specifications
701   * 
702   * @param path
703   * @return
704   */
705  private boolean isTranslatable(String path) {
706    return Utilities.existsInList(path, "TestCases.publisher",
707        "TestCases.contact.telecom.value",
708        "TestCases.definition",
709        "TestCases.parameter.name",
710        "TestCases.parameter.description",
711        "TestCases.scope.description ",
712        "TestCases.dependency.description",
713        "TestCases.mode.description",
714        "TestCases.suite",
715        "TestCases.suite.name",
716        "TestCases.suite.description",
717        "TestCases.suite.test",
718        "TestCases.suite.test.name",
719        "TestCases.suite.test.description",
720        "TestCases.suite.test.assert.human",
721        "ActorDefinition.title",
722        "ActorDefinition.description",
723        "ActorDefinition.purpose",
724        "ActorDefinition.copyright",
725        "ActorDefinition.copyrightLabel",
726        "ActorDefinition.documentation",
727        "Requirements.title",
728        "Requirements.publisher",
729        "Requirements.description",
730        "Requirements.purpose",
731        "Requirements.copyright",
732        "Requirements.copyrightLabel",
733        "Requirements.statement.label",
734        "Requirements.statement.requirement");    
735  }
736
737  private boolean isExemptFromTranslations(String path) {
738    if (path.endsWith(".reference")) {
739      return true;
740    }
741    return Utilities.existsInList(path, 
742        "ImplementationGuide.definition.parameter.value", "ImplementationGuide.dependsOn.version", "ImplementationGuide.dependsOn.id",
743        "CanonicalResource.name", 
744        "CapabilityStatement.rest.resource.searchRevInclude", "CapabilityStatement.rest.resource.searchInclude", "CapabilityStatement.rest.resource.searchParam.name",
745        "SearchParameter.expression", "SearchParameter.xpath",
746        "ExampleScenario.actor.actorId", "ExampleScenario.instance.resourceId", "ExampleScenario.instance.containedInstance.resourceId", "ExampleScenario.instance.version.versionId", 
747          "ExampleScenario.process.step.operation.number", "ExampleScenario.process.step.operation.initiator", "ExampleScenario.process.step.operation.receiver",
748        "ExampleScenario.process.step.operation.number", "ExampleScenario.process.step.operation.initiator", "ExampleScenario.process.step.operation.receiver",
749        "OperationDefinition.parameter.max", "OperationDefinition.overload.parameterName",
750        "StructureMap.group.rule.source.type", "StructureMap.group.rule.source.element", "StructureMap.group.rule.target.element");
751  }
752
753  private String getTranslation(Element e, String lang) {
754    if (!e.hasChildren()) {
755      return null;
756    }
757    for (Element ext : e.getChildren()) {
758      if ("Extension".equals(ext.fhirType()) && "http://hl7.org/fhir/StructureDefinition/translation".equals(ext.getNamedChildValue("url"))) {
759        String l = null;
760        String v = null;
761        for (Element subExt : ext.getChildren()) {
762          if ("Extension".equals(subExt.fhirType()) && "lang".equals(subExt.getNamedChildValue("url"))) {
763            l = subExt.getNamedChildValue("value");
764          }
765          if ("Extension".equals(subExt.fhirType()) && "content".equals(subExt.getNamedChildValue("url"))) {
766            v = subExt.getNamedChildValue("value");
767          }
768        }
769        if (lang.equals(l)) {
770          return v;
771        }
772      }
773    }
774    return null;
775  }
776 
777  public boolean switchLanguage(Base r, String lang, boolean markLanguage, boolean contained) {
778    boolean changed = false;
779    if (r.isPrimitive()) {
780
781      PrimitiveType<?> dt = (PrimitiveType<?>) r;
782      String cnt = ToolingExtensions.getLanguageTranslation(dt, lang); 
783      dt.removeExtension(ToolingExtensions.EXT_TRANSLATION);
784      if (cnt != null) {
785        dt.setValueAsString(cnt);
786        changed = true;
787      }
788    }
789
790    if (r.fhirType().equals("Narrative")) {
791      Base div = r.getChildValueByName("div");
792      Base status = r.getChildValueByName("status");
793
794      XhtmlNode xhtml = div.getXhtml();
795      xhtml = adjustToLang(xhtml, lang, status == null ? null : status.primitiveValue());
796      if (xhtml == null) {
797        r.removeChild("div", div);
798      } else {
799        div.setXhtml(xhtml);
800      }
801    }
802    for (Property p : r.children()) {
803      for (Base c : p.getValues()) {
804        changed = switchLanguage(c, lang, markLanguage, p.getName().equals("contained")) || changed;
805      }
806    }
807    if (markLanguage && r.isResource() && !contained) {
808      Resource res = (Resource) r;
809      res.setLanguage(lang);
810      changed = true;
811    }
812    return changed;
813  }
814
815  private XhtmlNode adjustToLang(XhtmlNode xhtml, String lang, String status) {
816    if (xhtml == null) {
817      return null;
818    }
819    //see if there's a language specific section
820    for (XhtmlNode div : xhtml.getChildNodes()) {
821      if ("div".equals(div.getName())) {
822        String l = div.hasAttribute("lang") ? div.getAttribute("lang") : div.getAttribute("xml:lang");
823        if (lang.equals(l)) {
824          return div;
825        }       
826      }
827    }
828    // if the root language is correct
829    // this isn't first because a narrative marked for one language might have other language subsections
830    String l = xhtml.hasAttribute("lang") ? xhtml.getAttribute("lang") : xhtml.getAttribute("xml:lang");
831    if (lang.equals(l)) {
832      return xhtml;
833    }
834    // if the root language is null and the status is not additional
835    // it can be regenerated
836    if (l == null && Utilities.existsInList(status, "generated", "extensions")) {
837      return null;
838    }    
839    // well, really, not much we can do...
840    return xhtml;
841  }
842
843  public boolean switchLanguage(Element e, String lang, boolean markLanguage) throws IOException {
844    boolean changed = false;
845    if (e.getProperty().isTranslatable()) {
846      String cnt = getTranslation(e, lang);
847      e.removeExtension(ToolingExtensions.EXT_TRANSLATION);
848      if (cnt != null) {
849        e.setValue(cnt);
850        changed = true;
851      }
852    }
853    if (e.fhirType().equals("Narrative") && e.hasChild("div")) {
854      XhtmlNode xhtml = e.getNamedChild("div").getXhtml();
855      xhtml = adjustToLang(xhtml, lang, e.getNamedChildValue("status"));
856      if (xhtml == null) {
857        e.removeChild("div");
858      } else {
859        e.getNamedChild("div").setXhtml(xhtml);
860      }
861    }
862    if (e.hasChildren()) {
863      for (Element c : e.getChildren()) {
864        changed = switchLanguage(c, lang, markLanguage) || changed;
865      }
866    }
867    if (markLanguage && e.isResource() && e.getSpecial() != SpecialElement.CONTAINED) {
868      e.setChildValue("language", lang);
869      changed = true;
870    }
871    return changed;
872  }
873
874  public boolean hasTranslation(org.hl7.fhir.r5.model.Element e, String lang) {
875    return getTranslation(e, lang) != null;
876  }
877
878  public String getTranslation(org.hl7.fhir.r5.model.Element e, String lang) {
879    for (Extension ext : e.getExtensionsByUrl(ToolingExtensions.EXT_TRANSLATION)) {
880      String l = ext.getExtensionString("lang");
881      String v =  ext.getExtensionString("content");
882      if (langsMatch(l, lang) && v != null) {
883        return v;
884      }
885    }
886    return null;
887  }
888
889  public String getTranslationOrBase(PrimitiveType<?> e, String lang) {
890    for (Extension ext : e.getExtensionsByUrl(ToolingExtensions.EXT_TRANSLATION)) {
891      String l = ext.getExtensionString("lang");
892      String v =  ext.getExtensionString("content");
893      if (langsMatch(l, lang) && v != null) {
894        return v;
895      }
896    }
897    return e.primitiveValue();
898  }
899
900  public Element copyToLanguage(Element element, String lang, boolean markLanguage) throws IOException {
901    Element result = (Element) element.copy();
902    switchLanguage(result, lang, markLanguage);
903    return result;
904  }
905
906  public Resource copyToLanguage(Resource res, String lang, boolean markLanguage) {
907    if (res == null) {
908      return null;
909    }
910    Resource r = res.copy();
911    switchLanguage(r, lang, markLanguage, false);    
912    return r;
913  }
914}