001package org.hl7.fhir.r5.test.utils;
002
003import org.apache.commons.codec.binary.Base64;
004import org.apache.commons.lang3.StringUtils;
005import org.hl7.fhir.exceptions.FHIRException;
006import org.hl7.fhir.utilities.*;
007import org.hl7.fhir.utilities.json.JsonUtilities;
008import org.hl7.fhir.utilities.json.model.JsonArray;
009import org.hl7.fhir.utilities.json.model.JsonElement;
010import org.hl7.fhir.utilities.json.model.JsonNull;
011import org.hl7.fhir.utilities.json.model.JsonObject;
012import org.hl7.fhir.utilities.json.model.JsonPrimitive;
013import org.hl7.fhir.utilities.json.model.JsonProperty;
014import org.hl7.fhir.utilities.json.parser.JsonParser;
015import org.hl7.fhir.utilities.settings.FhirSettings;
016import org.w3c.dom.Document;
017import org.w3c.dom.Element;
018import org.w3c.dom.NamedNodeMap;
019import org.w3c.dom.Node;
020
021import org.hl7.fhir.utilities.tests.BaseTestingUtilities;
022
023import javax.xml.parsers.DocumentBuilder;
024import javax.xml.parsers.DocumentBuilderFactory;
025import java.io.*;
026import java.util.ArrayList;
027import java.util.List;
028import java.util.Map;
029
030public class CompareUtilities extends BaseTestingUtilities {
031
032  private static final boolean SHOW_DIFF = false;
033  private JsonObject externals;
034  
035  public String createNotEqualMessage(final String message, final String expected, final String actual) {
036    return new StringBuilder()
037      .append(message).append('\n')
038      .append("Expected :").append(presentExpected(expected)).append('\n')
039      .append("Actual  :").append("\""+actual+"\"").toString();
040  }
041
042  private String presentExpected(String expected) {
043    if (expected.startsWith("$") && expected.endsWith("$")) {
044      if (expected.startsWith("$choice:")) {
045        return "Contains one of "+readChoices(expected.substring(8, expected.length()-1)).toString();
046      } else if (expected.startsWith("$fragments:")) {
047        List<String> fragments = readChoices(expected.substring(11, expected.length()-1));
048        return "Contains all of "+fragments.toString();
049      } else if (expected.startsWith("$external:")) {
050        String[] cmd = expected.substring(1, expected.length() - 1).split(":");
051        if (externals != null) {
052          String s = externals.asString(cmd[1]);
053          return "\""+s+"\" (Ext)";
054        } else {
055          List<String> fragments = readChoices(cmd[2]);
056          return "Contains all of "+fragments.toString()+" (because no external string provided for "+cmd[1]+")";
057        }
058      } else {
059        switch (expected) {
060        case "$$" : return "$$";
061        case "$instant$": return "\"An Instant\"";
062        case "$uuid$": return "\"A Uuid\"";
063        default: return "Unhandled template: "+expected;
064        }
065      }
066    } else {
067      return "\""+expected+"\"";
068    }
069  }
070
071  public static String checkXMLIsSame(InputStream expected, InputStream actual) throws Exception {
072    CompareUtilities self = new CompareUtilities();
073    String result = self.compareXml(expected, actual);
074    return result;
075  }
076
077  public static String checkXMLIsSame(String expected, String actual) throws Exception {
078    CompareUtilities self = new CompareUtilities();
079    String result = self.compareXml(expected, actual);
080    if (result != null && SHOW_DIFF) {
081      String diff = getDiffTool();
082      if (diff != null && new File(diff).exists() || Utilities.isToken(diff)) {
083        Runtime.getRuntime().exec(new String[]{diff, expected, actual});
084      }
085    }
086    return result;
087  }
088
089 private static String getDiffTool() throws IOException {
090    if (FhirSettings.hasDiffToolPath()) {
091      return FhirSettings.getDiffToolPath();
092    } else if (System.getenv("ProgramFiles") != null) { 
093      return Utilities.path(System.getenv("ProgramFiles"), "WinMerge", "WinMergeU.exe");
094    } else {
095      return null;
096    }
097  }
098
099  private String compareXml(InputStream expected, InputStream actual) throws Exception {
100    return compareElements("", loadXml(expected).getDocumentElement(), loadXml(actual).getDocumentElement());
101  }
102
103  private String compareXml(String expected, String actual) throws Exception {
104    return compareElements("", loadXml(expected).getDocumentElement(), loadXml(actual).getDocumentElement());
105  }
106
107  private String compareElements(String path, Element expectedElement, Element actualElement) {
108    if (!namespacesMatch(expectedElement.getNamespaceURI(), actualElement.getNamespaceURI()))
109      return createNotEqualMessage("Namespaces differ at " + path, expectedElement.getNamespaceURI(), actualElement.getNamespaceURI());
110    if (!expectedElement.getLocalName().equals(actualElement.getLocalName()))
111      return createNotEqualMessage("Names differ at " + path ,  expectedElement.getLocalName(), actualElement.getLocalName());
112    path = path + "/" + expectedElement.getLocalName();
113    String s = compareAttributes(path, expectedElement.getAttributes(), actualElement.getAttributes());
114    if (!Utilities.noString(s))
115      return s;
116    s = compareAttributes(path, expectedElement.getAttributes(), actualElement.getAttributes());
117    if (!Utilities.noString(s))
118      return s;
119
120    Node expectedChild = expectedElement.getFirstChild();
121    Node actualChild = actualElement.getFirstChild();
122    expectedChild = skipBlankText(expectedChild);
123    actualChild = skipBlankText(actualChild);
124    while (expectedChild != null && actualChild != null) {
125      if (expectedChild.getNodeType() != actualChild.getNodeType())
126        return createNotEqualMessage("node type mismatch in children of " + path, Short.toString(expectedElement.getNodeType()), Short.toString(actualElement.getNodeType()));
127      if (expectedChild.getNodeType() == Node.TEXT_NODE) {
128        if (!normalise(expectedChild.getTextContent()).equals(normalise(actualChild.getTextContent())))
129          return createNotEqualMessage("Text differs at " + path, normalise(expectedChild.getTextContent()).toString(), normalise(actualChild.getTextContent()).toString());
130      } else if (expectedChild.getNodeType() == Node.ELEMENT_NODE) {
131        s = compareElements(path, (Element) expectedChild, (Element) actualChild);
132        if (!Utilities.noString(s))
133          return s;
134      }
135
136      expectedChild = skipBlankText(expectedChild.getNextSibling());
137      actualChild = skipBlankText(actualChild.getNextSibling());
138    }
139    if (expectedChild != null)
140      return "node mismatch - more nodes in actual in children of " + path;
141    if (actualChild != null)
142      return "node mismatch - more nodes in expected in children of " + path;
143    return null;
144  }
145
146  private boolean namespacesMatch(String ns1, String ns2) {
147    return ns1 == null ? ns2 == null : ns1.equals(ns2);
148  }
149
150  private String normalise(String text) {
151    String result = text.trim().replace('\r', ' ').replace('\n', ' ').replace('\t', ' ');
152    while (result.contains("  "))
153      result = result.replace("  ", " ");
154    return result;
155  }
156
157  private String compareAttributes(String path, NamedNodeMap expected, NamedNodeMap actual) {
158    for (int i = 0; i < expected.getLength(); i++) {
159
160      Node expectedNode = expected.item(i);
161      String expectedNodeName = expectedNode.getNodeName();
162      if (!(expectedNodeName.equals("xmlns") || expectedNodeName.startsWith("xmlns:"))) {
163        Node actualNode = actual.getNamedItem(expectedNodeName);
164        if (actualNode == null)
165          return "Attributes differ at " + path + ": missing attribute " + expectedNodeName;
166        if (!normalise(expectedNode.getTextContent()).equals(normalise(actualNode.getTextContent()))) {
167          byte[] b1 = unBase64(expectedNode.getTextContent());
168          byte[] b2 = unBase64(actualNode.getTextContent());
169          if (!sameBytes(b1, b2))
170            return createNotEqualMessage("Attributes differ at " + path, normalise(expectedNode.getTextContent()).toString(), normalise(actualNode.getTextContent()).toString()) ;
171        }
172      }
173    }
174    return null;
175  }
176
177  private boolean sameBytes(byte[] b1, byte[] b2) {
178    if (b1.length == 0 || b2.length == 0)
179      return false;
180    if (b1.length != b2.length)
181      return false;
182    for (int i = 0; i < b1.length; i++)
183      if (b1[i] != b2[i])
184        return false;
185    return true;
186  }
187
188 private byte[] unBase64(String text) {
189    return Base64.decodeBase64(text);
190  }
191
192 private Node skipBlankText(Node node) {
193    while (node != null && (((node.getNodeType() == Node.TEXT_NODE) && StringUtils.isWhitespace(node.getTextContent())) || (node.getNodeType() == Node.COMMENT_NODE)))
194      node = node.getNextSibling();
195    return node;
196  }
197
198 private Document loadXml(String fn) throws Exception {
199    return loadXml(new FileInputStream(fn));
200  }
201
202 private Document loadXml(InputStream fn) throws Exception {
203    DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
204    factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
205    factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
206    factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
207    factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
208    factory.setXIncludeAware(false);
209    factory.setExpandEntityReferences(false);
210
211    factory.setNamespaceAware(true);
212    DocumentBuilder builder = factory.newDocumentBuilder();
213    return builder.parse(fn);
214  }
215
216  public static String checkJsonSrcIsSame(String expected, String actual, JsonObject externals) throws FileNotFoundException, IOException {
217    return checkJsonSrcIsSame(expected, actual, true, externals);
218  }
219
220  public static String checkJsonSrcIsSame(String expectedString, String actualString, boolean showDiff, JsonObject externals) throws FileNotFoundException, IOException {
221    CompareUtilities self = new CompareUtilities();
222    self.externals = externals;
223    String result = self.compareJsonSrc(expectedString, actualString);
224    if (result != null && SHOW_DIFF && showDiff) {
225      String diff = null;
226      if (System.getProperty("os.name").contains("Linux"))
227        diff = Utilities.path("/", "usr", "bin", "meld");
228      else if (System.getenv("ProgramFiles(X86)") != null) {
229        if (Utilities.checkFile("WinMerge", Utilities.path(System.getenv("ProgramFiles(X86)"), "WinMerge"), "\\WinMergeU.exe", null))
230          diff = Utilities.path(System.getenv("ProgramFiles(X86)"), "WinMerge", "WinMergeU.exe");
231        else if (Utilities.checkFile("WinMerge", Utilities.path(System.getenv("ProgramFiles(X86)"), "Meld"), "\\Meld.exe", null))
232          diff = Utilities.path(System.getenv("ProgramFiles(X86)"), "Meld", "Meld.exe");
233      }
234      if (diff == null || diff.isEmpty())
235        return result;
236
237      List<String> command = new ArrayList<String>();
238      String expected = Utilities.path("[tmp]", "expected" + expectedString.hashCode() + ".json");
239      String actual = Utilities.path("[tmp]", "actual" + actualString.hashCode() + ".json");
240      TextFile.stringToFile(expectedString, expected);
241      TextFile.stringToFile(actualString, actual);
242      command.add(diff);
243      if (diff.toLowerCase().contains("meld"))
244        command.add("--newtab");
245      command.add(expected);
246      command.add(actual);
247
248      ProcessBuilder builder = new ProcessBuilder(command);
249      builder.directory(new CSFile(Utilities.path("[tmp]")));
250      builder.start();
251
252    }
253    return result;
254  }
255
256  public static String checkJsonIsSame(String expected, String actual) throws FileNotFoundException, IOException {
257    CompareUtilities self = new CompareUtilities();
258    String result = self.compareJson(expected, actual);
259    if (result != null && SHOW_DIFF) {
260      String diff = Utilities.path(System.getenv("ProgramFiles(X86)"), "WinMerge", "WinMergeU.exe");
261      List<String> command = new ArrayList<String>();
262      command.add("\"" + diff + "\" \"" + expected +  "\" \"" + actual + "\"");
263
264      ProcessBuilder builder = new ProcessBuilder(command);
265      builder.directory(new CSFile(Utilities.path("[tmp]")));
266      builder.start();
267
268    }
269    return result;
270  }
271
272  private String compareJsonSrc(String expected, String actual) throws FileNotFoundException, IOException {
273    JsonObject actualJsonObject = JsonParser.parseObject(actual);
274    JsonObject expectedJsonObject = JsonParser.parseObject(expected);
275    return compareObjects("", expectedJsonObject, actualJsonObject);
276  }
277
278  private String compareJson(String expected, String actual) throws FileNotFoundException, IOException {
279    JsonObject actualJsonObject = JsonParser.parseObject(TextFile.fileToString(actual));
280    JsonObject expectedJsonObject = JsonParser.parseObject(TextFile.fileToString(expected));
281    return compareObjects("", expectedJsonObject, actualJsonObject);
282  }
283
284  private String compareObjects(String path, JsonObject expectedJsonObject, JsonObject actualJsonObject) {
285    List<String> optionals = listOptionals(expectedJsonObject);
286    for (JsonProperty en : actualJsonObject.getProperties()) {
287      String n = en.getName();
288      if (!n.equals("fhir_comments")) {
289        if (expectedJsonObject.has(n)) {
290          String s = compareNodes(path + '.' + n, expectedJsonObject.get(n), en.getValue());
291          if (!Utilities.noString(s))
292            return s;
293        } else
294          return "properties differ at " + path + ": missing property " + n;
295      }
296    }
297    for (JsonProperty en : expectedJsonObject.getProperties()) {
298      String n = en.getName();
299      if (!n.equals("fhir_comments") && !n.equals("$optional$") && !optionals.contains(n)) {
300        if (!actualJsonObject.has(n))
301          return "properties differ at " + path + ": missing property " + n;
302      }
303    }
304    return null;
305  }
306
307 private List<String> listOptionals(JsonObject expectedJsonObject) {
308    List<String> res = new ArrayList<>();
309    if (expectedJsonObject.has("$optional-properties$")) {
310      res.add("$optional-properties$");
311      for (String s : expectedJsonObject.getStrings("$optional-properties$")) {
312        res.add(s);
313      }
314    }
315    return res;
316  }
317
318  private String compareNodes(String path, JsonElement expectedJsonElement, JsonElement actualJsonElement) {
319    if (!(expectedJsonElement instanceof JsonPrimitive && actualJsonElement instanceof JsonPrimitive)) {
320      if (actualJsonElement.getClass() != expectedJsonElement.getClass()) {
321        return createNotEqualMessage("properties differ at " + path, expectedJsonElement.getClass().getName(), actualJsonElement.getClass().getName());
322      }
323    }
324    if (actualJsonElement instanceof JsonPrimitive) {
325      JsonPrimitive actualJsonPrimitive = (JsonPrimitive) actualJsonElement;
326      JsonPrimitive expectedJsonPrimitive = (JsonPrimitive) expectedJsonElement;
327      if (actualJsonPrimitive.isJsonBoolean() && expectedJsonPrimitive.isJsonBoolean()) {
328        if (actualJsonPrimitive.asBoolean() != expectedJsonPrimitive.asBoolean())
329          return createNotEqualMessage("boolean property values differ at " + path , expectedJsonPrimitive.asString(), actualJsonPrimitive.asString());
330      } else if (actualJsonPrimitive.isJsonString() && expectedJsonPrimitive.isJsonString()) {
331        String actualJsonString = actualJsonPrimitive.asString();
332        String expectedJsonString = expectedJsonPrimitive.asString();
333        if (!(actualJsonString.contains("<div") && expectedJsonString.contains("<div")))
334          if (!matches(actualJsonString, expectedJsonString))
335            if (!sameBytes(unBase64(actualJsonString), unBase64(expectedJsonString)))
336              return createNotEqualMessage("string property values differ at " + path, expectedJsonString, actualJsonString);
337      } else if (actualJsonPrimitive.isJsonNumber() && expectedJsonPrimitive.isJsonNumber()) {
338        if (!actualJsonPrimitive.asString().equals(expectedJsonPrimitive.asString()))
339          return createNotEqualMessage("number property values differ at " + path, expectedJsonPrimitive.asString(), actualJsonPrimitive.asString());
340      } else if (expectedJsonElement instanceof JsonNull) {
341        return actualJsonPrimitive instanceof JsonNull ? null : createNotEqualMessage("null Properties differ at " + path, "null", actualJsonPrimitive.asString());
342      } else {
343        return createNotEqualMessage("property types differ at " + path, expectedJsonPrimitive.asString(), actualJsonPrimitive.asString());
344      }
345    } else if (actualJsonElement instanceof JsonObject) {
346      String s = compareObjects(path, (JsonObject) expectedJsonElement, (JsonObject) actualJsonElement);
347      if (!Utilities.noString(s))
348        return s;
349    } else if (actualJsonElement instanceof JsonArray) {
350      JsonArray actualArray = (JsonArray) actualJsonElement;
351      JsonArray expectedArray = (JsonArray) expectedJsonElement;
352      
353      int expectedMin = countExpectedMin(expectedArray);
354      int as = actualArray.size();
355      int es = expectedArray.size();
356      int oc = optionalCount(expectedArray);
357      
358      if (as > es || as < expectedMin)
359        return createNotEqualMessage("array item count differs at " + path, Integer.toString(es), Integer.toString(as));
360      int c = 0;
361      for (int i = 0; i < es; i++) {
362        if (c >= as) {
363          if (i >= es - oc && isOptional(expectedArray.get(i))) {
364            return null; // this is OK 
365          } else {
366            return "One or more array items did not match at "+path+" starting at index "+i;
367          }
368        }
369        String s = compareNodes(path + "[" + Integer.toString(i) + "]", expectedArray.get(i), actualArray.get(c));
370        if (!Utilities.noString(s) && !isOptional(expectedArray.get(i))) {
371          return s;
372        }
373        if (Utilities.noString(s)) {
374          c++;
375        }
376      }
377      if (c < as) {
378        return "Unexpected Node found in array at index "+c;
379      }
380    } else
381      return "unhandled property " + actualJsonElement.getClass().getName();
382    return null;
383  }
384
385 private int optionalCount(JsonArray arr) {
386    int c = 0;
387    for (JsonElement e : arr) {
388      if (e.isJsonObject()) {
389        JsonObject j = e.asJsonObject();
390        if (j.has("$optional$") && j.asBoolean("$optional$")) {
391          c++;
392        }
393      }
394    }
395    return c;
396  }
397
398private boolean isOptional(JsonElement e) {
399    return e.isJsonObject() && e.asJsonObject().has("$optional$");
400  }
401
402 private int countExpectedMin(JsonArray array) {
403    int count = array.size();
404    for (JsonElement e : array) {
405      if (isOptional(e)) {
406        count--;
407      }
408    }
409    return count;
410  }
411
412  private boolean matches(String actualJsonString, String expectedJsonString) {
413    if (expectedJsonString.startsWith("$") && expectedJsonString.endsWith("$")) {
414      if (expectedJsonString.startsWith("$choice:")) {
415        return Utilities.existsInList(actualJsonString, readChoices(expectedJsonString.substring(8, expectedJsonString.length()-1)));
416
417      } else if (expectedJsonString.startsWith("$fragments:")) {
418        List<String> fragments = readChoices(expectedJsonString.substring(11, expectedJsonString.length()-1));
419        for (String f : fragments) {
420          if (!actualJsonString.toLowerCase().contains(f.toLowerCase())) {
421            return false;
422          }
423        }
424        return true;
425      } else if (expectedJsonString.startsWith("$external:")) {
426        String[] cmd = expectedJsonString.substring(1, expectedJsonString.length() - 1).split("\\:");
427        if (externals != null) {
428          String s = externals.asString(cmd[1]);
429          return actualJsonString.equals(s);
430        } else {
431          List<String> fragments = readChoices(cmd[2]);
432          for (String f : fragments) {
433            if (!actualJsonString.toLowerCase().contains(f.toLowerCase())) {
434              return false;
435            }
436          }
437          return true;
438        }
439      } else {
440        switch (expectedJsonString) {
441        case "$$" : return true;
442        case "$instant$": return actualJsonString.matches("([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\\.[0-9]{1,9})?(Z|(\\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))");
443        case "$uuid$": return actualJsonString.matches("urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}");
444        default: 
445          throw new Error("Unhandled template: "+expectedJsonString);
446        }
447      }
448    } else {
449      return actualJsonString.equals(expectedJsonString);
450    }
451  }
452
453 private List<String> readChoices(String s) {
454    List<String> list = new ArrayList<>();
455    for (String p : s.split("\\|")) {
456      list.add(p);
457    }
458    return list;
459  }
460
461  public static String checkTextIsSame(String expected, String actual) throws FileNotFoundException, IOException {
462    return checkTextIsSame(expected, actual, true);
463  }
464
465  public static String checkTextIsSame(String expectedString, String actualString, boolean showDiff) throws FileNotFoundException, IOException {
466    CompareUtilities self = new CompareUtilities();
467    String result = self.compareText(expectedString, actualString);
468    if (result != null && SHOW_DIFF && showDiff) {
469      String diff = null;
470      if (System.getProperty("os.name").contains("Linux"))
471        diff = Utilities.path("/", "usr", "bin", "meld");
472      else {
473        if (Utilities.checkFile("WinMerge", Utilities.path(System.getenv("ProgramFiles(X86)"), "WinMerge"), "\\WinMergeU.exe", null))
474          diff = Utilities.path(System.getenv("ProgramFiles(X86)"), "WinMerge", "WinMergeU.exe");
475        else if (Utilities.checkFile("WinMerge", Utilities.path(System.getenv("ProgramFiles(X86)"), "Meld"), "\\Meld.exe", null))
476          diff = Utilities.path(System.getenv("ProgramFiles(X86)"), "Meld", "Meld.exe");
477      }
478      if (diff == null || diff.isEmpty())
479        return result;
480
481      List<String> command = new ArrayList<String>();
482      String actual = Utilities.path("[tmp]", "actual" + actualString.hashCode() + ".json");
483      String expected = Utilities.path("[tmp]", "expected" + expectedString.hashCode() + ".json");
484      TextFile.stringToFile(expectedString, expected);
485      TextFile.stringToFile(actualString, actual);
486      command.add(diff);
487      if (diff.toLowerCase().contains("meld"))
488        command.add("--newtab");
489      command.add(expected);
490      command.add(actual);
491
492      ProcessBuilder builder = new ProcessBuilder(command);
493      builder.directory(new CSFile(Utilities.path("[tmp]")));
494      builder.start();
495
496    }
497    return result;
498  }
499
500
501 private String compareText(String expectedString, String actualString) {
502    for (int i = 0; i < Integer.min(expectedString.length(), actualString.length()); i++) {
503      if (expectedString.charAt(i) != actualString.charAt(i))
504        return createNotEqualMessage("Strings differ at character " + Integer.toString(i), String.valueOf(expectedString.charAt(i)), String.valueOf(actualString.charAt(i)));
505    }
506    if (expectedString.length() != actualString.length())
507      return createNotEqualMessage("Strings differ in length but match to the end of the shortest.", Integer.toString(expectedString.length()), Integer.toString(actualString.length()));
508    return null;
509  }
510
511}