
001package org.hl7.fhir.r4.utils.sql; 002 003import java.math.BigDecimal; 004import java.util.ArrayList; 005import java.util.List; 006 007import org.apache.commons.codec.binary.Base64; 008import org.hl7.fhir.exceptions.FHIRException; 009import org.hl7.fhir.exceptions.PathEngineException; 010import org.hl7.fhir.r4.context.IWorkerContext; 011import org.hl7.fhir.r4.fhirpath.ExpressionNode; 012import org.hl7.fhir.r4.fhirpath.FHIRPathEngine; 013import org.hl7.fhir.r4.fhirpath.TypeDetails; 014import org.hl7.fhir.r4.fhirpath.ExpressionNode.CollectionStatus; 015import org.hl7.fhir.r4.fhirpath.FHIRPathEngine.IEvaluationContext; 016import org.hl7.fhir.r4.fhirpath.FHIRPathUtilityClasses.FunctionDetails; 017import org.hl7.fhir.r4.model.Base; 018import org.hl7.fhir.r4.model.Base64BinaryType; 019import org.hl7.fhir.r4.model.BaseDateTimeType; 020import org.hl7.fhir.r4.model.BooleanType; 021import org.hl7.fhir.r4.model.DecimalType; 022import org.hl7.fhir.r4.model.IntegerType; 023import org.hl7.fhir.r4.model.Property; 024import org.hl7.fhir.r4.model.StringType; 025import org.hl7.fhir.r4.model.ValueSet; 026import org.hl7.fhir.r4.utils.sql.Cell; 027import org.hl7.fhir.r4.utils.sql.Column; 028import org.hl7.fhir.r4.utils.sql.Provider; 029import org.hl7.fhir.r4.utils.sql.Storage; 030import org.hl7.fhir.r4.utils.sql.Store; 031import org.hl7.fhir.r4.utils.sql.Validator; 032import org.hl7.fhir.r4.utils.sql.Value; 033import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage; 034import org.hl7.fhir.utilities.json.model.JsonObject; 035import org.hl7.fhir.utilities.validation.ValidationMessage; 036 037 038/** 039 * How to use the Runner: 040 * 041 * create a resource, and fill out: 042 * the context (supports the FHIRPathEngine) 043 * a store that handles the output 044 * a tracker - if you want 045 * 046 * Once it's created, you either run it as a batch, or in trickle mode 047 * 048 * (1) Batch Mode 049 * 050 * * provide a provider 051 * * call execute() with a ViewDefinition 052 * * wait... (watch with an observer if you want to track progress) 053 * 054 * (2) Trickle Mode 055 * * call 'prepare', and keep the WorkContext that's returned 056 * * each time there's a resource to process, call processResource and pass in the workContext and the resource 057 * * when done, call finish(WorkContext) 058 */ 059 060@MarkedToMoveToAdjunctPackage 061public class Runner implements IEvaluationContext { 062 063 public interface IRunnerObserver { 064 public void handleRow(Base resource, int total, int cursor); 065 } 066 067 public class WorkContext { 068 private JsonObject vd; 069 private Store store; 070 protected WorkContext(JsonObject vd) { 071 super(); 072 this.vd = vd; 073 } 074 075 } 076 private IWorkerContext context; 077 private Provider provider; 078 private Storage storage; 079 private IRunnerObserver observer; 080 private List<String> prohibitedNames = new ArrayList<String>(); 081 private FHIRPathEngine fpe; 082 083 private String resourceName; 084 private List<ValidationMessage> issues; 085 private int resCount; 086 087 088 public IWorkerContext getContext() { 089 return context; 090 } 091 public void setContext(IWorkerContext context) { 092 this.context = context; 093 } 094 095 public Provider getProvider() { 096 return provider; 097 } 098 public void setProvider(Provider provider) { 099 this.provider = provider; 100 } 101 102 public Storage getStorage() { 103 return storage; 104 } 105 public void setStorage(Storage storage) { 106 this.storage = storage; 107 } 108 109 public List<String> getProhibitedNames() { 110 return prohibitedNames; 111 } 112 113 public void execute(JsonObject viewDefinition) { 114 execute("$", viewDefinition); 115 } 116 117 public void execute(String path, JsonObject viewDefinition) { 118 WorkContext wc = prepare(path, viewDefinition); 119 try { 120 evaluate(wc); 121 } finally { 122 finish(wc); 123 } 124 } 125 126 private void evaluate(WorkContext wc) { 127 List<Base> data = provider.fetch(resourceName); 128 129 int i = 0; 130 for (Base b : data) { 131 if (observer != null) { 132 observer.handleRow(b, data.size(), i); 133 } 134 processResource(wc.vd, wc.store, b); 135 i++; 136 } 137 } 138 139 public WorkContext prepare(String path, JsonObject viewDefinition) { 140 WorkContext wc = new WorkContext(viewDefinition); 141 if (context == null) { 142 throw new FHIRException("No context provided"); 143 } 144 fpe = new FHIRPathEngine(context); 145 fpe.setHostServices(this); 146 fpe.setEmitSQLonFHIRWarning(true); 147 if (viewDefinition == null) { 148 throw new FHIRException("No viewDefinition provided"); 149 } 150 if (provider == null) { 151 throw new FHIRException("No provider provided"); 152 } 153 if (storage == null) { 154 throw new FHIRException("No storage provided"); 155 } 156 Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName()); 157 validator.checkViewDefinition(path, viewDefinition); 158 issues = validator.getIssues(); 159 validator.dump(); 160 validator.check(); 161 resourceName = validator.getResourceName(); 162 wc.store = storage.createStore(wc.vd.asString("name"), (List<Column>) wc.vd.getUserData("columns")); 163 return wc; 164 } 165 166 public void processResource(WorkContext wc, Base b) { 167 if (observer != null) { 168 observer.handleRow(b, -1, resCount); 169 } 170 processResource(wc.vd, wc.store, b); 171 resCount++; 172 wc.store.flush(); 173 } 174 175 private void processResource(JsonObject vd, Store store, Base b) { 176 boolean ok = true; 177 for (JsonObject w : vd.getJsonObjects("where")) { 178 String expr = w.asString("path"); 179 ExpressionNode node = fpe.parse(expr); 180 boolean pass = fpe.evaluateToBoolean(vd, b, b, b, node); 181 if (!pass) { 182 ok = false; 183 break; 184 } 185 } 186 if (ok) { 187 List<List<Cell>> rows = new ArrayList<>(); 188 rows.add(new ArrayList<Cell>()); 189 190 for (JsonObject select : vd.getJsonObjects("select")) { 191 executeSelect(vd, select, b, rows); 192 } 193 for (List<Cell> row : rows) { 194 storage.addRow(store, row); 195 } 196 } 197 } 198 199 public void finish(WorkContext wc) { 200 storage.finish(wc.store); 201 } 202 203 private void executeSelect(JsonObject vd, JsonObject select, Base b, List<List<Cell>> rows) { 204 List<Base> focus = new ArrayList<>(); 205 206 if (select.has("forEach")) { 207 focus.addAll(executeForEach(vd, select, b)); 208 } else if (select.has("forEachOrNull")) { 209 210 focus.addAll(executeForEachOrNull(vd, select, b)); 211 if (focus.isEmpty()) { 212 List<Column> columns = (List<Column>) select.getUserData("columns"); 213 for (List<Cell> row : rows) { 214 for (Column c : columns) { 215 Cell cell = cell(row, c.getName()); 216 if (cell == null) { 217 row.add(new Cell(c, null)); 218 } 219 } 220 } 221 return; 222 } 223 } else { 224 focus.add(b); 225 } 226 227 // } else if (select.has("unionAll")) { 228 // focus.addAll(executeUnion(select, b)); 229 230 List<List<Cell>> tempRows = new ArrayList<>(); 231 tempRows.addAll(rows); 232 rows.clear(); 233 234 for (Base f : focus) { 235 List<List<Cell>> rowsToAdd = cloneRows(tempRows); 236 237 for (JsonObject column : select.getJsonObjects("column")) { 238 executeColumn(vd, column, f, rowsToAdd); 239 } 240 241 for (JsonObject sub : select.getJsonObjects("select")) { 242 executeSelect(vd, sub, f, rowsToAdd); 243 } 244 245 executeUnionAll(vd, select.getJsonObjects("unionAll"), f, rowsToAdd); 246 247 rows.addAll(rowsToAdd); 248 } 249 } 250 251 private void executeUnionAll(JsonObject vd, List<JsonObject> unionList, Base b, List<List<Cell>> rows) { 252 if (unionList.isEmpty()) { 253 return; 254 } 255 List<List<Cell>> sourceRows = new ArrayList<>(); 256 sourceRows.addAll(rows); 257 rows.clear(); 258 259 for (JsonObject union : unionList) { 260 List<List<Cell>> tempRows = new ArrayList<>(); 261 tempRows.addAll(sourceRows); 262 executeSelect(vd, union, b, tempRows); 263 rows.addAll(tempRows); 264 } 265 } 266 267 private List<List<Cell>> cloneRows(List<List<Cell>> rows) { 268 List<List<Cell>> list = new ArrayList<>(); 269 for (List<Cell> row : rows) { 270 list.add(cloneRow(row)); 271 } 272 return list; 273 } 274 275 private List<Cell> cloneRow(List<Cell> cells) { 276 List<Cell> list = new ArrayList<>(); 277 for (Cell cell : cells) { 278 list.add(cell.copy()); 279 } 280 return list; 281 } 282 283 private List<Base> executeForEach(JsonObject vd, JsonObject focus, Base b) { 284 ExpressionNode n = (ExpressionNode) focus.getUserData("forEach"); 285 List<Base> result = new ArrayList<>(); 286 result.addAll(fpe.evaluate(vd, b, n)); 287 return result; 288 } 289 290 private List<Base> executeForEachOrNull(JsonObject vd, JsonObject focus, Base b) { 291 ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull"); 292 List<Base> result = new ArrayList<>(); 293 result.addAll(fpe.evaluate(vd, b, n)); 294 return result; 295 } 296 297 private void executeColumn(JsonObject vd, JsonObject column, Base b, List<List<Cell>> rows) { 298 ExpressionNode n = (ExpressionNode) column.getUserData("path"); 299 List<Base> bl2 = new ArrayList<>(); 300 if (b != null) { 301 bl2.addAll(fpe.evaluate(vd, b, n)); 302 } 303 Column col = (Column) column.getUserData("column"); 304 if (col == null) { 305 System.out.println("Error"); 306 } else { 307 for (List<Cell> row : rows) { 308 Cell c = cell(row, col.getName()); 309 if (c == null) { 310 c = new Cell(col); 311 row.add(c); 312 } 313 if (!bl2.isEmpty()) { 314 if (bl2.size() + c.getValues().size() > 1) { 315 // this is a problem if collection != true or if the storage can't deal with it 316 // though this should've been picked up before now - but there are circumstances where it wouldn't be 317 if (!c.getColumn().isColl()) { 318 throw new FHIRException("The column "+c.getColumn().getName()+" is not allowed multiple values, but at least one row has multiple values"); 319 } 320 } 321 for (Base b2 : bl2) { 322 c.getValues().add(genValue(c.getColumn(), b2)); 323 } 324 } 325 } 326 } 327 } 328 329 330 private Value genValue(Column column, Base b) { 331 if (column.getKind() == null) { 332 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type (null) for column "+column.getName()); // can't happen 333 } 334 switch (column.getKind()) { 335 case Binary: 336 if (b instanceof Base64BinaryType) { 337 Base64BinaryType bb = (Base64BinaryType) b; 338 return Value.makeBinary(bb.primitiveValue(), bb.getValue()); 339 } else if (b.isBooleanPrimitive()) { // ElementModel 340 return Value.makeBinary(b.primitiveValue(), Base64.decodeBase64(b.primitiveValue())); 341 } else { 342 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a binary column for column "+column.getName()); 343 } 344 case Boolean: 345 if (b instanceof BooleanType) { 346 BooleanType bb = (BooleanType) b; 347 return Value.makeBoolean(bb.primitiveValue(), bb.booleanValue()); 348 } else if (b.isBooleanPrimitive()) { // ElementModel 349 return Value.makeBoolean(b.primitiveValue(), "true".equals(b.primitiveValue())); 350 } else { 351 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a boolean column for column "+column.getName()); 352 } 353 case Complex: 354 if (b.isPrimitive()) { 355 throw new FHIRException("Attempt to add a primitive type "+b.fhirType()+" to a complex column for column "+column.getName()); 356 } else { 357 return Value.makeComplex(b); 358 } 359 case DateTime: 360 if (b instanceof BaseDateTimeType) { 361 BaseDateTimeType d = (BaseDateTimeType) b; 362 return Value.makeDate(d.primitiveValue(), d.getValue()); 363 } else if (b.isPrimitive() && b.isDateTime()) { // ElementModel 364 return Value.makeDate(b.primitiveValue(), b.dateTimeValue().getValue()); 365 } else { 366 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 367 } 368 case Decimal: 369 if (b instanceof DecimalType) { 370 DecimalType d = (DecimalType) b; 371 return Value.makeDecimal(d.primitiveValue(), d.getValue()); 372 } else if (b.isPrimitive()) { // ElementModel 373 return Value.makeDecimal(b.primitiveValue(), new BigDecimal(b.primitiveValue())); 374 } else { 375 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 376 } 377 case Integer: 378 if (b instanceof IntegerType) { 379 IntegerType i = (IntegerType) b; 380 return Value.makeInteger(i.primitiveValue(), i.getValue()); 381 } else if (b.isPrimitive()) { // ElementModel 382 return Value.makeInteger(b.primitiveValue(), Integer.valueOf(b.primitiveValue())); 383 } else { 384 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 385 } 386 case String: 387 if (b.isPrimitive()) { 388 return Value.makeString(b.primitiveValue()); 389 } else { 390 throw new FHIRException("Attempt to add a complex type "+b.fhirType()+" to a string column for column "+column.getName()); 391 } 392 case Time: 393 if (b.fhirType().equals("time")) { 394 return Value.makeString(b.primitiveValue()); 395 } else { 396 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a time column for column "+column.getName()); 397 } 398 default: 399 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type for column "+column.getName()); 400 } 401 } 402 403 private Column column(String columnName, List<Column> columns) { 404 for (Column t : columns) { 405 if (t.getName().equalsIgnoreCase(columnName)) { 406 return t; 407 } 408 } 409 return null; 410 } 411 412 private Cell cell(List<Cell> cells, String columnName) { 413 for (Cell t : cells) { 414 if (t.getColumn().getName().equalsIgnoreCase(columnName)) { 415 return t; 416 } 417 } 418 return null; 419 } 420 421 @Override 422 public List<Base> resolveConstant(FHIRPathEngine engine, Object appContext, String name, boolean beforeContext, boolean explicitConstant) throws PathEngineException { 423 List<Base> list = new ArrayList<Base>(); 424 if (explicitConstant) { 425 JsonObject vd = (JsonObject) appContext; 426 JsonObject constant = findConstant(vd, name); 427 if (constant != null) { 428 Base b = (Base) constant.getUserData("value"); 429 if (b != null) { 430 list.add(b); 431 } 432 } 433 } 434 return list; 435 } 436 437 @Override 438 public TypeDetails resolveConstantType(FHIRPathEngine engine, Object appContext, String name, boolean explicitConstant) throws PathEngineException { 439 if (explicitConstant) { 440 JsonObject vd = (JsonObject) appContext; 441 JsonObject constant = findConstant(vd, name.substring(1)); 442 if (constant != null) { 443 Base b = (Base) constant.getUserData("value"); 444 if (b != null) { 445 return new TypeDetails(CollectionStatus.SINGLETON, b.fhirType()); 446 } 447 } 448 } 449 return null; 450 } 451 452 private JsonObject findConstant(JsonObject vd, String name) { 453 for (JsonObject o : vd.getJsonObjects("constant")) { 454 if (name.equals(o.asString("name"))) { 455 return o; 456 } 457 } 458 return null; 459 } 460 @Override 461 public boolean log(String argument, List<Base> focus) { 462 throw new Error("Not implemented yet: log"); 463 } 464 465 @Override 466 public FunctionDetails resolveFunction(FHIRPathEngine engine, String functionName) { 467 switch (functionName) { 468 case "getResourceKey" : return new FunctionDetails("Unique Key for resource", 0, 0); 469 case "getReferenceKey" : return new FunctionDetails("Unique Key for resource that is the target of the reference", 0, 1); 470 default: return null; 471 } 472 } 473 @Override 474 public TypeDetails checkFunction(FHIRPathEngine engine, Object appContext, String functionName, TypeDetails focus, List<TypeDetails> parameters) throws PathEngineException { 475 switch (functionName) { 476 case "getResourceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string"); 477 case "getReferenceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string"); 478 default: throw new Error("Not known: "+functionName); 479 } 480 } 481 482 @Override 483 public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) { 484 switch (functionName) { 485 case "getResourceKey" : return executeResourceKey(focus); 486 case "getReferenceKey" : return executeReferenceKey(null, focus, parameters); 487 default: throw new Error("Not known: "+functionName); 488 } 489 } 490 491 private List<Base> executeResourceKey(List<Base> focus) { 492 List<Base> base = new ArrayList<Base>(); 493 if (focus.size() == 1) { 494 Base res = focus.get(0); 495 if (!res.hasUserData("Storage.key")) { 496 String key = storage.getKeyForSourceResource(res); 497 if (key == null) { 498 throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase()); 499 } else { 500 res.setUserData("Storage.key", key); 501 } 502 } 503 base.add(new StringType(res.getUserString("Storage.key"))); 504 } 505 return base; 506 } 507 508 private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) { 509 String rt = null; 510 if (parameters.size() > 0) { 511 rt = parameters.get(0).get(0).primitiveValue(); 512 if (rt.startsWith("FHIR.")) { 513 rt = rt.substring(5); 514 } 515 } 516 List<Base> base = new ArrayList<Base>(); 517 if (focus.size() == 1) { 518 Base res = focus.get(0); 519 String ref = null; 520 if (res.fhirType().equals("Reference")) { 521 ref = getRef(res); 522 } else if (res.isPrimitive()) { 523 ref = res.primitiveValue(); 524 } else { 525 throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType()); 526 } 527 if (ref != null) { 528 Base target = provider.resolveReference(rootResource, ref, rt); 529 if (target != null) { 530 if (!res.hasUserData("Storage.key")) { 531 String key = storage.getKeyForTargetResource(target); 532 if (key == null) { 533 throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase()); 534 } else { 535 res.setUserData("Storage.key", key); 536 } 537 } 538 base.add(new StringType(res.getUserString("Storage.key"))); 539 } 540 } 541 } 542 return base; 543 } 544 545 private String getRef(Base res) { 546 Property prop = res.getChildByName("reference"); 547 if (prop != null && prop.getValues().size() == 1) { 548 return prop.getValues().get(0).primitiveValue(); 549 } 550 return null; 551 } 552 553 @Override 554 public Base resolveReference(FHIRPathEngine engine, Object appContext, String url, Base refContext) throws FHIRException { 555 throw new Error("Not implemented yet: resolveReference"); 556 } 557 558 @Override 559 public boolean conformsToProfile(FHIRPathEngine engine, Object appContext, Base item, String url) throws FHIRException { 560 throw new Error("Not implemented yet: conformsToProfile"); 561 } 562 563 @Override 564 public ValueSet resolveValueSet(FHIRPathEngine engine, Object appContext, String url) { 565 throw new Error("Not implemented yet: resolveValueSet"); 566 } 567 @Override 568 public boolean paramIsType(String name, int index) { 569 return "getReferenceKey".equals(name); 570 } 571 public List<ValidationMessage> getIssues() { 572 return issues; 573 } 574 575 576}