
001package org.hl7.fhir.r4.utils.sql; 002 003import java.math.BigDecimal; 004import java.util.ArrayList; 005import java.util.List; 006 007import lombok.extern.slf4j.Slf4j; 008import org.apache.commons.codec.binary.Base64; 009import org.hl7.fhir.exceptions.FHIRException; 010import org.hl7.fhir.exceptions.PathEngineException; 011import org.hl7.fhir.r4.context.IWorkerContext; 012import org.hl7.fhir.r4.fhirpath.ExpressionNode; 013import org.hl7.fhir.r4.fhirpath.FHIRPathEngine; 014import org.hl7.fhir.r4.fhirpath.TypeDetails; 015import org.hl7.fhir.r4.fhirpath.ExpressionNode.CollectionStatus; 016import org.hl7.fhir.r4.fhirpath.FHIRPathEngine.IEvaluationContext; 017import org.hl7.fhir.r4.fhirpath.FHIRPathUtilityClasses.FunctionDetails; 018import org.hl7.fhir.r4.model.Base; 019import org.hl7.fhir.r4.model.Base64BinaryType; 020import org.hl7.fhir.r4.model.BaseDateTimeType; 021import org.hl7.fhir.r4.model.BooleanType; 022import org.hl7.fhir.r4.model.DecimalType; 023import org.hl7.fhir.r4.model.IntegerType; 024import org.hl7.fhir.r4.model.Property; 025import org.hl7.fhir.r4.model.StringType; 026import org.hl7.fhir.r4.model.ValueSet; 027import org.hl7.fhir.r4.utils.sql.Cell; 028import org.hl7.fhir.r4.utils.sql.Column; 029import org.hl7.fhir.r4.utils.sql.Provider; 030import org.hl7.fhir.r4.utils.sql.Storage; 031import org.hl7.fhir.r4.utils.sql.Store; 032import org.hl7.fhir.r4.utils.sql.Validator; 033import org.hl7.fhir.r4.utils.sql.Value; 034import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage; 035import org.hl7.fhir.utilities.json.model.JsonObject; 036import org.hl7.fhir.utilities.validation.ValidationMessage; 037 038 039/** 040 * How to use the Runner: 041 * 042 * create a resource, and fill out: 043 * the context (supports the FHIRPathEngine) 044 * a store that handles the output 045 * a tracker - if you want 046 * 047 * Once it's created, you either run it as a batch, or in trickle mode 048 * 049 * (1) Batch Mode 050 * 051 * * provide a provider 052 * * call execute() with a ViewDefinition 053 * * wait... (watch with an observer if you want to track progress) 054 * 055 * (2) Trickle Mode 056 * * call 'prepare', and keep the WorkContext that's returned 057 * * each time there's a resource to process, call processResource and pass in the workContext and the resource 058 * * when done, call finish(WorkContext) 059 */ 060 061@MarkedToMoveToAdjunctPackage 062@Slf4j 063public class Runner implements IEvaluationContext { 064 065 public interface IRunnerObserver { 066 public void handleRow(Base resource, int total, int cursor); 067 } 068 069 public class WorkContext { 070 private JsonObject vd; 071 private Store store; 072 protected WorkContext(JsonObject vd) { 073 super(); 074 this.vd = vd; 075 } 076 077 } 078 private IWorkerContext context; 079 private Provider provider; 080 private Storage storage; 081 private IRunnerObserver observer; 082 private List<String> prohibitedNames = new ArrayList<String>(); 083 private FHIRPathEngine fpe; 084 085 private String resourceName; 086 private List<ValidationMessage> issues; 087 private int resCount; 088 089 090 public IWorkerContext getContext() { 091 return context; 092 } 093 public void setContext(IWorkerContext context) { 094 this.context = context; 095 } 096 097 public Provider getProvider() { 098 return provider; 099 } 100 public void setProvider(Provider provider) { 101 this.provider = provider; 102 } 103 104 public Storage getStorage() { 105 return storage; 106 } 107 public void setStorage(Storage storage) { 108 this.storage = storage; 109 } 110 111 public List<String> getProhibitedNames() { 112 return prohibitedNames; 113 } 114 115 public void execute(JsonObject viewDefinition) { 116 execute("$", viewDefinition); 117 } 118 119 public void execute(String path, JsonObject viewDefinition) { 120 WorkContext wc = prepare(path, viewDefinition); 121 try { 122 evaluate(wc); 123 } finally { 124 finish(wc); 125 } 126 } 127 128 private void evaluate(WorkContext wc) { 129 List<Base> data = provider.fetch(resourceName); 130 131 int i = 0; 132 for (Base b : data) { 133 if (observer != null) { 134 observer.handleRow(b, data.size(), i); 135 } 136 processResource(wc.vd, wc.store, b); 137 i++; 138 } 139 } 140 141 public WorkContext prepare(String path, JsonObject viewDefinition) { 142 WorkContext wc = new WorkContext(viewDefinition); 143 if (context == null) { 144 throw new FHIRException("No context provided"); 145 } 146 fpe = new FHIRPathEngine(context); 147 fpe.setHostServices(this); 148 fpe.setEmitSQLonFHIRWarning(true); 149 if (viewDefinition == null) { 150 throw new FHIRException("No viewDefinition provided"); 151 } 152 if (provider == null) { 153 throw new FHIRException("No provider provided"); 154 } 155 if (storage == null) { 156 throw new FHIRException("No storage provided"); 157 } 158 Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName()); 159 validator.checkViewDefinition(path, viewDefinition); 160 issues = validator.getIssues(); 161 validator.dump(); 162 validator.check(); 163 resourceName = validator.getResourceName(); 164 wc.store = storage.createStore(wc.vd.asString("name"), (List<Column>) wc.vd.getUserData("columns")); 165 return wc; 166 } 167 168 public void processResource(WorkContext wc, Base b) { 169 if (observer != null) { 170 observer.handleRow(b, -1, resCount); 171 } 172 processResource(wc.vd, wc.store, b); 173 resCount++; 174 wc.store.flush(); 175 } 176 177 private void processResource(JsonObject vd, Store store, Base b) { 178 boolean ok = true; 179 for (JsonObject w : vd.getJsonObjects("where")) { 180 String expr = w.asString("path"); 181 ExpressionNode node = fpe.parse(expr); 182 boolean pass = fpe.evaluateToBoolean(vd, b, b, b, node); 183 if (!pass) { 184 ok = false; 185 break; 186 } 187 } 188 if (ok) { 189 List<List<Cell>> rows = new ArrayList<>(); 190 rows.add(new ArrayList<Cell>()); 191 192 for (JsonObject select : vd.getJsonObjects("select")) { 193 executeSelect(vd, select, b, rows); 194 } 195 for (List<Cell> row : rows) { 196 storage.addRow(store, row); 197 } 198 } 199 } 200 201 public void finish(WorkContext wc) { 202 storage.finish(wc.store); 203 } 204 205 private void executeSelect(JsonObject vd, JsonObject select, Base b, List<List<Cell>> rows) { 206 List<Base> focus = new ArrayList<>(); 207 208 if (select.has("forEach")) { 209 focus.addAll(executeForEach(vd, select, b)); 210 } else if (select.has("forEachOrNull")) { 211 212 focus.addAll(executeForEachOrNull(vd, select, b)); 213 if (focus.isEmpty()) { 214 List<Column> columns = (List<Column>) select.getUserData("columns"); 215 for (List<Cell> row : rows) { 216 for (Column c : columns) { 217 Cell cell = cell(row, c.getName()); 218 if (cell == null) { 219 row.add(new Cell(c, null)); 220 } 221 } 222 } 223 return; 224 } 225 } else { 226 focus.add(b); 227 } 228 229 // } else if (select.has("unionAll")) { 230 // focus.addAll(executeUnion(select, b)); 231 232 List<List<Cell>> tempRows = new ArrayList<>(); 233 tempRows.addAll(rows); 234 rows.clear(); 235 236 for (Base f : focus) { 237 List<List<Cell>> rowsToAdd = cloneRows(tempRows); 238 239 for (JsonObject column : select.getJsonObjects("column")) { 240 executeColumn(vd, column, f, rowsToAdd); 241 } 242 243 for (JsonObject sub : select.getJsonObjects("select")) { 244 executeSelect(vd, sub, f, rowsToAdd); 245 } 246 247 executeUnionAll(vd, select.getJsonObjects("unionAll"), f, rowsToAdd); 248 249 rows.addAll(rowsToAdd); 250 } 251 } 252 253 private void executeUnionAll(JsonObject vd, List<JsonObject> unionList, Base b, List<List<Cell>> rows) { 254 if (unionList.isEmpty()) { 255 return; 256 } 257 List<List<Cell>> sourceRows = new ArrayList<>(); 258 sourceRows.addAll(rows); 259 rows.clear(); 260 261 for (JsonObject union : unionList) { 262 List<List<Cell>> tempRows = new ArrayList<>(); 263 tempRows.addAll(sourceRows); 264 executeSelect(vd, union, b, tempRows); 265 rows.addAll(tempRows); 266 } 267 } 268 269 private List<List<Cell>> cloneRows(List<List<Cell>> rows) { 270 List<List<Cell>> list = new ArrayList<>(); 271 for (List<Cell> row : rows) { 272 list.add(cloneRow(row)); 273 } 274 return list; 275 } 276 277 private List<Cell> cloneRow(List<Cell> cells) { 278 List<Cell> list = new ArrayList<>(); 279 for (Cell cell : cells) { 280 list.add(cell.copy()); 281 } 282 return list; 283 } 284 285 private List<Base> executeForEach(JsonObject vd, JsonObject focus, Base b) { 286 ExpressionNode n = (ExpressionNode) focus.getUserData("forEach"); 287 List<Base> result = new ArrayList<>(); 288 result.addAll(fpe.evaluate(vd, b, n)); 289 return result; 290 } 291 292 private List<Base> executeForEachOrNull(JsonObject vd, JsonObject focus, Base b) { 293 ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull"); 294 List<Base> result = new ArrayList<>(); 295 result.addAll(fpe.evaluate(vd, b, n)); 296 return result; 297 } 298 299 private void executeColumn(JsonObject vd, JsonObject column, Base b, List<List<Cell>> rows) { 300 ExpressionNode n = (ExpressionNode) column.getUserData("path"); 301 List<Base> bl2 = new ArrayList<>(); 302 if (b != null) { 303 bl2.addAll(fpe.evaluate(vd, b, n)); 304 } 305 Column col = (Column) column.getUserData("column"); 306 if (col == null) { 307 log.error("Error"); 308 } else { 309 for (List<Cell> row : rows) { 310 Cell c = cell(row, col.getName()); 311 if (c == null) { 312 c = new Cell(col); 313 row.add(c); 314 } 315 if (!bl2.isEmpty()) { 316 if (bl2.size() + c.getValues().size() > 1) { 317 // this is a problem if collection != true or if the storage can't deal with it 318 // though this should've been picked up before now - but there are circumstances where it wouldn't be 319 if (!c.getColumn().isColl()) { 320 throw new FHIRException("The column "+c.getColumn().getName()+" is not allowed multiple values, but at least one row has multiple values"); 321 } 322 } 323 for (Base b2 : bl2) { 324 c.getValues().add(genValue(c.getColumn(), b2)); 325 } 326 } 327 } 328 } 329 } 330 331 332 private Value genValue(Column column, Base b) { 333 if (column.getKind() == null) { 334 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type (null) for column "+column.getName()); // can't happen 335 } 336 switch (column.getKind()) { 337 case Binary: 338 if (b instanceof Base64BinaryType) { 339 Base64BinaryType bb = (Base64BinaryType) b; 340 return Value.makeBinary(bb.primitiveValue(), bb.getValue()); 341 } else if (b.isBooleanPrimitive()) { // ElementModel 342 return Value.makeBinary(b.primitiveValue(), Base64.decodeBase64(b.primitiveValue())); 343 } else { 344 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a binary column for column "+column.getName()); 345 } 346 case Boolean: 347 if (b instanceof BooleanType) { 348 BooleanType bb = (BooleanType) b; 349 return Value.makeBoolean(bb.primitiveValue(), bb.booleanValue()); 350 } else if (b.isBooleanPrimitive()) { // ElementModel 351 return Value.makeBoolean(b.primitiveValue(), "true".equals(b.primitiveValue())); 352 } else { 353 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a boolean column for column "+column.getName()); 354 } 355 case Complex: 356 if (b.isPrimitive()) { 357 throw new FHIRException("Attempt to add a primitive type "+b.fhirType()+" to a complex column for column "+column.getName()); 358 } else { 359 return Value.makeComplex(b); 360 } 361 case DateTime: 362 if (b instanceof BaseDateTimeType) { 363 BaseDateTimeType d = (BaseDateTimeType) b; 364 return Value.makeDate(d.primitiveValue(), d.getValue()); 365 } else if (b.isPrimitive() && b.isDateTime()) { // ElementModel 366 return Value.makeDate(b.primitiveValue(), b.dateTimeValue().getValue()); 367 } else { 368 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 369 } 370 case Decimal: 371 if (b instanceof DecimalType) { 372 DecimalType d = (DecimalType) b; 373 return Value.makeDecimal(d.primitiveValue(), d.getValue()); 374 } else if (b.isPrimitive()) { // ElementModel 375 return Value.makeDecimal(b.primitiveValue(), new BigDecimal(b.primitiveValue())); 376 } else { 377 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 378 } 379 case Integer: 380 if (b instanceof IntegerType) { 381 IntegerType i = (IntegerType) b; 382 return Value.makeInteger(i.primitiveValue(), i.getValue()); 383 } else if (b.isPrimitive()) { // ElementModel 384 return Value.makeInteger(b.primitiveValue(), Integer.valueOf(b.primitiveValue())); 385 } else { 386 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName()); 387 } 388 case String: 389 if (b.isPrimitive()) { 390 return Value.makeString(b.primitiveValue()); 391 } else { 392 throw new FHIRException("Attempt to add a complex type "+b.fhirType()+" to a string column for column "+column.getName()); 393 } 394 case Time: 395 if (b.fhirType().equals("time")) { 396 return Value.makeString(b.primitiveValue()); 397 } else { 398 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a time column for column "+column.getName()); 399 } 400 default: 401 throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type for column "+column.getName()); 402 } 403 } 404 405 private Column column(String columnName, List<Column> columns) { 406 for (Column t : columns) { 407 if (t.getName().equalsIgnoreCase(columnName)) { 408 return t; 409 } 410 } 411 return null; 412 } 413 414 private Cell cell(List<Cell> cells, String columnName) { 415 for (Cell t : cells) { 416 if (t.getColumn().getName().equalsIgnoreCase(columnName)) { 417 return t; 418 } 419 } 420 return null; 421 } 422 423 @Override 424 public List<Base> resolveConstant(FHIRPathEngine engine, Object appContext, String name, boolean beforeContext, boolean explicitConstant) throws PathEngineException { 425 List<Base> list = new ArrayList<Base>(); 426 if (explicitConstant) { 427 JsonObject vd = (JsonObject) appContext; 428 JsonObject constant = findConstant(vd, name); 429 if (constant != null) { 430 Base b = (Base) constant.getUserData("value"); 431 if (b != null) { 432 list.add(b); 433 } 434 } 435 } 436 return list; 437 } 438 439 @Override 440 public TypeDetails resolveConstantType(FHIRPathEngine engine, Object appContext, String name, boolean explicitConstant) throws PathEngineException { 441 if (explicitConstant) { 442 JsonObject vd = (JsonObject) appContext; 443 JsonObject constant = findConstant(vd, name.substring(1)); 444 if (constant != null) { 445 Base b = (Base) constant.getUserData("value"); 446 if (b != null) { 447 return new TypeDetails(CollectionStatus.SINGLETON, b.fhirType()); 448 } 449 } 450 } 451 return null; 452 } 453 454 private JsonObject findConstant(JsonObject vd, String name) { 455 for (JsonObject o : vd.getJsonObjects("constant")) { 456 if (name.equals(o.asString("name"))) { 457 return o; 458 } 459 } 460 return null; 461 } 462 @Override 463 public boolean log(String argument, List<Base> focus) { 464 throw new Error("Not implemented yet: log"); 465 } 466 467 @Override 468 public FunctionDetails resolveFunction(FHIRPathEngine engine, String functionName) { 469 switch (functionName) { 470 case "getResourceKey" : return new FunctionDetails("Unique Key for resource", 0, 0); 471 case "getReferenceKey" : return new FunctionDetails("Unique Key for resource that is the target of the reference", 0, 1); 472 default: return null; 473 } 474 } 475 @Override 476 public TypeDetails checkFunction(FHIRPathEngine engine, Object appContext, String functionName, TypeDetails focus, List<TypeDetails> parameters) throws PathEngineException { 477 switch (functionName) { 478 case "getResourceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string"); 479 case "getReferenceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string"); 480 default: throw new Error("Not known: "+functionName); 481 } 482 } 483 484 @Override 485 public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) { 486 switch (functionName) { 487 case "getResourceKey" : return executeResourceKey(focus); 488 case "getReferenceKey" : return executeReferenceKey(null, focus, parameters); 489 default: throw new Error("Not known: "+functionName); 490 } 491 } 492 493 private List<Base> executeResourceKey(List<Base> focus) { 494 List<Base> base = new ArrayList<Base>(); 495 if (focus.size() == 1) { 496 Base res = focus.get(0); 497 if (!res.hasUserData("Storage.key")) { 498 String key = storage.getKeyForSourceResource(res); 499 if (key == null) { 500 throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase()); 501 } else { 502 res.setUserData("Storage.key", key); 503 } 504 } 505 base.add(new StringType(res.getUserString("Storage.key"))); 506 } 507 return base; 508 } 509 510 private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) { 511 String rt = null; 512 if (parameters.size() > 0) { 513 rt = parameters.get(0).get(0).primitiveValue(); 514 if (rt.startsWith("FHIR.")) { 515 rt = rt.substring(5); 516 } 517 } 518 List<Base> base = new ArrayList<Base>(); 519 if (focus.size() == 1) { 520 Base res = focus.get(0); 521 String ref = null; 522 if (res.fhirType().equals("Reference")) { 523 ref = getRef(res); 524 } else if (res.isPrimitive()) { 525 ref = res.primitiveValue(); 526 } else { 527 throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType()); 528 } 529 if (ref != null) { 530 Base target = provider.resolveReference(rootResource, ref, rt); 531 if (target != null) { 532 if (!res.hasUserData("Storage.key")) { 533 String key = storage.getKeyForTargetResource(target); 534 if (key == null) { 535 throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase()); 536 } else { 537 res.setUserData("Storage.key", key); 538 } 539 } 540 base.add(new StringType(res.getUserString("Storage.key"))); 541 } 542 } 543 } 544 return base; 545 } 546 547 private String getRef(Base res) { 548 Property prop = res.getChildByName("reference"); 549 if (prop != null && prop.getValues().size() == 1) { 550 return prop.getValues().get(0).primitiveValue(); 551 } 552 return null; 553 } 554 555 @Override 556 public Base resolveReference(FHIRPathEngine engine, Object appContext, String url, Base refContext) throws FHIRException { 557 throw new Error("Not implemented yet: resolveReference"); 558 } 559 560 @Override 561 public boolean conformsToProfile(FHIRPathEngine engine, Object appContext, Base item, String url) throws FHIRException { 562 throw new Error("Not implemented yet: conformsToProfile"); 563 } 564 565 @Override 566 public ValueSet resolveValueSet(FHIRPathEngine engine, Object appContext, String url) { 567 throw new Error("Not implemented yet: resolveValueSet"); 568 } 569 @Override 570 public boolean paramIsType(String name, int index) { 571 return "getReferenceKey".equals(name); 572 } 573 public List<ValidationMessage> getIssues() { 574 return issues; 575 } 576 577 578}