
001package ca.uhn.fhir.jpa.provider; 002 003/* 004 * #%L 005 * HAPI FHIR JPA Server 006 * %% 007 * Copyright (C) 2014 - 2022 Smile CDR, Inc. 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.context.FhirContext; 025import ca.uhn.fhir.context.RuntimeResourceDefinition; 026import ca.uhn.fhir.jpa.model.util.JpaConstants; 027import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl; 028import ca.uhn.fhir.jpa.term.UploadStatistics; 029import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc; 030import ca.uhn.fhir.jpa.term.custom.ConceptHandler; 031import ca.uhn.fhir.jpa.term.custom.HierarchyHandler; 032import ca.uhn.fhir.jpa.term.custom.PropertyHandler; 033import ca.uhn.fhir.rest.annotation.Operation; 034import ca.uhn.fhir.rest.annotation.OperationParam; 035import ca.uhn.fhir.rest.api.server.RequestDetails; 036import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 037import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 038import ca.uhn.fhir.util.AttachmentUtil; 039import ca.uhn.fhir.util.ParametersUtil; 040import ca.uhn.fhir.util.ValidateUtil; 041import com.google.common.base.Charsets; 042import com.google.common.collect.ArrayListMultimap; 043import com.google.common.collect.Multimap; 044import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_30_40; 045import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50; 046import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_40; 047import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50; 048import org.hl7.fhir.instance.model.api.IBaseParameters; 049import org.hl7.fhir.instance.model.api.IBaseResource; 050import org.hl7.fhir.instance.model.api.ICompositeType; 051import org.hl7.fhir.instance.model.api.IPrimitiveType; 052import org.hl7.fhir.r4.model.CodeSystem; 053import org.springframework.beans.factory.annotation.Autowired; 054 055import javax.annotation.Nonnull; 056import javax.servlet.http.HttpServletRequest; 057import java.io.File; 058import java.io.FileInputStream; 059import java.io.FileNotFoundException; 060import java.io.InputStream; 061import java.util.ArrayList; 062import java.util.LinkedHashMap; 063import java.util.List; 064import java.util.Map; 065 066import static org.apache.commons.lang3.StringUtils.*; 067 068public class TerminologyUploaderProvider extends BaseJpaProvider { 069 070 public static final String PARAM_FILE = "file"; 071 public static final String PARAM_CODESYSTEM = "codeSystem"; 072 public static final String PARAM_SYSTEM = "system"; 073 private static final String RESP_PARAM_CONCEPT_COUNT = "conceptCount"; 074 private static final String RESP_PARAM_TARGET = "target"; 075 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProvider.class); 076 private static final String RESP_PARAM_SUCCESS = "success"; 077 078 @Autowired 079 private ITermLoaderSvc myTerminologyLoaderSvc; 080 081 /** 082 * Constructor 083 */ 084 public TerminologyUploaderProvider() { 085 this(null, null); 086 } 087 088 /** 089 * Constructor 090 */ 091 public TerminologyUploaderProvider(FhirContext theContext, ITermLoaderSvc theTerminologyLoaderSvc) { 092 setContext(theContext); 093 myTerminologyLoaderSvc = theTerminologyLoaderSvc; 094 } 095 096 /** 097 * <code> 098 * $upload-external-codesystem 099 * </code> 100 */ 101 @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_UPLOAD_EXTERNAL_CODE_SYSTEM, idempotent = false, returnParameters = { 102// @OperationParam(name = "conceptCount", type = IntegerType.class, min = 1) 103 }) 104 public IBaseParameters uploadSnapshot( 105 HttpServletRequest theServletRequest, 106 @OperationParam(name = PARAM_SYSTEM, min = 1, typeName = "uri") IPrimitiveType<String> theCodeSystemUrl, 107 @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles, 108 RequestDetails theRequestDetails 109 ) { 110 111 startRequest(theServletRequest); 112 113 if (theCodeSystemUrl == null || isBlank(theCodeSystemUrl.getValueAsString())) { 114 throw new InvalidRequestException(Msg.code(1137) + "Missing mandatory parameter: " + PARAM_SYSTEM); 115 } 116 117 if (theFiles == null || theFiles.size() == 0) { 118 throw new InvalidRequestException(Msg.code(1138) + "No '" + PARAM_FILE + "' parameter, or package had no data"); 119 } 120 for (ICompositeType next : theFiles) { 121 ValidateUtil.isTrueOrThrowInvalidRequest(getContext().getElementDefinition(next.getClass()).getName().equals("Attachment"), "Package must be of type Attachment"); 122 } 123 124 try { 125 List<ITermLoaderSvc.FileDescriptor> localFiles = convertAttachmentsToFileDescriptors(theFiles); 126 127 String codeSystemUrl = theCodeSystemUrl.getValue(); 128 codeSystemUrl = trim(codeSystemUrl); 129 130 UploadStatistics stats; 131 switch (codeSystemUrl) { 132 case ITermLoaderSvc.ICD10CM_URI: 133 stats = myTerminologyLoaderSvc.loadIcd10cm(localFiles, theRequestDetails); 134 break; 135 case ITermLoaderSvc.IMGTHLA_URI: 136 stats = myTerminologyLoaderSvc.loadImgthla(localFiles, theRequestDetails); 137 break; 138 case ITermLoaderSvc.LOINC_URI: 139 stats = myTerminologyLoaderSvc.loadLoinc(localFiles, theRequestDetails); 140 break; 141 case ITermLoaderSvc.SCT_URI: 142 stats = myTerminologyLoaderSvc.loadSnomedCt(localFiles, theRequestDetails); 143 break; 144 default: 145 stats = myTerminologyLoaderSvc.loadCustom(codeSystemUrl, localFiles, theRequestDetails); 146 break; 147 } 148 149 IBaseParameters retVal = ParametersUtil.newInstance(getContext()); 150 ParametersUtil.addParameterToParametersBoolean(getContext(), retVal, RESP_PARAM_SUCCESS, true); 151 ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount()); 152 ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, stats.getTarget().getValue()); 153 154 return retVal; 155 } finally { 156 endRequest(theServletRequest); 157 } 158 } 159 160 /** 161 * <code> 162 * $apply-codesystem-delta-add 163 * </code> 164 */ 165 @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_ADD, idempotent = false, returnParameters = { 166 }) 167 public IBaseParameters uploadDeltaAdd( 168 HttpServletRequest theServletRequest, 169 @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem, 170 @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles, 171 @OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems, 172 RequestDetails theRequestDetails 173 ) { 174 175 startRequest(theServletRequest); 176 try { 177 validateHaveSystem(theSystem); 178 validateHaveFiles(theFiles, theCodeSystems); 179 180 List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles); 181 convertCodeSystemsToFileDescriptors(files, theCodeSystems); 182 UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaAdd(theSystem.getValue(), files, theRequestDetails); 183 return toDeltaResponse(outcome); 184 } finally { 185 endRequest(theServletRequest); 186 } 187 188 } 189 190 191 /** 192 * <code> 193 * $apply-codesystem-delta-remove 194 * </code> 195 */ 196 @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_REMOVE, idempotent = false, returnParameters = { 197 }) 198 public IBaseParameters uploadDeltaRemove( 199 HttpServletRequest theServletRequest, 200 @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem, 201 @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles, 202 @OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems, 203 RequestDetails theRequestDetails 204 ) { 205 206 startRequest(theServletRequest); 207 try { 208 validateHaveSystem(theSystem); 209 validateHaveFiles(theFiles, theCodeSystems); 210 211 List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles); 212 convertCodeSystemsToFileDescriptors(files, theCodeSystems); 213 UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaRemove(theSystem.getValue(), files, theRequestDetails); 214 return toDeltaResponse(outcome); 215 } finally { 216 endRequest(theServletRequest); 217 } 218 219 } 220 221 private void convertCodeSystemsToFileDescriptors(List<ITermLoaderSvc.FileDescriptor> theFiles, List<IBaseResource> theCodeSystems) { 222 Map<String, String> codes = new LinkedHashMap<>(); 223 Map<String, List<CodeSystem.ConceptPropertyComponent>> codeToProperties = new LinkedHashMap<>(); 224 225 Multimap<String, String> codeToParentCodes = ArrayListMultimap.create(); 226 227 if (theCodeSystems != null) { 228 for (IBaseResource nextCodeSystemUncast : theCodeSystems) { 229 CodeSystem nextCodeSystem = canonicalizeCodeSystem(nextCodeSystemUncast); 230 convertCodeSystemCodesToCsv(nextCodeSystem.getConcept(), codes, codeToProperties, null, codeToParentCodes); 231 } 232 } 233 234 // Create concept file 235 if (codes.size() > 0) { 236 StringBuilder b = new StringBuilder(); 237 b.append(ConceptHandler.CODE); 238 b.append(","); 239 b.append(ConceptHandler.DISPLAY); 240 b.append("\n"); 241 for (Map.Entry<String, String> nextEntry : codes.entrySet()) { 242 b.append(csvEscape(nextEntry.getKey())); 243 b.append(","); 244 b.append(csvEscape(nextEntry.getValue())); 245 b.append("\n"); 246 } 247 byte[] bytes = b.toString().getBytes(Charsets.UTF_8); 248 String fileName = TermLoaderSvcImpl.CUSTOM_CONCEPTS_FILE; 249 ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes); 250 theFiles.add(fileDescriptor); 251 } 252 253 // Create hierarchy file 254 if (codeToParentCodes.size() > 0) { 255 StringBuilder b = new StringBuilder(); 256 b.append(HierarchyHandler.CHILD); 257 b.append(","); 258 b.append(HierarchyHandler.PARENT); 259 b.append("\n"); 260 for (Map.Entry<String, String> nextEntry : codeToParentCodes.entries()) { 261 b.append(csvEscape(nextEntry.getKey())); 262 b.append(","); 263 b.append(csvEscape(nextEntry.getValue())); 264 b.append("\n"); 265 } 266 byte[] bytes = b.toString().getBytes(Charsets.UTF_8); 267 String fileName = TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE; 268 ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes); 269 theFiles.add(fileDescriptor); 270 } 271 // Create codeToProperties file 272 if (codeToProperties.size() > 0) { 273 StringBuilder b = new StringBuilder(); 274 b.append(PropertyHandler.CODE); 275 b.append(","); 276 b.append(PropertyHandler.KEY); 277 b.append(","); 278 b.append(PropertyHandler.VALUE); 279 b.append(","); 280 b.append(PropertyHandler.TYPE); 281 b.append("\n"); 282 283 for (Map.Entry<String, List<CodeSystem.ConceptPropertyComponent>> nextEntry : codeToProperties.entrySet()) { 284 for (CodeSystem.ConceptPropertyComponent propertyComponent : nextEntry.getValue()) { 285 b.append(csvEscape(nextEntry.getKey())); 286 b.append(","); 287 b.append(csvEscape(propertyComponent.getCode())); 288 b.append(","); 289 //TODO: check this for different types, other types should be added once TermConceptPropertyTypeEnum contain different types 290 b.append(csvEscape(propertyComponent.getValueStringType().getValue())); 291 b.append(","); 292 b.append(csvEscape(propertyComponent.getValue().primitiveValue())); 293 b.append("\n"); 294 } 295 } 296 byte[] bytes = b.toString().getBytes(Charsets.UTF_8); 297 String fileName = TermLoaderSvcImpl.CUSTOM_PROPERTIES_FILE; 298 ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes); 299 theFiles.add(fileDescriptor); 300 } 301 302 } 303 304 @SuppressWarnings("EnumSwitchStatementWhichMissesCases") 305 @Nonnull 306 CodeSystem canonicalizeCodeSystem(@Nonnull IBaseResource theCodeSystem) { 307 RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(theCodeSystem); 308 ValidateUtil.isTrueOrThrowInvalidRequest(resourceDef.getName().equals("CodeSystem"), "Resource '%s' is not a CodeSystem", resourceDef.getName()); 309 310 CodeSystem nextCodeSystem; 311 switch (getContext().getVersion().getVersion()) { 312 case DSTU3: 313 nextCodeSystem = (CodeSystem) VersionConvertorFactory_30_40.convertResource((org.hl7.fhir.dstu3.model.CodeSystem) theCodeSystem, new BaseAdvisor_30_40(false)); 314 break; 315 case R5: 316 nextCodeSystem = (CodeSystem) VersionConvertorFactory_40_50.convertResource((org.hl7.fhir.r5.model.CodeSystem) theCodeSystem, new BaseAdvisor_40_50(false)); 317 break; 318 default: 319 nextCodeSystem = (CodeSystem) theCodeSystem; 320 } 321 return nextCodeSystem; 322 } 323 324 private void convertCodeSystemCodesToCsv(List<CodeSystem.ConceptDefinitionComponent> theConcept, Map<String, String> theCodes, Map<String, List<CodeSystem.ConceptPropertyComponent>> theProperties, String theParentCode, Multimap<String, String> theCodeToParentCodes) { 325 for (CodeSystem.ConceptDefinitionComponent nextConcept : theConcept) { 326 if (isNotBlank(nextConcept.getCode())) { 327 theCodes.put(nextConcept.getCode(), nextConcept.getDisplay()); 328 if (isNotBlank(theParentCode)) { 329 theCodeToParentCodes.put(nextConcept.getCode(), theParentCode); 330 } 331 if (nextConcept.getProperty() != null) { 332 theProperties.put(nextConcept.getCode(), nextConcept.getProperty()); 333 } 334 convertCodeSystemCodesToCsv(nextConcept.getConcept(), theCodes, theProperties, nextConcept.getCode(), theCodeToParentCodes); 335 } 336 } 337 } 338 339 private void validateHaveSystem(IPrimitiveType<String> theSystem) { 340 if (theSystem == null || isBlank(theSystem.getValueAsString())) { 341 throw new InvalidRequestException(Msg.code(1139) + "Missing mandatory parameter: " + PARAM_SYSTEM); 342 } 343 } 344 345 private void validateHaveFiles(List<ICompositeType> theFiles, List<IBaseResource> theCodeSystems) { 346 if (theFiles != null) { 347 for (ICompositeType nextFile : theFiles) { 348 if (!nextFile.isEmpty()) { 349 return; 350 } 351 } 352 } 353 if (theCodeSystems != null) { 354 for (IBaseResource next : theCodeSystems) { 355 if (!next.isEmpty()) { 356 return; 357 } 358 } 359 } 360 throw new InvalidRequestException(Msg.code(1140) + "Missing mandatory parameter: " + PARAM_FILE); 361 } 362 363 @Nonnull 364 private List<ITermLoaderSvc.FileDescriptor> convertAttachmentsToFileDescriptors(@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles) { 365 List<ITermLoaderSvc.FileDescriptor> files = new ArrayList<>(); 366 if (theFiles != null) { 367 for (ICompositeType next : theFiles) { 368 369 String nextUrl = AttachmentUtil.getOrCreateUrl(getContext(), next).getValue(); 370 ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value"); 371 372 byte[] nextData; 373 if (nextUrl.startsWith("localfile:")) { 374 String nextLocalFile = nextUrl.substring("localfile:".length()); 375 376 377 if (isNotBlank(nextLocalFile)) { 378 ourLog.info("Reading in local file: {}", nextLocalFile); 379 File nextFile = new File(nextLocalFile); 380 if (!nextFile.exists() || !nextFile.isFile()) { 381 throw new InvalidRequestException(Msg.code(1141) + "Unknown file: " + nextFile.getName()); 382 } 383 files.add(new FileBackedFileDescriptor(nextFile)); 384 } 385 386 } else { 387 nextData = AttachmentUtil.getOrCreateData(getContext(), next).getValue(); 388 ValidateUtil.isTrueOrThrowInvalidRequest(nextData != null && nextData.length > 0, "Missing Attachment.data value"); 389 files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData)); 390 } 391 } 392 } 393 return files; 394 } 395 396 private IBaseParameters toDeltaResponse(UploadStatistics theOutcome) { 397 IBaseParameters retVal = ParametersUtil.newInstance(getContext()); 398 ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount()); 399 ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue()); 400 return retVal; 401 } 402 403 public static class FileBackedFileDescriptor implements ITermLoaderSvc.FileDescriptor { 404 private final File myNextFile; 405 406 public FileBackedFileDescriptor(File theNextFile) { 407 myNextFile = theNextFile; 408 } 409 410 @Override 411 public String getFilename() { 412 return myNextFile.getAbsolutePath(); 413 } 414 415 @Override 416 public InputStream getInputStream() { 417 try { 418 return new FileInputStream(myNextFile); 419 } catch (FileNotFoundException theE) { 420 throw new InternalErrorException(Msg.code(1142) + theE); 421 } 422 } 423 } 424 425 private static String csvEscape(String theValue) { 426 return '"' + 427 theValue 428 .replace("\"", "\"\"") 429 .replace("\n", "\\n") 430 .replace("\r", "") + 431 '"'; 432 } 433}