Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.provider;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.RuntimeResourceDefinition;
024import ca.uhn.fhir.i18n.Msg;
025import ca.uhn.fhir.jpa.model.util.JpaConstants;
026import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
027import ca.uhn.fhir.jpa.term.UploadStatistics;
028import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
029import ca.uhn.fhir.jpa.term.custom.ConceptHandler;
030import ca.uhn.fhir.jpa.term.custom.HierarchyHandler;
031import ca.uhn.fhir.jpa.term.custom.PropertyHandler;
032import ca.uhn.fhir.rest.annotation.Operation;
033import ca.uhn.fhir.rest.annotation.OperationParam;
034import ca.uhn.fhir.rest.api.server.RequestDetails;
035import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
036import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
037import ca.uhn.fhir.util.AttachmentUtil;
038import ca.uhn.fhir.util.ParametersUtil;
039import ca.uhn.fhir.util.ValidateUtil;
040import com.google.common.base.Charsets;
041import com.google.common.collect.ArrayListMultimap;
042import com.google.common.collect.Multimap;
043import jakarta.annotation.Nonnull;
044import jakarta.servlet.http.HttpServletRequest;
045import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_30_40;
046import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
047import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_40;
048import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
049import org.hl7.fhir.instance.model.api.IBaseParameters;
050import org.hl7.fhir.instance.model.api.IBaseResource;
051import org.hl7.fhir.instance.model.api.ICompositeType;
052import org.hl7.fhir.instance.model.api.IPrimitiveType;
053import org.hl7.fhir.r4.model.CodeSystem;
054import org.springframework.beans.factory.annotation.Autowired;
055
056import java.io.File;
057import java.io.FileInputStream;
058import java.io.FileNotFoundException;
059import java.io.InputStream;
060import java.util.ArrayList;
061import java.util.LinkedHashMap;
062import java.util.List;
063import java.util.Map;
064
065import static org.apache.commons.lang3.StringUtils.isBlank;
066import static org.apache.commons.lang3.StringUtils.isNotBlank;
067import static org.apache.commons.lang3.StringUtils.trim;
068
069public class TerminologyUploaderProvider extends BaseJpaProvider {
070
071        public static final String PARAM_FILE = "file";
072        public static final String PARAM_CODESYSTEM = "codeSystem";
073        public static final String PARAM_SYSTEM = "system";
074        private static final String RESP_PARAM_CONCEPT_COUNT = "conceptCount";
075        private static final String RESP_PARAM_TARGET = "target";
076        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProvider.class);
077        private static final String RESP_PARAM_SUCCESS = "success";
078
079        @Autowired
080        private ITermLoaderSvc myTerminologyLoaderSvc;
081
082        /**
083         * Constructor
084         */
085        public TerminologyUploaderProvider() {
086                this(null, null);
087        }
088
089        /**
090         * Constructor
091         */
092        public TerminologyUploaderProvider(FhirContext theContext, ITermLoaderSvc theTerminologyLoaderSvc) {
093                setContext(theContext);
094                myTerminologyLoaderSvc = theTerminologyLoaderSvc;
095        }
096
097        /**
098         * <code>
099         * $upload-external-codesystem
100         * </code>
101         */
102        @Operation(
103                        typeName = "CodeSystem",
104                        name = JpaConstants.OPERATION_UPLOAD_EXTERNAL_CODE_SYSTEM,
105                        idempotent = false,
106                        returnParameters = {
107                                //              @OperationParam(name = "conceptCount", type = IntegerType.class, min = 1)
108                        })
109        public IBaseParameters uploadSnapshot(
110                        HttpServletRequest theServletRequest,
111                        @OperationParam(name = PARAM_SYSTEM, min = 1, typeName = "uri") IPrimitiveType<String> theCodeSystemUrl,
112                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
113                                        List<ICompositeType> theFiles,
114                        RequestDetails theRequestDetails) {
115
116                startRequest(theServletRequest);
117
118                if (theCodeSystemUrl == null || isBlank(theCodeSystemUrl.getValueAsString())) {
119                        throw new InvalidRequestException(Msg.code(1137) + "Missing mandatory parameter: " + PARAM_SYSTEM);
120                }
121
122                if (theFiles == null || theFiles.size() == 0) {
123                        throw new InvalidRequestException(
124                                        Msg.code(1138) + "No '" + PARAM_FILE + "' parameter, or package had no data");
125                }
126                for (ICompositeType next : theFiles) {
127                        ValidateUtil.isTrueOrThrowInvalidRequest(
128                                        getContext().getElementDefinition(next.getClass()).getName().equals("Attachment"),
129                                        "Package must be of type Attachment");
130                }
131
132                try {
133                        List<ITermLoaderSvc.FileDescriptor> localFiles = convertAttachmentsToFileDescriptors(theFiles);
134
135                        String codeSystemUrl = theCodeSystemUrl.getValue();
136                        codeSystemUrl = trim(codeSystemUrl);
137
138                        UploadStatistics stats;
139                        switch (codeSystemUrl) {
140                                case ITermLoaderSvc.ICD10_URI:
141                                        stats = myTerminologyLoaderSvc.loadIcd10(localFiles, theRequestDetails);
142                                        break;
143                                case ITermLoaderSvc.ICD10CM_URI:
144                                        stats = myTerminologyLoaderSvc.loadIcd10cm(localFiles, theRequestDetails);
145                                        break;
146                                case ITermLoaderSvc.IMGTHLA_URI:
147                                        stats = myTerminologyLoaderSvc.loadImgthla(localFiles, theRequestDetails);
148                                        break;
149                                case ITermLoaderSvc.LOINC_URI:
150                                        stats = myTerminologyLoaderSvc.loadLoinc(localFiles, theRequestDetails);
151                                        break;
152                                case ITermLoaderSvc.SCT_URI:
153                                        stats = myTerminologyLoaderSvc.loadSnomedCt(localFiles, theRequestDetails);
154                                        break;
155                                default:
156                                        stats = myTerminologyLoaderSvc.loadCustom(codeSystemUrl, localFiles, theRequestDetails);
157                                        break;
158                        }
159
160                        IBaseParameters retVal = ParametersUtil.newInstance(getContext());
161                        ParametersUtil.addParameterToParametersBoolean(getContext(), retVal, RESP_PARAM_SUCCESS, true);
162                        ParametersUtil.addParameterToParametersInteger(
163                                        getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount());
164                        ParametersUtil.addParameterToParametersReference(
165                                        getContext(), retVal, RESP_PARAM_TARGET, stats.getTarget().getValue());
166
167                        return retVal;
168                } finally {
169                        endRequest(theServletRequest);
170                }
171        }
172
173        /**
174         * <code>
175         * $apply-codesystem-delta-add
176         * </code>
177         */
178        @Operation(
179                        typeName = "CodeSystem",
180                        name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_ADD,
181                        idempotent = false,
182                        returnParameters = {})
183        public IBaseParameters uploadDeltaAdd(
184                        HttpServletRequest theServletRequest,
185                        @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
186                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
187                                        List<ICompositeType> theFiles,
188                        @OperationParam(
189                                                        name = PARAM_CODESYSTEM,
190                                                        min = 0,
191                                                        max = OperationParam.MAX_UNLIMITED,
192                                                        typeName = "CodeSystem")
193                                        List<IBaseResource> theCodeSystems,
194                        RequestDetails theRequestDetails) {
195
196                startRequest(theServletRequest);
197                try {
198                        validateHaveSystem(theSystem);
199                        validateHaveFiles(theFiles, theCodeSystems);
200
201                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
202                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
203                        UploadStatistics outcome =
204                                        myTerminologyLoaderSvc.loadDeltaAdd(theSystem.getValue(), files, theRequestDetails);
205                        return toDeltaResponse(outcome);
206                } finally {
207                        endRequest(theServletRequest);
208                }
209        }
210
211        /**
212         * <code>
213         * $apply-codesystem-delta-remove
214         * </code>
215         */
216        @Operation(
217                        typeName = "CodeSystem",
218                        name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_REMOVE,
219                        idempotent = false,
220                        returnParameters = {})
221        public IBaseParameters uploadDeltaRemove(
222                        HttpServletRequest theServletRequest,
223                        @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
224                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
225                                        List<ICompositeType> theFiles,
226                        @OperationParam(
227                                                        name = PARAM_CODESYSTEM,
228                                                        min = 0,
229                                                        max = OperationParam.MAX_UNLIMITED,
230                                                        typeName = "CodeSystem")
231                                        List<IBaseResource> theCodeSystems,
232                        RequestDetails theRequestDetails) {
233
234                startRequest(theServletRequest);
235                try {
236                        validateHaveSystem(theSystem);
237                        validateHaveFiles(theFiles, theCodeSystems);
238
239                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
240                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
241                        UploadStatistics outcome =
242                                        myTerminologyLoaderSvc.loadDeltaRemove(theSystem.getValue(), files, theRequestDetails);
243                        return toDeltaResponse(outcome);
244                } finally {
245                        endRequest(theServletRequest);
246                }
247        }
248
249        private void convertCodeSystemsToFileDescriptors(
250                        List<ITermLoaderSvc.FileDescriptor> theFiles, List<IBaseResource> theCodeSystems) {
251                Map<String, String> codes = new LinkedHashMap<>();
252                Map<String, List<CodeSystem.ConceptPropertyComponent>> codeToProperties = new LinkedHashMap<>();
253
254                Multimap<String, String> codeToParentCodes = ArrayListMultimap.create();
255
256                if (theCodeSystems != null) {
257                        for (IBaseResource nextCodeSystemUncast : theCodeSystems) {
258                                CodeSystem nextCodeSystem = canonicalizeCodeSystem(nextCodeSystemUncast);
259                                convertCodeSystemCodesToCsv(
260                                                nextCodeSystem.getConcept(), codes, codeToProperties, null, codeToParentCodes);
261                        }
262                }
263
264                // Create concept file
265                if (codes.size() > 0) {
266                        StringBuilder b = new StringBuilder();
267                        b.append(ConceptHandler.CODE);
268                        b.append(",");
269                        b.append(ConceptHandler.DISPLAY);
270                        b.append("\n");
271                        for (Map.Entry<String, String> nextEntry : codes.entrySet()) {
272                                b.append(csvEscape(nextEntry.getKey()));
273                                b.append(",");
274                                b.append(csvEscape(nextEntry.getValue()));
275                                b.append("\n");
276                        }
277                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
278                        String fileName = TermLoaderSvcImpl.CUSTOM_CONCEPTS_FILE;
279                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
280                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
281                        theFiles.add(fileDescriptor);
282                }
283
284                // Create hierarchy file
285                if (codeToParentCodes.size() > 0) {
286                        StringBuilder b = new StringBuilder();
287                        b.append(HierarchyHandler.CHILD);
288                        b.append(",");
289                        b.append(HierarchyHandler.PARENT);
290                        b.append("\n");
291                        for (Map.Entry<String, String> nextEntry : codeToParentCodes.entries()) {
292                                b.append(csvEscape(nextEntry.getKey()));
293                                b.append(",");
294                                b.append(csvEscape(nextEntry.getValue()));
295                                b.append("\n");
296                        }
297                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
298                        String fileName = TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE;
299                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
300                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
301                        theFiles.add(fileDescriptor);
302                }
303                // Create codeToProperties file
304                if (codeToProperties.size() > 0) {
305                        StringBuilder b = new StringBuilder();
306                        b.append(PropertyHandler.CODE);
307                        b.append(",");
308                        b.append(PropertyHandler.KEY);
309                        b.append(",");
310                        b.append(PropertyHandler.VALUE);
311                        b.append(",");
312                        b.append(PropertyHandler.TYPE);
313                        b.append("\n");
314
315                        for (Map.Entry<String, List<CodeSystem.ConceptPropertyComponent>> nextEntry : codeToProperties.entrySet()) {
316                                for (CodeSystem.ConceptPropertyComponent propertyComponent : nextEntry.getValue()) {
317                                        b.append(csvEscape(nextEntry.getKey()));
318                                        b.append(",");
319                                        b.append(csvEscape(propertyComponent.getCode()));
320                                        b.append(",");
321                                        // TODO: check this for different types, other types should be added once
322                                        // TermConceptPropertyTypeEnum contain different types
323                                        b.append(csvEscape(propertyComponent.getValueStringType().getValue()));
324                                        b.append(",");
325                                        b.append(csvEscape(propertyComponent.getValue().primitiveValue()));
326                                        b.append("\n");
327                                }
328                        }
329                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
330                        String fileName = TermLoaderSvcImpl.CUSTOM_PROPERTIES_FILE;
331                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
332                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
333                        theFiles.add(fileDescriptor);
334                }
335        }
336
337        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
338        @Nonnull
339        CodeSystem canonicalizeCodeSystem(@Nonnull IBaseResource theCodeSystem) {
340                RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(theCodeSystem);
341                ValidateUtil.isTrueOrThrowInvalidRequest(
342                                resourceDef.getName().equals("CodeSystem"), "Resource '%s' is not a CodeSystem", resourceDef.getName());
343
344                CodeSystem nextCodeSystem;
345                switch (getContext().getVersion().getVersion()) {
346                        case DSTU3:
347                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_30_40.convertResource(
348                                                (org.hl7.fhir.dstu3.model.CodeSystem) theCodeSystem, new BaseAdvisor_30_40(false));
349                                break;
350                        case R5:
351                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_40_50.convertResource(
352                                                (org.hl7.fhir.r5.model.CodeSystem) theCodeSystem, new BaseAdvisor_40_50(false));
353                                break;
354                        default:
355                                nextCodeSystem = (CodeSystem) theCodeSystem;
356                }
357                return nextCodeSystem;
358        }
359
360        private void convertCodeSystemCodesToCsv(
361                        List<CodeSystem.ConceptDefinitionComponent> theConcept,
362                        Map<String, String> theCodes,
363                        Map<String, List<CodeSystem.ConceptPropertyComponent>> theProperties,
364                        String theParentCode,
365                        Multimap<String, String> theCodeToParentCodes) {
366                for (CodeSystem.ConceptDefinitionComponent nextConcept : theConcept) {
367                        if (isNotBlank(nextConcept.getCode())) {
368                                theCodes.put(nextConcept.getCode(), nextConcept.getDisplay());
369                                if (isNotBlank(theParentCode)) {
370                                        theCodeToParentCodes.put(nextConcept.getCode(), theParentCode);
371                                }
372                                if (nextConcept.getProperty() != null) {
373                                        theProperties.put(nextConcept.getCode(), nextConcept.getProperty());
374                                }
375                                convertCodeSystemCodesToCsv(
376                                                nextConcept.getConcept(), theCodes, theProperties, nextConcept.getCode(), theCodeToParentCodes);
377                        }
378                }
379        }
380
381        private void validateHaveSystem(IPrimitiveType<String> theSystem) {
382                if (theSystem == null || isBlank(theSystem.getValueAsString())) {
383                        throw new InvalidRequestException(Msg.code(1139) + "Missing mandatory parameter: " + PARAM_SYSTEM);
384                }
385        }
386
387        private void validateHaveFiles(List<ICompositeType> theFiles, List<IBaseResource> theCodeSystems) {
388                if (theFiles != null) {
389                        for (ICompositeType nextFile : theFiles) {
390                                if (!nextFile.isEmpty()) {
391                                        return;
392                                }
393                        }
394                }
395                if (theCodeSystems != null) {
396                        for (IBaseResource next : theCodeSystems) {
397                                if (!next.isEmpty()) {
398                                        return;
399                                }
400                        }
401                }
402                throw new InvalidRequestException(Msg.code(1140) + "Missing mandatory parameter: " + PARAM_FILE);
403        }
404
405        @Nonnull
406        private List<ITermLoaderSvc.FileDescriptor> convertAttachmentsToFileDescriptors(
407                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
408                                        List<ICompositeType> theFiles) {
409                List<ITermLoaderSvc.FileDescriptor> files = new ArrayList<>();
410                if (theFiles != null) {
411                        for (ICompositeType next : theFiles) {
412
413                                String nextUrl =
414                                                AttachmentUtil.getOrCreateUrl(getContext(), next).getValue();
415                                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value");
416
417                                byte[] nextData;
418                                if (nextUrl.startsWith("localfile:")) {
419                                        String nextLocalFile = nextUrl.substring("localfile:".length());
420
421                                        if (isNotBlank(nextLocalFile)) {
422                                                ourLog.info("Reading in local file: {}", nextLocalFile);
423                                                File nextFile = new File(nextLocalFile);
424                                                if (!nextFile.exists() || !nextFile.isFile()) {
425                                                        throw new InvalidRequestException(Msg.code(1141) + "Unknown file: " + nextFile.getName());
426                                                }
427                                                files.add(new FileBackedFileDescriptor(nextFile));
428                                        }
429
430                                } else {
431                                        nextData =
432                                                        AttachmentUtil.getOrCreateData(getContext(), next).getValue();
433                                        ValidateUtil.isTrueOrThrowInvalidRequest(
434                                                        nextData != null && nextData.length > 0, "Missing Attachment.data value");
435                                        files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData));
436                                }
437                        }
438                }
439                return files;
440        }
441
442        private IBaseParameters toDeltaResponse(UploadStatistics theOutcome) {
443                IBaseParameters retVal = ParametersUtil.newInstance(getContext());
444                ParametersUtil.addParameterToParametersInteger(
445                                getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount());
446                ParametersUtil.addParameterToParametersReference(
447                                getContext(), retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue());
448                return retVal;
449        }
450
451        public void setTerminologyLoaderSvc(ITermLoaderSvc theTermLoaderSvc) {
452                myTerminologyLoaderSvc = theTermLoaderSvc;
453        }
454
455        public static class FileBackedFileDescriptor implements ITermLoaderSvc.FileDescriptor {
456                private final File myNextFile;
457
458                public FileBackedFileDescriptor(File theNextFile) {
459                        myNextFile = theNextFile;
460                }
461
462                @Override
463                public String getFilename() {
464                        return myNextFile.getAbsolutePath();
465                }
466
467                @Override
468                public InputStream getInputStream() {
469                        try {
470                                return new FileInputStream(myNextFile);
471                        } catch (FileNotFoundException theE) {
472                                throw new InternalErrorException(Msg.code(1142) + theE);
473                        }
474                }
475        }
476
477        private static String csvEscape(String theValue) {
478                if (theValue == null) {
479                        return "";
480                }
481                return '"' + theValue.replace("\"", "\"\"").replace("\n", "\\n").replace("\r", "") + '"';
482        }
483}