View Javadoc
1   package ca.uhn.fhir.jpa.term;
2   
3   import ca.uhn.fhir.context.FhirContext;
4   import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
5   import ca.uhn.fhir.jpa.entity.TermConcept;
6   import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
7   import ca.uhn.fhir.jpa.term.loinc.*;
8   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerConcept;
9   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerDescription;
10  import ca.uhn.fhir.jpa.term.snomedct.SctHandlerRelationship;
11  import ca.uhn.fhir.jpa.util.Counter;
12  import ca.uhn.fhir.rest.api.server.RequestDetails;
13  import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
14  import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
15  import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
16  import com.google.common.annotations.VisibleForTesting;
17  import com.google.common.base.Charsets;
18  import org.apache.commons.csv.CSVFormat;
19  import org.apache.commons.csv.CSVParser;
20  import org.apache.commons.csv.CSVRecord;
21  import org.apache.commons.csv.QuoteMode;
22  import org.apache.commons.io.FileUtils;
23  import org.apache.commons.io.IOUtils;
24  import org.apache.commons.io.input.BOMInputStream;
25  import org.apache.commons.lang3.ObjectUtils;
26  import org.apache.commons.lang3.StringUtils;
27  import org.apache.commons.lang3.Validate;
28  import org.hl7.fhir.instance.model.api.IIdType;
29  import org.hl7.fhir.r4.model.CodeSystem;
30  import org.hl7.fhir.r4.model.ConceptMap;
31  import org.hl7.fhir.r4.model.ValueSet;
32  import org.springframework.beans.factory.annotation.Autowired;
33  
34  import java.io.*;
35  import java.util.*;
36  import java.util.Map.Entry;
37  import java.util.zip.ZipEntry;
38  import java.util.zip.ZipInputStream;
39  
40  import static org.apache.commons.lang3.StringUtils.isNotBlank;
41  
42  /*
43   * #%L
44   * HAPI FHIR JPA Server
45   * %%
46   * Copyright (C) 2014 - 2018 University Health Network
47   * %%
48   * Licensed under the Apache License, Version 2.0 (the "License");
49   * you may not use this file except in compliance with the License.
50   * You may obtain a copy of the License at
51   * 
52   *      http://www.apache.org/licenses/LICENSE-2.0
53   * 
54   * Unless required by applicable law or agreed to in writing, software
55   * distributed under the License is distributed on an "AS IS" BASIS,
56   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
57   * See the License for the specific language governing permissions and
58   * limitations under the License.
59   * #L%
60   */
61  
62  public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
63  	public static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_";
64  	public static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full-en";
65  	public static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
66  	public static final String LOINC_ANSWERLIST_FILE = "AnswerList.csv";
67  	public static final String LOINC_ANSWERLIST_LINK_FILE = "LoincAnswerListLink.csv";
68  	public static final String LOINC_DOCUMENT_ONTOLOGY_FILE = "DocumentOntology.csv";
69  	public static final String LOINC_UPLOAD_PROPERTIES_FILE = "loincupload.properties";
70  	public static final String LOINC_FILE = "Loinc.csv";
71  	public static final String LOINC_HIERARCHY_FILE = "MultiAxialHierarchy.csv";
72  	public static final String LOINC_PART_FILE = "Part.csv";
73  	public static final String LOINC_PART_LINK_FILE = "LoincPartLink.csv";
74  	public static final String LOINC_PART_RELATED_CODE_MAPPING_FILE = "PartRelatedCodeMapping.csv";
75  	public static final String LOINC_RSNA_PLAYBOOK_FILE = "LoincRsnaRadiologyPlaybook.csv";
76  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE = "Top2000CommonLabResultsUs.csv";
77  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE = "Top2000CommonLabResultsSi.csv";
78  	public static final String LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE = "LoincUniversalLabOrdersValueSet.csv";
79  	public static final String LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV = "LoincIeeeMedicalDeviceCodeMappingTable.csv";
80  	public static final String LOINC_IMAGING_DOCUMENT_CODES_FILE = "ImagingDocumentCodes.csv";
81  	public static final String LOINC_GROUP_FILE = "Group.csv";
82  	public static final String LOINC_GROUP_TERMS_FILE = "GroupLoincTerms.csv";
83  	public static final String LOINC_PARENT_GROUP_FILE = "ParentGroup.csv";
84  	private static final int LOG_INCREMENT = 1000;
85  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvcImpl.class);
86  
87  	@Autowired
88  	private IHapiTerminologySvc myTermSvc;
89  	@Autowired(required = false)
90  	private IHapiTerminologySvcDstu3 myTermSvcDstu3;
91  	@Autowired(required = false)
92  	private IHapiTerminologySvcR4 myTermSvcR4;
93  
94  	private void dropCircularRefs(TermConcept theConcept, ArrayList<String> theChain, Map<String, TermConcept> theCode2concept, Counter theCircularCounter) {
95  
96  		theChain.add(theConcept.getCode());
97  		for (Iterator<TermConceptParentChildLink> childIter = theConcept.getChildren().iterator(); childIter.hasNext(); ) {
98  			TermConceptParentChildLink next = childIter.next();
99  			TermConcept nextChild = next.getChild();
100 			if (theChain.contains(nextChild.getCode())) {
101 
102 				StringBuilder b = new StringBuilder();
103 				b.append("Removing circular reference code ");
104 				b.append(nextChild.getCode());
105 				b.append(" from parent ");
106 				b.append(next.getParent().getCode());
107 				b.append(". Chain was: ");
108 				for (String nextInChain : theChain) {
109 					TermConcept nextCode = theCode2concept.get(nextInChain);
110 					b.append(nextCode.getCode());
111 					b.append('[');
112 					b.append(StringUtils.substring(nextCode.getDisplay(), 0, 20).replace("[", "").replace("]", "").trim());
113 					b.append("] ");
114 				}
115 				ourLog.info(b.toString(), theConcept.getCode());
116 				childIter.remove();
117 				nextChild.getParents().remove(next);
118 
119 			} else {
120 				dropCircularRefs(nextChild, theChain, theCode2concept, theCircularCounter);
121 			}
122 		}
123 		theChain.remove(theChain.size() - 1);
124 
125 	}
126 
127 	private void iterateOverZipFile(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode, boolean theIsPartialFilename) {
128 
129 		boolean foundMatch = false;
130 		for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
131 			String nextFilename = nextZipBytes.getFilename();
132 			boolean matches;
133 			if (theIsPartialFilename) {
134 				matches = nextFilename.contains(theFileNamePart);
135 			} else {
136 				matches = nextFilename.endsWith("/" + theFileNamePart) || nextFilename.equals(theFileNamePart);
137 			}
138 			if (matches) {
139 				ourLog.info("Processing file {}", nextFilename);
140 				foundMatch = true;
141 
142 				Reader reader;
143 				CSVParser parsed;
144 				try {
145 					reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
146 
147 					if (ourLog.isTraceEnabled()) {
148 						String contents = IOUtils.toString(reader);
149 						ourLog.info("File contents for: {}\n{}", nextFilename, contents);
150 						reader = new StringReader(contents);
151 					}
152 
153 					CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
154 					if (theQuoteMode != null) {
155 						format = format.withQuote('"').withQuoteMode(theQuoteMode);
156 					}
157 					parsed = new CSVParser(reader, format);
158 					Iterator<CSVRecord> iter = parsed.iterator();
159 					ourLog.debug("Header map: {}", parsed.getHeaderMap());
160 
161 					int count = 0;
162 					int nextLoggedCount = 0;
163 					while (iter.hasNext()) {
164 						CSVRecord nextRecord = iter.next();
165 						if (nextRecord.isConsistent()==false) {
166 							continue;
167 						}
168 						theHandler.accept(nextRecord);
169 						count++;
170 						if (count >= nextLoggedCount) {
171 							ourLog.info(" * Processed {} records in {}", count, nextFilename);
172 							nextLoggedCount += LOG_INCREMENT;
173 						}
174 					}
175 
176 				} catch (IOException e) {
177 					throw new InternalErrorException(e);
178 				}
179 			}
180 
181 		}
182 
183 		if (!foundMatch) {
184 			throw new InvalidRequestException("Did not find file matching " + theFileNamePart);
185 		}
186 
187 	}
188 
189 	@Override
190 	public UploadStatistics loadLoinc(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
191 		LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles);
192 		List<String> mandatoryFilenameFragments = Arrays.asList(
193 			LOINC_FILE,
194 			LOINC_HIERARCHY_FILE,
195 			LOINC_UPLOAD_PROPERTIES_FILE,
196 			LOINC_ANSWERLIST_FILE,
197 			LOINC_ANSWERLIST_LINK_FILE,
198 			LOINC_PART_FILE,
199 			LOINC_PART_LINK_FILE,
200 			LOINC_PART_RELATED_CODE_MAPPING_FILE,
201 			LOINC_DOCUMENT_ONTOLOGY_FILE,
202 			LOINC_RSNA_PLAYBOOK_FILE,
203 			LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE,
204 			LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE,
205 			LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE,
206 			LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV,
207 			LOINC_IMAGING_DOCUMENT_CODES_FILE
208 		);
209 		descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments);
210 
211 		List<String> optionalFilenameFragments = Arrays.asList(
212 		);
213 		descriptors.verifyOptionalFilesExist(optionalFilenameFragments);
214 
215 		ourLog.info("Beginning LOINC processing");
216 
217 		return processLoincFiles(descriptors, theRequestDetails);
218 	}
219 
220 	@Override
221 	public UploadStatistics loadSnomedCt(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
222 		LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles);
223 
224 		List<String> expectedFilenameFragments = Arrays.asList(
225 			SCT_FILE_DESCRIPTION,
226 			SCT_FILE_RELATIONSHIP,
227 			SCT_FILE_CONCEPT);
228 		descriptors.verifyMandatoryFilesExist(expectedFilenameFragments);
229 
230 		ourLog.info("Beginning SNOMED CT processing");
231 
232 		return processSnomedCtFiles(descriptors, theRequestDetails);
233 	}
234 
235 	UploadStatistics processLoincFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
236 		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
237 		final Map<String, TermConcept> code2concept = new HashMap<>();
238 		final List<ValueSet> valueSets = new ArrayList<>();
239 		final List<ConceptMap> conceptMaps = new ArrayList<>();
240 
241 		CodeSystem loincCs;
242 		try {
243 			String loincCsString = IOUtils.toString(BaseHapiTerminologySvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/loinc/loinc.xml"), Charsets.UTF_8);
244 			loincCs = FhirContext.forR4().newXmlParser().parseResource(CodeSystem.class, loincCsString);
245 		} catch (IOException e) {
246 			throw new InternalErrorException("Failed to load loinc.xml", e);
247 		}
248 
249 		Map<String, CodeSystem.PropertyType> propertyNamesToTypes = new HashMap<>();
250 		for (CodeSystem.PropertyComponent nextProperty : loincCs.getProperty()) {
251 			String nextPropertyCode = nextProperty.getCode();
252 			CodeSystem.PropertyType nextPropertyType = nextProperty.getType();
253 			if (isNotBlank(nextPropertyCode)) {
254 				propertyNamesToTypes.put(nextPropertyCode, nextPropertyType);
255 			}
256 		}
257 
258 		IRecordHandler handler;
259 
260 		Properties uploadProperties = new Properties();
261 		for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) {
262 			if (next.getFilename().endsWith("loincupload.properties")) {
263 				try {
264 					try (InputStream inputStream = next.getInputStream()) {
265 						uploadProperties.load(inputStream);
266 					}
267 				} catch (IOException e) {
268 					throw new InternalErrorException("Failed to read loincupload.properties", e);
269 				}
270 			}
271 		}
272 
273 		// Part file
274 		handler = new LoincPartHandler(codeSystemVersion, code2concept);
275 		iterateOverZipFile(theDescriptors, LOINC_PART_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
276 		Map<PartTypeAndPartName, String> partTypeAndPartNameToPartNumber = ((LoincPartHandler) handler).getPartTypeAndPartNameToPartNumber();
277 
278 		// Loinc Codes
279 		handler = new LoincHandler(codeSystemVersion, code2concept, propertyNamesToTypes, partTypeAndPartNameToPartNumber);
280 		iterateOverZipFile(theDescriptors, LOINC_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
281 
282 		// Loinc Hierarchy
283 		handler = new LoincHierarchyHandler(codeSystemVersion, code2concept);
284 		iterateOverZipFile(theDescriptors, LOINC_HIERARCHY_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
285 
286 		// Answer lists (ValueSets of potential answers/values for loinc "questions")
287 		handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, valueSets, conceptMaps, uploadProperties);
288 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
289 
290 		// Answer list links (connects loinc observation codes to answerlist codes)
291 		handler = new LoincAnswerListLinkHandler(code2concept, valueSets);
292 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
293 
294 		// RSNA Playbook file
295 		// Note that this should come before the "Part Related Code Mapping"
296 		// file because there are some duplicate mappings between these
297 		// two files, and the RSNA Playbook file has more metadata
298 		handler = new LoincRsnaPlaybookHandler(code2concept, valueSets, conceptMaps, uploadProperties);
299 		iterateOverZipFile(theDescriptors, LOINC_RSNA_PLAYBOOK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
300 
301 		// Part link file
302 		handler = new LoincPartLinkHandler(codeSystemVersion, code2concept);
303 		iterateOverZipFile(theDescriptors, LOINC_PART_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
304 
305 		// Part related code mapping
306 		handler = new LoincPartRelatedCodeMappingHandler(code2concept, valueSets, conceptMaps, uploadProperties);
307 		iterateOverZipFile(theDescriptors, LOINC_PART_RELATED_CODE_MAPPING_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
308 
309 		// Document Ontology File
310 		handler = new LoincDocumentOntologyHandler(code2concept, propertyNamesToTypes, valueSets, conceptMaps, uploadProperties);
311 		iterateOverZipFile(theDescriptors, LOINC_DOCUMENT_ONTOLOGY_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
312 
313 		// Top 2000 Codes - US
314 		handler = new LoincTop2000LabResultsUsHandler(code2concept, valueSets, conceptMaps, uploadProperties);
315 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
316 
317 		// Top 2000 Codes - SI
318 		handler = new LoincTop2000LabResultsSiHandler(code2concept, valueSets, conceptMaps, uploadProperties);
319 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
320 
321 		// Universal Lab Order ValueSet
322 		handler = new LoincUniversalOrderSetHandler(code2concept, valueSets, conceptMaps, uploadProperties);
323 		iterateOverZipFile(theDescriptors, LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
324 
325 		// IEEE Medical Device Codes
326 		handler = new LoincIeeeMedicalDeviceCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
327 		iterateOverZipFile(theDescriptors, LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV, handler, ',', QuoteMode.NON_NUMERIC, false);
328 
329 		// Imaging Document Codes
330 		handler = new LoincImagingDocumentCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
331 		iterateOverZipFile(theDescriptors, LOINC_IMAGING_DOCUMENT_CODES_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
332 
333 		// Group File
334 		handler = new LoincGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
335 		iterateOverZipFile(theDescriptors, LOINC_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
336 
337 		// Group Terms File
338 		handler = new LoincGroupTermsFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
339 		iterateOverZipFile(theDescriptors, LOINC_GROUP_TERMS_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
340 
341 		// Parent Group File
342 		handler = new LoincParentGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
343 		iterateOverZipFile(theDescriptors, LOINC_PARENT_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
344 
345 		IOUtils.closeQuietly(theDescriptors);
346 
347 		for (Entry<String, TermConcept> next : code2concept.entrySet()) {
348 			TermConcept nextConcept = next.getValue();
349 			if (nextConcept.getParents().isEmpty()) {
350 				codeSystemVersion.getConcepts().add(nextConcept);
351 			}
352 		}
353 
354 		int valueSetCount = valueSets.size();
355 		int rootConceptCount = codeSystemVersion.getConcepts().size();
356 		int conceptCount = code2concept.size();
357 		ourLog.info("Have {} total concepts, {} root concepts, {} ValueSets", conceptCount, rootConceptCount, valueSetCount);
358 
359 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps);
360 
361 		return new UploadStatistics(conceptCount, target);
362 	}
363 
364 	private UploadStatistics processSnomedCtFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
365 		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
366 		final Map<String, TermConcept> id2concept = new HashMap<>();
367 		final Map<String, TermConcept> code2concept = new HashMap<>();
368 		final Set<String> validConceptIds = new HashSet<>();
369 
370 		IRecordHandler handler = new SctHandlerConcept(validConceptIds);
371 		iterateOverZipFile(theDescriptors, SCT_FILE_CONCEPT, handler, '\t', null, true);
372 
373 		ourLog.info("Have {} valid concept IDs", validConceptIds.size());
374 
375 		handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion);
376 		iterateOverZipFile(theDescriptors, SCT_FILE_DESCRIPTION, handler, '\t', null, true);
377 
378 		ourLog.info("Got {} concepts, cloning map", code2concept.size());
379 		final HashMap<String, TermConcept> rootConcepts = new HashMap<>(code2concept);
380 
381 		handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept);
382 		iterateOverZipFile(theDescriptors, SCT_FILE_RELATIONSHIP, handler, '\t', null, true);
383 
384 		IOUtils.closeQuietly(theDescriptors);
385 
386 		ourLog.info("Looking for root codes");
387 		rootConcepts
388 			.entrySet()
389 			.removeIf(theStringTermConceptEntry -> theStringTermConceptEntry.getValue().getParents().isEmpty() == false);
390 
391 		ourLog.info("Done loading SNOMED CT files - {} root codes, {} total codes", rootConcepts.size(), code2concept.size());
392 
393 		Counter circularCounter = new Counter();
394 		for (TermConcept next : rootConcepts.values()) {
395 			long count = circularCounter.getThenAdd();
396 			float pct = ((float) count / rootConcepts.size()) * 100.0f;
397 			ourLog.info(" * Scanning for circular refs - have scanned {} / {} codes ({}%)", count, rootConcepts.size(), pct);
398 			dropCircularRefs(next, new ArrayList<>(), code2concept, circularCounter);
399 		}
400 
401 		codeSystemVersion.getConcepts().addAll(rootConcepts.values());
402 
403 		CodeSystem cs = new org.hl7.fhir.r4.model.CodeSystem();
404 		cs.setUrl(SCT_URI);
405 		cs.setName("SNOMED CT");
406 		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
407 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
408 
409 		return new UploadStatistics(code2concept.size(), target);
410 	}
411 
412 	@VisibleForTesting
413 	void setTermSvcDstu3ForUnitTest(IHapiTerminologySvcDstu3 theTermSvcDstu3) {
414 		myTermSvcDstu3 = theTermSvcDstu3;
415 	}
416 
417 	@VisibleForTesting
418 	void setTermSvcForUnitTests(IHapiTerminologySvc theTermSvc) {
419 		myTermSvc = theTermSvc;
420 	}
421 
422 	private IIdType storeCodeSystem(RequestDetails theRequestDetails, final TermCodeSystemVersion theCodeSystemVersion, CodeSystem theCodeSystem, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
423 		Validate.isTrue(theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT);
424 
425 		List<ValueSet> valueSets = ObjectUtils.defaultIfNull(theValueSets, Collections.emptyList());
426 		List<ConceptMap> conceptMaps = ObjectUtils.defaultIfNull(theConceptMaps, Collections.emptyList());
427 
428 		IIdType retVal;
429 		myTermSvc.setProcessDeferred(false);
430 		if (myTermSvcDstu3 != null) {
431 			retVal = myTermSvcDstu3.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
432 		} else {
433 			retVal = myTermSvcR4.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
434 		}
435 		myTermSvc.setProcessDeferred(true);
436 
437 		return retVal;
438 	}
439 
440 
441 	public static String firstNonBlank(String... theStrings) {
442 		String retVal = "";
443 		for (String nextString : theStrings) {
444 			if (isNotBlank(nextString)) {
445 				retVal = nextString;
446 				break;
447 			}
448 		}
449 		return retVal;
450 	}
451 
452 	public static TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map<String, TermConcept> id2concept, String id) {
453 		TermConcept concept = id2concept.get(id);
454 		if (concept == null) {
455 			concept = new TermConcept();
456 			id2concept.put(id, concept);
457 			concept.setCodeSystemVersion(codeSystemVersion);
458 		}
459 		return concept;
460 	}
461 
462 	static class LoadedFileDescriptors implements Closeable {
463 
464 		private List<File> myTemporaryFiles = new ArrayList<>();
465 		private List<IHapiTerminologyLoaderSvc.FileDescriptor> myUncompressedFileDescriptors = new ArrayList<>();
466 
467 		LoadedFileDescriptors(List<IHapiTerminologyLoaderSvc.FileDescriptor> theFileDescriptors) {
468 			try {
469 				for (FileDescriptor next : theFileDescriptors) {
470 					if (next.getFilename().toLowerCase().endsWith(".zip")) {
471 						ourLog.info("Uncompressing {} into temporary files", next.getFilename());
472 						try (InputStream inputStream = next.getInputStream()) {
473 							ZipInputStream zis = new ZipInputStream(new BufferedInputStream(inputStream));
474 							for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
475 								BOMInputStream fis = new BOMInputStream(zis);
476 								File nextTemporaryFile = File.createTempFile("hapifhir", ".tmp");
477 								nextTemporaryFile.deleteOnExit();
478 								FileOutputStream fos = new FileOutputStream(nextTemporaryFile, false);
479 								IOUtils.copy(fis, fos);
480 								String nextEntryFileName = nextEntry.getName();
481 								myUncompressedFileDescriptors.add(new FileDescriptor() {
482 									@Override
483 									public String getFilename() {
484 										return nextEntryFileName;
485 									}
486 
487 									@Override
488 									public InputStream getInputStream() {
489 										try {
490 											return new FileInputStream(nextTemporaryFile);
491 										} catch (FileNotFoundException e) {
492 											throw new InternalErrorException(e);
493 										}
494 									}
495 								});
496 								myTemporaryFiles.add(nextTemporaryFile);
497 							}
498 						}
499 					} else {
500 						myUncompressedFileDescriptors.add(next);
501 					}
502 
503 				}
504 			} catch (Exception e) {
505 				close();
506 				throw new InternalErrorException(e);
507 			}
508 		}
509 
510 		@Override
511 		public void close() {
512 			for (File next : myTemporaryFiles) {
513 				FileUtils.deleteQuietly(next);
514 			}
515 		}
516 
517 		List<IHapiTerminologyLoaderSvc.FileDescriptor> getUncompressedFileDescriptors() {
518 			return myUncompressedFileDescriptors;
519 		}
520 
521 		private List<String> notFound(List<String> theExpectedFilenameFragments) {
522 			Set<String> foundFragments = new HashSet<>();
523 			for (String nextExpected : theExpectedFilenameFragments) {
524 				for (FileDescriptor next : myUncompressedFileDescriptors) {
525 					if (next.getFilename().contains(nextExpected)) {
526 						foundFragments.add(nextExpected);
527 						break;
528 					}
529 				}
530 			}
531 
532 			ArrayList<String> notFoundFileNameFragments = new ArrayList<>(theExpectedFilenameFragments);
533 			notFoundFileNameFragments.removeAll(foundFragments);
534 			return notFoundFileNameFragments;
535 		}
536 
537 		private void verifyMandatoryFilesExist(List<String> theExpectedFilenameFragments) {
538 			List<String> notFound = notFound(theExpectedFilenameFragments);
539 			if (!notFound.isEmpty()) {
540 				throw new UnprocessableEntityException("Could not find the following mandatory files in input: " + notFound);
541 			}
542 		}
543 
544 		private void verifyOptionalFilesExist(List<String> theExpectedFilenameFragments) {
545 			List<String> notFound = notFound(theExpectedFilenameFragments);
546 			if (!notFound.isEmpty()) {
547 				ourLog.warn("Could not find the following optional file: " + notFound);
548 			}
549 		}
550 
551 
552 	}
553 }