View Javadoc
1   package ca.uhn.fhir.jpa.term;
2   
3   import ca.uhn.fhir.context.FhirContext;
4   import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
5   import ca.uhn.fhir.jpa.entity.TermConcept;
6   import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
7   import ca.uhn.fhir.jpa.term.loinc.*;
8   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerConcept;
9   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerDescription;
10  import ca.uhn.fhir.jpa.term.snomedct.SctHandlerRelationship;
11  import ca.uhn.fhir.jpa.util.Counter;
12  import ca.uhn.fhir.rest.api.server.RequestDetails;
13  import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
14  import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
15  import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
16  import com.google.common.annotations.VisibleForTesting;
17  import com.google.common.base.Charsets;
18  import org.apache.commons.csv.CSVFormat;
19  import org.apache.commons.csv.CSVParser;
20  import org.apache.commons.csv.CSVRecord;
21  import org.apache.commons.csv.QuoteMode;
22  import org.apache.commons.io.FileUtils;
23  import org.apache.commons.io.IOUtils;
24  import org.apache.commons.io.input.BOMInputStream;
25  import org.apache.commons.lang3.ObjectUtils;
26  import org.apache.commons.lang3.StringUtils;
27  import org.apache.commons.lang3.Validate;
28  import org.hl7.fhir.instance.model.api.IIdType;
29  import org.hl7.fhir.r4.model.CodeSystem;
30  import org.hl7.fhir.r4.model.ConceptMap;
31  import org.hl7.fhir.r4.model.ValueSet;
32  import org.springframework.beans.factory.annotation.Autowired;
33  
34  import java.io.*;
35  import java.util.*;
36  import java.util.Map.Entry;
37  import java.util.zip.ZipEntry;
38  import java.util.zip.ZipInputStream;
39  
40  import static org.apache.commons.lang3.StringUtils.isNotBlank;
41  
42  /*
43   * #%L
44   * HAPI FHIR JPA Server
45   * %%
46   * Copyright (C) 2014 - 2019 University Health Network
47   * %%
48   * Licensed under the Apache License, Version 2.0 (the "License");
49   * you may not use this file except in compliance with the License.
50   * You may obtain a copy of the License at
51   * 
52   *      http://www.apache.org/licenses/LICENSE-2.0
53   * 
54   * Unless required by applicable law or agreed to in writing, software
55   * distributed under the License is distributed on an "AS IS" BASIS,
56   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
57   * See the License for the specific language governing permissions and
58   * limitations under the License.
59   * #L%
60   */
61  
62  public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
63  	public static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_";
64  	public static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full-en";
65  	public static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
66  	public static final String LOINC_ANSWERLIST_FILE = "AnswerList.csv";
67  	public static final String LOINC_ANSWERLIST_LINK_FILE = "LoincAnswerListLink.csv";
68  	public static final String LOINC_DOCUMENT_ONTOLOGY_FILE = "DocumentOntology.csv";
69  	public static final String LOINC_UPLOAD_PROPERTIES_FILE = "loincupload.properties";
70  	public static final String LOINC_FILE = "LoincTable/Loinc.csv";
71  	public static final String LOINC_HIERARCHY_FILE = "MultiAxialHierarchy.csv";
72  	public static final String LOINC_PART_FILE = "Part.csv";
73  	public static final String LOINC_PART_LINK_FILE = "LoincPartLink.csv";
74  	public static final String LOINC_PART_RELATED_CODE_MAPPING_FILE = "PartRelatedCodeMapping.csv";
75  	public static final String LOINC_RSNA_PLAYBOOK_FILE = "LoincRsnaRadiologyPlaybook.csv";
76  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE = "Top2000CommonLabResultsUs.csv";
77  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE = "Top2000CommonLabResultsSi.csv";
78  	public static final String LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE = "LoincUniversalLabOrdersValueSet.csv";
79  	public static final String LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV = "LoincIeeeMedicalDeviceCodeMappingTable.csv";
80  	public static final String LOINC_IMAGING_DOCUMENT_CODES_FILE = "ImagingDocumentCodes.csv";
81  	public static final String LOINC_GROUP_FILE = "Group.csv";
82  	public static final String LOINC_GROUP_TERMS_FILE = "GroupLoincTerms.csv";
83  	public static final String LOINC_PARENT_GROUP_FILE = "ParentGroup.csv";
84  	private static final int LOG_INCREMENT = 1000;
85  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvcImpl.class);
86  
87  	@Autowired
88  	private IHapiTerminologySvc myTermSvc;
89  	@Autowired(required = false)
90  	private IHapiTerminologySvcDstu3 myTermSvcDstu3;
91  	@Autowired(required = false)
92  	private IHapiTerminologySvcR4 myTermSvcR4;
93  
94  	private void dropCircularRefs(TermConcept theConcept, ArrayList<String> theChain, Map<String, TermConcept> theCode2concept, Counter theCircularCounter) {
95  
96  		theChain.add(theConcept.getCode());
97  		for (Iterator<TermConceptParentChildLink> childIter = theConcept.getChildren().iterator(); childIter.hasNext(); ) {
98  			TermConceptParentChildLink next = childIter.next();
99  			TermConcept nextChild = next.getChild();
100 			if (theChain.contains(nextChild.getCode())) {
101 
102 				StringBuilder b = new StringBuilder();
103 				b.append("Removing circular reference code ");
104 				b.append(nextChild.getCode());
105 				b.append(" from parent ");
106 				b.append(next.getParent().getCode());
107 				b.append(". Chain was: ");
108 				for (String nextInChain : theChain) {
109 					TermConcept nextCode = theCode2concept.get(nextInChain);
110 					b.append(nextCode.getCode());
111 					b.append('[');
112 					b.append(StringUtils.substring(nextCode.getDisplay(), 0, 20).replace("[", "").replace("]", "").trim());
113 					b.append("] ");
114 				}
115 				ourLog.info(b.toString(), theConcept.getCode());
116 				childIter.remove();
117 				nextChild.getParents().remove(next);
118 
119 			} else {
120 				dropCircularRefs(nextChild, theChain, theCode2concept, theCircularCounter);
121 			}
122 		}
123 		theChain.remove(theChain.size() - 1);
124 
125 	}
126 
127 	private void iterateOverZipFile(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode, boolean theIsPartialFilename) {
128 
129 		boolean foundMatch = false;
130 		for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
131 			String nextFilename = nextZipBytes.getFilename();
132 			boolean matches;
133 			if (theIsPartialFilename) {
134 				matches = nextFilename.contains(theFileNamePart);
135 			} else {
136 				matches = nextFilename.endsWith("/" + theFileNamePart) || nextFilename.equals(theFileNamePart);
137 			}
138 
139 			if (matches) {
140 				ourLog.info("Processing file {}", nextFilename);
141 				foundMatch = true;
142 
143 				Reader reader;
144 				CSVParser parsed;
145 				try {
146 					reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
147 
148 					if (ourLog.isTraceEnabled()) {
149 						String contents = IOUtils.toString(reader);
150 						ourLog.info("File contents for: {}\n{}", nextFilename, contents);
151 						reader = new StringReader(contents);
152 					}
153 
154 					CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
155 					if (theQuoteMode != null) {
156 						format = format.withQuote('"').withQuoteMode(theQuoteMode);
157 					}
158 					parsed = new CSVParser(reader, format);
159 					Iterator<CSVRecord> iter = parsed.iterator();
160 					ourLog.debug("Header map: {}", parsed.getHeaderMap());
161 
162 					int count = 0;
163 					int nextLoggedCount = 0;
164 					while (iter.hasNext()) {
165 						CSVRecord nextRecord = iter.next();
166 						if (nextRecord.isConsistent()==false) {
167 							continue;
168 						}
169 						theHandler.accept(nextRecord);
170 						count++;
171 						if (count >= nextLoggedCount) {
172 							ourLog.info(" * Processed {} records in {}", count, nextFilename);
173 							nextLoggedCount += LOG_INCREMENT;
174 						}
175 					}
176 
177 				} catch (IOException e) {
178 					throw new InternalErrorException(e);
179 				}
180 			}
181 
182 		}
183 
184 		if (!foundMatch) {
185 			throw new InvalidRequestException("Did not find file matching " + theFileNamePart);
186 		}
187 
188 	}
189 
190 	@Override
191 	public UploadStatistics loadLoinc(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
192 		try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
193 			List<String> mandatoryFilenameFragments = Arrays.asList(
194 				LOINC_FILE,
195 				LOINC_HIERARCHY_FILE,
196 				LOINC_UPLOAD_PROPERTIES_FILE,
197 				LOINC_ANSWERLIST_FILE,
198 				LOINC_ANSWERLIST_LINK_FILE,
199 				LOINC_PART_FILE,
200 				LOINC_PART_LINK_FILE,
201 				LOINC_PART_RELATED_CODE_MAPPING_FILE,
202 				LOINC_DOCUMENT_ONTOLOGY_FILE,
203 				LOINC_RSNA_PLAYBOOK_FILE,
204 				LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE,
205 				LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE,
206 				LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE,
207 				LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV,
208 				LOINC_IMAGING_DOCUMENT_CODES_FILE
209 			);
210 		descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments);
211 
212 			List<String> optionalFilenameFragments = Arrays.asList(
213 			);
214 		descriptors.verifyOptionalFilesExist(optionalFilenameFragments);
215 
216 		ourLog.info("Beginning LOINC processing");
217 
218 		return processLoincFiles(descriptors, theRequestDetails);
219 		}
220 	}
221 
222 	@Override
223 	public UploadStatistics loadSnomedCt(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
224 		try (LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles)) {
225 
226 			List<String> expectedFilenameFragments = Arrays.asList(
227 				SCT_FILE_DESCRIPTION,
228 				SCT_FILE_RELATIONSHIP,
229 				SCT_FILE_CONCEPT);
230 			descriptors.verifyMandatoryFilesExist(expectedFilenameFragments);
231 
232 			ourLog.info("Beginning SNOMED CT processing");
233 
234 			return processSnomedCtFiles(descriptors, theRequestDetails);
235 		}
236 	}
237 
238 	UploadStatistics processLoincFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
239 		final TermCodeSystemVersionermCodeSystemVersion">TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
240 		final Map<String, TermConcept> code2concept = new HashMap<>();
241 		final List<ValueSet> valueSets = new ArrayList<>();
242 		final List<ConceptMap> conceptMaps = new ArrayList<>();
243 
244 		CodeSystem loincCs;
245 		try {
246 			String loincCsString = IOUtils.toString(BaseHapiTerminologySvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/loinc/loinc.xml"), Charsets.UTF_8);
247 			loincCs = FhirContext.forR4().newXmlParser().parseResource(CodeSystem.class, loincCsString);
248 		} catch (IOException e) {
249 			throw new InternalErrorException("Failed to load loinc.xml", e);
250 		}
251 
252 		Map<String, CodeSystem.PropertyType> propertyNamesToTypes = new HashMap<>();
253 		for (CodeSystem.PropertyComponent nextProperty : loincCs.getProperty()) {
254 			String nextPropertyCode = nextProperty.getCode();
255 			CodeSystem.PropertyType nextPropertyType = nextProperty.getType();
256 			if (isNotBlank(nextPropertyCode)) {
257 				propertyNamesToTypes.put(nextPropertyCode, nextPropertyType);
258 			}
259 		}
260 
261 		IRecordHandler handler;
262 
263 		Properties uploadProperties = new Properties();
264 		for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) {
265 			if (next.getFilename().endsWith("loincupload.properties")) {
266 				try {
267 					try (InputStream inputStream = next.getInputStream()) {
268 						uploadProperties.load(inputStream);
269 					}
270 				} catch (IOException e) {
271 					throw new InternalErrorException("Failed to read loincupload.properties", e);
272 				}
273 			}
274 		}
275 
276 		// Part file
277 		handler = new LoincPartHandler(codeSystemVersion, code2concept);
278 		iterateOverZipFile(theDescriptors, LOINC_PART_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
279 		Map<PartTypeAndPartName, String> partTypeAndPartNameToPartNumber = ((LoincPartHandler) handler).getPartTypeAndPartNameToPartNumber();
280 
281 		// Loinc Codes
282 		handler = new LoincHandler(codeSystemVersion, code2concept, propertyNamesToTypes, partTypeAndPartNameToPartNumber);
283 		iterateOverZipFile(theDescriptors, LOINC_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
284 
285 		// Loinc Hierarchy
286 		handler = new LoincHierarchyHandler(codeSystemVersion, code2concept);
287 		iterateOverZipFile(theDescriptors, LOINC_HIERARCHY_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
288 
289 		// Answer lists (ValueSets of potential answers/values for loinc "questions")
290 		handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, valueSets, conceptMaps, uploadProperties);
291 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
292 
293 		// Answer list links (connects loinc observation codes to answerlist codes)
294 		handler = new LoincAnswerListLinkHandler(code2concept, valueSets);
295 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
296 
297 		// RSNA Playbook file
298 		// Note that this should come before the "Part Related Code Mapping"
299 		// file because there are some duplicate mappings between these
300 		// two files, and the RSNA Playbook file has more metadata
301 		handler = new LoincRsnaPlaybookHandler(code2concept, valueSets, conceptMaps, uploadProperties);
302 		iterateOverZipFile(theDescriptors, LOINC_RSNA_PLAYBOOK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
303 
304 		// Part link file
305 		handler = new LoincPartLinkHandler(codeSystemVersion, code2concept);
306 		iterateOverZipFile(theDescriptors, LOINC_PART_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
307 
308 		// Part related code mapping
309 		handler = new LoincPartRelatedCodeMappingHandler(code2concept, valueSets, conceptMaps, uploadProperties);
310 		iterateOverZipFile(theDescriptors, LOINC_PART_RELATED_CODE_MAPPING_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
311 
312 		// Document Ontology File
313 		handler = new LoincDocumentOntologyHandler(code2concept, propertyNamesToTypes, valueSets, conceptMaps, uploadProperties);
314 		iterateOverZipFile(theDescriptors, LOINC_DOCUMENT_ONTOLOGY_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
315 
316 		// Top 2000 Codes - US
317 		handler = new LoincTop2000LabResultsUsHandler(code2concept, valueSets, conceptMaps, uploadProperties);
318 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
319 
320 		// Top 2000 Codes - SI
321 		handler = new LoincTop2000LabResultsSiHandler(code2concept, valueSets, conceptMaps, uploadProperties);
322 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
323 
324 		// Universal Lab Order ValueSet
325 		handler = new LoincUniversalOrderSetHandler(code2concept, valueSets, conceptMaps, uploadProperties);
326 		iterateOverZipFile(theDescriptors, LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
327 
328 		// IEEE Medical Device Codes
329 		handler = new LoincIeeeMedicalDeviceCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
330 		iterateOverZipFile(theDescriptors, LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV, handler, ',', QuoteMode.NON_NUMERIC, false);
331 
332 		// Imaging Document Codes
333 		handler = new LoincImagingDocumentCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
334 		iterateOverZipFile(theDescriptors, LOINC_IMAGING_DOCUMENT_CODES_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
335 
336 		// Group File
337 		handler = new LoincGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
338 		iterateOverZipFile(theDescriptors, LOINC_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
339 
340 		// Group Terms File
341 		handler = new LoincGroupTermsFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
342 		iterateOverZipFile(theDescriptors, LOINC_GROUP_TERMS_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
343 
344 		// Parent Group File
345 		handler = new LoincParentGroupFileHandler(code2concept, valueSets, conceptMaps, uploadProperties);
346 		iterateOverZipFile(theDescriptors, LOINC_PARENT_GROUP_FILE, handler, ',', QuoteMode.NON_NUMERIC, false);
347 
348 		IOUtils.closeQuietly(theDescriptors);
349 
350 		for (Entry<String, TermConcept> next : code2concept.entrySet()) {
351 			TermConcept nextConcept = next.getValue();
352 			if (nextConcept.getParents().isEmpty()) {
353 				codeSystemVersion.getConcepts().add(nextConcept);
354 			}
355 		}
356 
357 		int valueSetCount = valueSets.size();
358 		int rootConceptCount = codeSystemVersion.getConcepts().size();
359 		int conceptCount = code2concept.size();
360 		ourLog.info("Have {} total concepts, {} root concepts, {} ValueSets", conceptCount, rootConceptCount, valueSetCount);
361 
362 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps);
363 
364 		return new UploadStatistics(conceptCount, target);
365 	}
366 
367 	private UploadStatistics processSnomedCtFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
368 		final TermCodeSystemVersionermCodeSystemVersion">TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
369 		final Map<String, TermConcept> id2concept = new HashMap<>();
370 		final Map<String, TermConcept> code2concept = new HashMap<>();
371 		final Set<String> validConceptIds = new HashSet<>();
372 
373 		IRecordHandler handler = new SctHandlerConcept(validConceptIds);
374 		iterateOverZipFile(theDescriptors, SCT_FILE_CONCEPT, handler, '\t', null, true);
375 
376 		ourLog.info("Have {} valid concept IDs", validConceptIds.size());
377 
378 		handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion);
379 		iterateOverZipFile(theDescriptors, SCT_FILE_DESCRIPTION, handler, '\t', null, true);
380 
381 		ourLog.info("Got {} concepts, cloning map", code2concept.size());
382 		final HashMap<String, TermConcept> rootConcepts = new HashMap<>(code2concept);
383 
384 		handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept);
385 		iterateOverZipFile(theDescriptors, SCT_FILE_RELATIONSHIP, handler, '\t', null, true);
386 
387 		IOUtils.closeQuietly(theDescriptors);
388 
389 		ourLog.info("Looking for root codes");
390 		rootConcepts
391 			.entrySet()
392 			.removeIf(theStringTermConceptEntry -> theStringTermConceptEntry.getValue().getParents().isEmpty() == false);
393 
394 		ourLog.info("Done loading SNOMED CT files - {} root codes, {} total codes", rootConcepts.size(), code2concept.size());
395 
396 		Counter circularCounter = new Counter();
397 		for (TermConcept next : rootConcepts.values()) {
398 			long count = circularCounter.getThenAdd();
399 			float pct = ((float) count / rootConcepts.size()) * 100.0f;
400 			ourLog.info(" * Scanning for circular refs - have scanned {} / {} codes ({}%)", count, rootConcepts.size(), pct);
401 			dropCircularRefs(next, new ArrayList<>(), code2concept, circularCounter);
402 		}
403 
404 		codeSystemVersion.getConcepts().addAll(rootConcepts.values());
405 
406 		CodeSystem cs = new org.hl7.fhir.r4.model.CodeSystem();
407 		cs.setUrl(SCT_URI);
408 		cs.setName("SNOMED CT");
409 		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
410 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
411 
412 		return new UploadStatistics(code2concept.size(), target);
413 	}
414 
415 	@VisibleForTesting
416 	void setTermSvcDstu3ForUnitTest(IHapiTerminologySvcDstu3 theTermSvcDstu3) {
417 		myTermSvcDstu3 = theTermSvcDstu3;
418 	}
419 
420 	@VisibleForTesting
421 	void setTermSvcForUnitTests(IHapiTerminologySvc theTermSvc) {
422 		myTermSvc = theTermSvc;
423 	}
424 
425 	private IIdType storeCodeSystem(RequestDetails theRequestDetails, final TermCodeSystemVersion theCodeSystemVersion, CodeSystem theCodeSystem, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
426 		Validate.isTrue(theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT);
427 
428 		List<ValueSet> valueSets = ObjectUtils.defaultIfNull(theValueSets, Collections.emptyList());
429 		List<ConceptMap> conceptMaps = ObjectUtils.defaultIfNull(theConceptMaps, Collections.emptyList());
430 
431 		IIdType retVal;
432 		myTermSvc.setProcessDeferred(false);
433 		if (myTermSvcDstu3 != null) {
434 			retVal = myTermSvcDstu3.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
435 		} else {
436 			retVal = myTermSvcR4.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
437 		}
438 		myTermSvc.setProcessDeferred(true);
439 
440 		return retVal;
441 	}
442 
443 
444 	public static String firstNonBlank(String... theStrings) {
445 		String retVal = "";
446 		for (String nextString : theStrings) {
447 			if (isNotBlank(nextString)) {
448 				retVal = nextString;
449 				break;
450 			}
451 		}
452 		return retVal;
453 	}
454 
455 	public static TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map<String, TermConcept> id2concept, String id) {
456 		TermConcept concept = id2concept.get(id);
457 		if (concept == null) {
458 			concept = new TermConcept();
459 			id2concept.put(id, concept);
460 			concept.setCodeSystemVersion(codeSystemVersion);
461 		}
462 		return concept;
463 	}
464 
465 	static class LoadedFileDescriptors implements Closeable {
466 
467 		private List<File> myTemporaryFiles = new ArrayList<>();
468 		private List<IHapiTerminologyLoaderSvc.FileDescriptor> myUncompressedFileDescriptors = new ArrayList<>();
469 
470 		LoadedFileDescriptors(List<IHapiTerminologyLoaderSvc.FileDescriptor> theFileDescriptors) {
471 			try {
472 				for (FileDescriptor next : theFileDescriptors) {
473 					if (next.getFilename().toLowerCase().endsWith(".zip")) {
474 						ourLog.info("Uncompressing {} into temporary files", next.getFilename());
475 						try (InputStream inputStream = next.getInputStream()) {
476 							ZipInputStream zis = new ZipInputStream(new BufferedInputStream(inputStream));
477 							for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
478 								BOMInputStream fis = new BOMInputStream(zis);
479 								File nextTemporaryFile = File.createTempFile("hapifhir", ".tmp");
480 								nextTemporaryFile.deleteOnExit();
481 								FileOutputStream fos = new FileOutputStream(nextTemporaryFile, false);
482 								IOUtils.copy(fis, fos);
483 								String nextEntryFileName = nextEntry.getName();
484 								myUncompressedFileDescriptors.add(new FileDescriptor() {
485 									@Override
486 									public String getFilename() {
487 										return nextEntryFileName;
488 									}
489 
490 									@Override
491 									public InputStream getInputStream() {
492 										try {
493 											return new FileInputStream(nextTemporaryFile);
494 										} catch (FileNotFoundException e) {
495 											throw new InternalErrorException(e);
496 										}
497 									}
498 								});
499 								myTemporaryFiles.add(nextTemporaryFile);
500 							}
501 						}
502 					} else {
503 						myUncompressedFileDescriptors.add(next);
504 					}
505 
506 				}
507 			} catch (Exception e) {
508 				close();
509 				throw new InternalErrorException(e);
510 			}
511 		}
512 
513 		@Override
514 		public void close() {
515 			for (File next : myTemporaryFiles) {
516 				FileUtils.deleteQuietly(next);
517 			}
518 		}
519 
520 		List<IHapiTerminologyLoaderSvc.FileDescriptor> getUncompressedFileDescriptors() {
521 			return myUncompressedFileDescriptors;
522 		}
523 
524 		private List<String> notFound(List<String> theExpectedFilenameFragments) {
525 			Set<String> foundFragments = new HashSet<>();
526 			for (String nextExpected : theExpectedFilenameFragments) {
527 				for (FileDescriptor next : myUncompressedFileDescriptors) {
528 					if (next.getFilename().contains(nextExpected)) {
529 						foundFragments.add(nextExpected);
530 						break;
531 					}
532 				}
533 			}
534 
535 			ArrayList<String> notFoundFileNameFragments = new ArrayList<>(theExpectedFilenameFragments);
536 			notFoundFileNameFragments.removeAll(foundFragments);
537 			return notFoundFileNameFragments;
538 		}
539 
540 		private void verifyMandatoryFilesExist(List<String> theExpectedFilenameFragments) {
541 			List<String> notFound = notFound(theExpectedFilenameFragments);
542 			if (!notFound.isEmpty()) {
543 				throw new UnprocessableEntityException("Could not find the following mandatory files in input: " + notFound);
544 			}
545 		}
546 
547 		private void verifyOptionalFilesExist(List<String> theExpectedFilenameFragments) {
548 			List<String> notFound = notFound(theExpectedFilenameFragments);
549 			if (!notFound.isEmpty()) {
550 				ourLog.warn("Could not find the following optional file: " + notFound);
551 			}
552 		}
553 
554 
555 	}
556 }