001package org.hl7.fhir.convertors.misc;
002
003
004import java.io.File;
005import java.io.FileInputStream;
006import java.io.FileNotFoundException;
007import java.io.FileOutputStream;
008import java.io.IOException;
009import java.sql.Connection;
010import java.sql.DriverManager;
011import java.sql.SQLException;
012import java.sql.Statement;
013import java.util.Date;
014import java.util.List;
015import java.util.ArrayList;
016import java.util.Scanner;
017
018import org.hl7.fhir.exceptions.FHIRException;
019import org.hl7.fhir.r5.formats.IParser.OutputStyle;
020import org.hl7.fhir.r5.formats.JsonParser;
021import org.hl7.fhir.r5.model.CodeSystem;
022import org.hl7.fhir.r5.model.BooleanType;
023import org.hl7.fhir.r5.model.StringType;
024import org.hl7.fhir.r5.model.CodeType;
025import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
026import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
027import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
028import org.hl7.fhir.r5.model.CodeSystem.PropertyType;
029import org.hl7.fhir.r5.model.Coding;
030import org.hl7.fhir.r5.model.Enumerations.CodeSystemContentMode;
031import org.hl7.fhir.r5.model.Enumerations.PublicationStatus;
032import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
033import org.hl7.fhir.utilities.CSVReader;
034import org.hl7.fhir.utilities.FileUtilities;
035import org.hl7.fhir.utilities.Utilities;
036import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
037
038public class CPTImporter {
039
040  public static void main(String[] args) throws FHIRException, FileNotFoundException, IOException, ClassNotFoundException, SQLException {
041    new CPTImporter().doImport(args[0], args[1], args[2]);
042
043  }
044
045
046  private void doImport(String src, String version, String dst) throws FHIRException, FileNotFoundException, IOException, ClassNotFoundException, SQLException {
047
048    CodeSystem cs = new CodeSystem();
049    cs.setId("cpt");
050    cs.setUrl("http://www.ama-assn.org/go/cpt");
051    cs.setVersion(version);
052    cs.setName("AmaCPT");
053    cs.setTitle("AMA CPT");
054    cs.setStatus(PublicationStatus.ACTIVE);
055    cs.setDate(new Date());
056    cs.setContent(CodeSystemContentMode.COMPLETE);
057    cs.setCompositional(true);
058    cs.setPublisher("AMA");
059    cs.setValueSet("http://hl7.org/fhir/ValueSet/cpt-all");
060    cs.setCopyright("CPT © Copyright 2019 American Medical Association. All rights reserved. AMA and CPT are registered trademarks of the American Medical Association.");
061    cs.addProperty().setCode("modifier").setDescription("Whether code is a modifier code").setType(PropertyType.BOOLEAN);
062    cs.addProperty().setCode("modified").setDescription("Whether code has been modified (all base codes are not modified)").setType(PropertyType.BOOLEAN);
063    cs.addProperty().setCode("orthopox").setDescription("Whether code is one of the Pathology and Laboratory and Immunization Code(s) for Orthopoxvirus").setType(PropertyType.BOOLEAN);
064    cs.addProperty().setCode("telemedicine").setDescription("Whether code is appropriate for use with telemedicine (and the telemedicine modifier)").setType(PropertyType.BOOLEAN);
065    cs.addProperty().setCode("kind").setDescription("Kind of Code (see metadata)").setType(PropertyType.CODE);
066
067    defineMetadata(cs);
068    
069    System.out.println("LONGULT: "+readCodes(cs, Utilities.path(src, "LONGULT.txt"), false, null, null, null));
070    System.out.println("LONGUT: "+readCodes(cs, Utilities.path(src, "LONGUT.txt"), false, "upper", null, null));
071    System.out.println("MEDU: "+readCodes(cs, Utilities.path(src, "MEDU.txt"), false, "med", null, null));
072    System.out.println("SHORTU: "+readCodes(cs, Utilities.path(src, "SHORTU.txt"), false, "short", null, null));
073    System.out.println("ConsumerDescriptor: "+readCodes(cs, Utilities.path(src, "ConsumerDescriptor.txt"), true, "consumer", null, null));
074    System.out.println("ClinicianDescriptor: "+readCodes(cs, Utilities.path(src, "ClinicianDescriptor.txt"), true, "clinician", null, null));
075    System.out.println("OrthopoxvirusCodes: "+readCodes(cs, Utilities.path(src, "OrthopoxvirusCodes.txt"), false, null, null, "orthopox"));
076  
077    System.out.println("modifiers: "+processModifiers(cs, Utilities.path(src, "modifiers.csv")));
078    System.out.println("appendix P: "+processAppendixP(cs));
079    
080    
081    System.out.println("-------------------");
082    System.out.println(cs.getConcept().size());
083    int c = 0;
084    int k = 0;
085    for (ConceptDefinitionComponent cc: cs.getConcept()) {
086      c = Integer.max(c, cc.getProperty().size());
087      if (cc.getProperty().size() > 3) {
088        k++;
089      }
090    }
091    System.out.println(c);
092    System.out.println(k);
093    
094    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(dst), cs); 
095    produceDB(FileUtilities.changeFileExt(dst, ".db"), cs);
096    
097    cs.setContent(CodeSystemContentMode.FRAGMENT);
098    cs.getConcept().removeIf(cc -> !Utilities.existsInList(cc.getCode(), "metadata-kinds", "metadata-designations", "99202", "99203", "0001A", "99252", "25", "P1", "1P", "F1", "95"));
099    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(FileUtilities.changeFileExt(dst, "-fragment.json")), cs); 
100    produceDB(FileUtilities.changeFileExt(dst, "-fragment.db"), cs);
101  }
102
103  private String processAppendixP(CodeSystem cs) {
104    List<String> tcodes = new ArrayList<>();
105    tcodes.add("90785");
106    tcodes.add("90791");
107    tcodes.add("90792");
108    tcodes.add("90832");
109    tcodes.add("90833");
110    tcodes.add("90834");
111    tcodes.add("90836");
112    tcodes.add("90837");
113    tcodes.add("90838");
114    tcodes.add("90839");
115    tcodes.add("90840");
116    tcodes.add("90845");
117    tcodes.add("90846");
118    tcodes.add("90847");
119    tcodes.add("92507");
120    tcodes.add("92508");
121    tcodes.add("92521");
122    tcodes.add("92522");
123    tcodes.add("92523");
124    tcodes.add("92524");
125    tcodes.add("96040");
126    tcodes.add("96110");
127    tcodes.add("96116");
128    tcodes.add("96160");
129    tcodes.add("96161");
130    tcodes.add("97802");
131    tcodes.add("97803");
132    tcodes.add("97804");
133    tcodes.add("99406");
134    tcodes.add("99407");
135    tcodes.add("99408");
136    tcodes.add("99409");
137    tcodes.add("99497");
138    tcodes.add("99498");
139
140    for (String c : tcodes) { 
141      ConceptDefinitionComponent cc = CodeSystemUtilities.findCode(cs.getConcept(), c);
142      if (cc == null) {
143        throw new Error("unable to find tcode "+c);
144      }
145      cc.addProperty().setCode("telemedicine").setValue(new BooleanType(true));
146    }    
147    return String.valueOf(tcodes.size());
148  }
149
150
151  private void produceDB(String path, CodeSystem cs) throws ClassNotFoundException, SQLException, IOException {
152    Connection con = connect(path);
153
154    Statement stmt = con.createStatement();
155    stmt.execute("insert into Information (name, value) values ('version', "+cs.getVersion()+")");        
156    for (ConceptDefinitionComponent cc: cs.getConcept()) {
157      if (!cc.getCode().startsWith("metadata")) {
158        stmt.execute("insert into Concepts (code, modifier) values ('"+cc.getCode()+"', "+isModifier(cc)+")");
159        int i = 0;
160        if (cc.hasDisplay()) {
161          stmt.execute("insert into Designations (code, type, sequence, value) values ('"+cc.getCode()+"', 'display', 0, '"+Utilities.escapeSql(cc.getDisplay())+"')");
162          i++;
163        }
164        for (ConceptDefinitionDesignationComponent d : cc.getDesignation()) {
165          stmt.execute("insert into Designations (code, type, sequence, value) values ('"+cc.getCode()+"', '"+d.getUse().getCode()+"', "+i+", '"+Utilities.escapeSql(d.getValue())+"')");
166          i++;
167        }
168        i = 0;
169        for (ConceptPropertyComponent p : cc.getProperty()) {
170          if (!Utilities.existsInList(p.getCode(), "modified", "modifier")) {
171            stmt.execute("insert into Properties (code, name, sequence, value) values ('"+cc.getCode()+"', '"+p.getCode()+"', "+i+", '"+p.getValue().primitiveValue()+"')");
172            i++;
173          }
174        }    
175      }
176    }
177
178  }
179
180  private String isModifier(ConceptDefinitionComponent cc) {
181    for (ConceptPropertyComponent p : cc.getProperty()) {
182      if (p.getCode().equals("modifier")) {
183        return p.getValue().primitiveValue().equals("true") ? "1" : "0";
184      }
185    }
186    return "0";
187  }
188
189
190  private Connection connect(String dest) throws SQLException, ClassNotFoundException, IOException {
191    //    Class.forName("com.mysql.jdbc.Driver");  
192    //    con = DriverManager.getConnection("jdbc:mysql://localhost:3306/omop?useSSL=false","root",{pwd}); 
193    ManagedFileAccess.file(dest).delete();
194    Connection con = DriverManager.getConnection("jdbc:sqlite:"+dest); 
195    makeMetadataTable(con);
196    makeConceptsTable(con);
197    makeDesignationsTable(con);
198    makePropertiesTable(con);
199    return con;    
200  }
201  
202  private void makeDesignationsTable(Connection con) throws SQLException {
203    Statement stmt = con.createStatement();
204    stmt.execute("CREATE TABLE Designations (\r\n"+
205        "`code` varchar(15) NOT NULL,\r\n"+
206        "`type` varchar(15) NOT NULL,\r\n"+
207        "`sequence` int NOT NULL,\r\n"+
208        "`value` text NOT NULL,\r\n"+
209        "PRIMARY KEY (`code`, `type`, `sequence`))\r\n");
210  }
211
212
213  private void makePropertiesTable(Connection con) throws SQLException {
214
215    Statement stmt = con.createStatement();
216    stmt.execute("CREATE TABLE Properties (\r\n"+
217        "`code` varchar(15) NOT NULL,\r\n"+
218        "`name` varchar(15) NOT NULL,\r\n"+
219        "`sequence` int NOT NULL,\r\n"+
220        "`value` varchar(15) NOT NULL,\r\n"+
221        "PRIMARY KEY (`code`, `name`, `sequence`))\r\n");
222
223  }
224
225
226  private void makeConceptsTable(Connection con) throws SQLException {
227
228    Statement stmt = con.createStatement();
229    stmt.execute("CREATE TABLE Concepts (\r\n"+
230        "`code` varchar(15) NOT NULL,\r\n"+
231        "`modifier` int DEFAULT NULL,\r\n"+
232        "PRIMARY KEY (`code`))\r\n");
233
234  }
235
236
237  private void makeMetadataTable(Connection con) throws SQLException {
238
239    Statement stmt = con.createStatement();
240    stmt.execute("CREATE TABLE Information (\r\n"+
241        "`name` varchar(64) NOT NULL,\r\n"+
242        "`value` varchar(64) DEFAULT NULL,\r\n"+
243        "PRIMARY KEY (`name`))\r\n");
244
245  }
246
247
248  private void defineMetadata(CodeSystem cs) {
249    ConceptDefinitionComponent pc = mm(cs.addConcept().setCode("metadata-kinds"));
250    mm(pc.addConcept()).setCode("code").setDisplay("A normal CPT code");
251    mm(pc.addConcept()).setCode("cat-1").setDisplay("CPT Level I Modifiers");
252    mm(pc.addConcept()).setCode("cat-2").setDisplay("A Category II code or modifier");
253    mm(pc.addConcept()).setCode("physical-status").setDisplay("Anesthesia Physical Status Modifiers");
254    mm(pc.addConcept()).setCode("general").setDisplay("A general modifier");
255    mm(pc.addConcept()).setCode("hcpcs").setDisplay("Level II (HCPCS/National) Modifiers");
256    mm(pc.addConcept()).setCode("metadata").setDisplay("A kind of code or designation");
257
258    ConceptDefinitionComponent dc = mm(cs.addConcept().setCode("metadata-designations"));
259    mm(dc.addConcept()).setCode("upper").setDisplay("Uppercase variant of the display");
260    mm(dc.addConcept()).setCode("med").setDisplay("Medium length variant of the display (all uppercase)");
261    mm(dc.addConcept()).setCode("short").setDisplay("Short length variant of the display (all uppercase)");
262    mm(dc.addConcept()).setCode("consumer").setDisplay("Consumer Friendly representation for the concept");
263    mm(dc.addConcept()).setCode("clinician").setDisplay("Clinician Friendly representation for the concept (can be more than one per concept)");
264  }
265
266  private ConceptDefinitionComponent mm(ConceptDefinitionComponent cc) {
267    cc.addProperty().setCode("kind").setValue(new CodeType("metadata"));
268    return cc;
269  }
270
271  private int processModifiers(CodeSystem cs, String path) throws FHIRException, FileNotFoundException, IOException {
272    CSVReader csv = new CSVReader(ManagedFileAccess.inStream(path));
273    csv.readHeaders();
274
275    int res = 0;
276    while (csv.line()) {
277      String code = csv.cell("Code");
278      String general = csv.cell("General");
279      String physicalStatus = csv.cell("PhysicalStatus");
280      String levelOne = csv.cell("LevelOne");
281      String levelTwo = csv.cell("LevelTwo");
282      String hcpcs = csv.cell("HCPCS");
283      String defn = csv.cell("Definition");
284
285      res = Integer.max(res, defn.length());
286      ConceptDefinitionComponent cc = cs.addConcept().setCode(code);
287      cc.setDisplay(defn);    
288      cc.addProperty().setCode("modified").setValue(new BooleanType(false));
289      cc.addProperty().setCode("modifier").setValue(new BooleanType(true));
290      if ("1".equals(general)) { 
291        cc.addProperty().setCode("kind").setValue(new CodeType("general"));
292      }
293      if ("1".equals(physicalStatus)) { 
294        cc.addProperty().setCode("kind").setValue(new CodeType("physical-status"));
295      }
296      if ("1".equals(levelOne)) { 
297        cc.addProperty().setCode("kind").setValue(new CodeType("cat-1"));
298      }
299      if ("1".equals(levelTwo)) { 
300        cc.addProperty().setCode("kind").setValue(new CodeType("cat-2"));
301      }
302      if ("1".equals(hcpcs)) { 
303        cc.addProperty().setCode("kind").setValue(new CodeType("hcpcs"));
304      }
305    }
306    return res;
307  }
308
309  private int readCodes(CodeSystem cs, String path, boolean hasConceptId, String use, String type, String boolProp) throws IOException {
310    int res = 0;
311    FileInputStream inputStream = null;
312    Scanner sc = null;
313    try {
314        inputStream = ManagedFileAccess.inStream(path);
315        sc = new Scanner(inputStream, "UTF-8");
316        while (sc.hasNextLine()) {
317          String line = sc.nextLine();
318          if (hasConceptId) {
319            line = line.substring(7).trim();
320          }
321          String code = line.substring(0, 5);
322          String desc = line.substring(6);
323          if (desc.contains("\t")) {
324            desc = desc.substring(desc.indexOf("\t")+1);
325          }
326          res = Integer.max(res, desc.length());
327          ConceptDefinitionComponent cc = CodeSystemUtilities.getCode(cs, code);
328          if (cc == null) {
329            cc = cs.addConcept().setCode(code);
330            cc.addProperty().setCode("modifier").setValue(new BooleanType(false));
331            cc.addProperty().setCode("modified").setValue(new BooleanType(false));
332            if (type == null) {
333              if (Utilities.isInteger(code)) {
334                cc.addProperty().setCode("kind").setValue(new CodeType("code"));              
335              } else {
336                cc.addProperty().setCode("kind").setValue(new CodeType("cat-2"));                            
337              }
338            } else { 
339              cc.addProperty().setCode("kind").setValue(new CodeType(type));
340            }
341          } else if (type != null) {
342            cc.addProperty().setCode("kind").setValue(new CodeType(type));
343          }
344          if (boolProp != null) {
345            cc.addProperty().setCode(boolProp).setValue(new BooleanType(true));
346          }
347          if (use == null) {
348            if (cc.hasDisplay()) {
349              System.err.println("?");
350            }
351            cc.setDisplay(desc);            
352          } else {
353            cc.addDesignation().setUse(new Coding("http://www.ama-assn.org/go/cpt", use, null)).setValue(desc);
354          }
355        }
356        // note that Scanner suppresses exceptions
357        if (sc.ioException() != null) {
358          throw sc.ioException();
359        }
360    } finally {
361        if (inputStream != null) {
362            inputStream.close();
363        }
364        if (sc != null) {
365            sc.close();
366        }
367    }
368    return res;
369  }
370
371
372}