From a03e7ca03e5478c065e458eed864b7506bbda3e7 Mon Sep 17 00:00:00 2001 From: Michael Seaton Date: Fri, 30 Aug 2024 22:55:48 -0400 Subject: [PATCH 1/5] In progress checkpoint --- .../analysis/HtmlFormCsvExporter.java | 284 ++++++++++++++++++ .../analysis/HtmlFormParser.java | 114 +++++++ .../htmlformentry/analysis/HtmlFormTag.java | 75 +++++ .../analysis/HtmlFormTagConverter.java | 118 ++++++++ .../analysis/HtmlFormTagCounter.java | 67 +++++ .../htmlformentry/analysis/HtmlFormUtils.java | 46 +++ .../pihcore/htmlformentry/analysis/Tags.java | 35 +++ .../analysis/HtmlFormCsvExporterTest.java | 101 +++++++ .../analysis/HtmlFormNonMemoryTest.java | 102 +++++++ .../analysis/HtmlFormTagCounterTest.java | 19 ++ pom.xml | 8 + 11 files changed, 969 insertions(+) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTag.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java create mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java create mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormNonMemoryTest.java create mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounterTest.java diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java new file mode 100644 index 000000000..24e2f3a2b --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java @@ -0,0 +1,284 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.apache.commons.lang.StringUtils; +import org.openmrs.module.htmlformentry.FormEntryContext; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.htmlformentry.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; +import java.util.TreeSet; + +public class HtmlFormCsvExporter { + + private final Logger log = LoggerFactory.getLogger(HtmlFormCsvExporter.class); + + public void export(File file) { + HtmlFormTag tag = loadFormFromFile(file); + HtmlFormTagConverter converter = new HtmlFormTagConverter(); + for (Tags.Standard standardTag : Tags.Standard.values()) { + tag = converter.removeTagsByName(tag, standardTag.getNodeName()); + } + tag = converter.removeTagsByName(tag, "uimessage"); + tag = converter.removeTagsByName(tag, "lookup"); + tag = converter.removeTagsByName(tag, "submit"); + tag = converter.removeIfModeTags(tag); + tag = converter.mergeTagIntoChildren(tag, "includeIf"); + tag = converter.mergeTagIntoChildren(tag, "excludeIf"); + + // Handle obs group + tag = converter.mergeObsGroupIntoChildren(tag); + + // Handle the complex web of velocity conditions for entry of date/location/provider + tag = converter.mergeSiblings(tag, "encounterDate"); + tag = converter.mergeSiblings(tag, "encounterLocation"); + tag = converter.mergeSiblings(tag, "encounterProviderAndRole"); + + // Handle when/then tags + tag = converter.mergeSiblings(tag, "when"); + tag = converter.removeTagsByName(tag,"controls"); + + + HtmlFormUtils.printTag(tag, ""); + } + + protected HtmlFormTag loadFormFromFile(File file) { + HtmlFormParser parser = new HtmlFormParser() { + @Override + protected String preprocessXml(String xml) throws Exception { + HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + xml = generator.stripComments(xml); + xml = generator.applyMacros(xml); + xml = generator.applyRepeats(xml); + xml = generator.applyTranslations(xml, new FormEntryContext(FormEntryContext.Mode.EDIT) { + @Override + public Translator getTranslator() { + return new Translator() { + @Override + public String translate(String localeStr, String key) { + String ret = getTranslations(localeStr).get(key); + return (ret == null ? key : ret); + } + }; + } + }); + return xml; + } + }; + return parser.parseFile(file); + } + + + + + + + + + + public static final List AVAILABLE_COLUMNS = Arrays.asList( + "sectionName", "tagName", "style", "groupingConceptId", "conceptId", "conceptIds", "valueCoded", + "answerConceptId", "answerConceptIds", "answers", "answerConceptSetIds", "answerClasses", + "answerDrugs", "answerLocationTags", "labelCode", "labelText", "conceptLabels", + "answerCode", "answerLabel", "answerCodes", "answerLabels", "required", + "obsCommentUsed", "showCommentField", "commentFieldLabel", "commentFieldCode", + "dateLabel", "allowFutureDates", "showTime", "defaultValue", "whenValueThen", "toggle" + ); + + private final Set columns = new TreeSet<>(new ColumnComparator()); + + private final List> rows = new ArrayList<>(); + + private final Stack processedTagStack = new Stack<>(); + + private void processTag(HtmlFormTag tag) { + ProcessedTag processedTag = getProcessedTag(tag); + if (!processedTag.getDataRows().isEmpty()) { + Map dataFromParents = new HashMap<>(); + for (ProcessedTag parentTag : processedTagStack) { + dataFromParents.putAll(parentTag.getDataForChildren()); + } + for (Map row : processedTag.getDataRows()) { + if (!row.isEmpty()) { + row.putAll(dataFromParents); + rows.add(row); + columns.addAll(row.keySet()); + } + } + } + if (processedTag.isProcessChildren()) { + processedTagStack.push(processedTag); + for (HtmlFormTag childTag : tag.getChildTags()) { + processTag(childTag); + } + processedTagStack.pop(); + } + } + + // This exists in the event that some tag needs to produce multiple rows, most will simply delegate to getDataRow + private ProcessedTag getProcessedTag(HtmlFormTag tag) { + ProcessedTag processedTag = new ProcessedTag(); + log.trace("Processing tag: " + tag.getName() + ": " + tag.getAttributes()); + Map dataRow = new HashMap<>(); + Map unprocessedValues = new HashMap<>(tag.getAttributes()); + boolean tagHandled = true; + switch (tag.getName()) { + + + // section: only contribute section as a column on child rows. only sectionName is relevant + case "section": { + ignoreAttributes(unprocessedValues, "id", "headerTag", "sectionTag", "headerStyle"); + String headerCode = unprocessedValues.remove("headerCode"); + String headerLabel = unprocessedValues.remove("headerLabel"); + processedTag.getDataForChildren().put("sectionName", getDisplay(headerCode, headerLabel)); + break; + } + + case "obs": { + + for (String obsField : AVAILABLE_COLUMNS) { + String value = unprocessedValues.remove(obsField); + if (StringUtils.isNotBlank(value)) { + dataRow.put(obsField, value); + } + } + + // There are situations where only an empty commentFieldLabel is configured to display the comment box, so we try to handle this + boolean obsCommentUsed = "true".equals(unprocessedValues.get("showCommentField")); + obsCommentUsed = obsCommentUsed || unprocessedValues.containsKey("commentFieldLabel"); + obsCommentUsed = obsCommentUsed || unprocessedValues.containsKey("commentFieldCode"); + if (obsCommentUsed) { + dataRow.put("obsCommentUsed", "true"); + } + + // Roll up any when tags + List whenConditions = new ArrayList<>(); + for (Iterator i = tag.getChildTags().iterator(); i.hasNext();) { + HtmlFormTag controlsTag = i.next(); + if (controlsTag.getName().equalsIgnoreCase("controls")) { + for (Iterator j = controlsTag.getChildTags().iterator(); j.hasNext();) { + HtmlFormTag whenTag = j.next(); + if (whenTag.getName().equalsIgnoreCase("when")) { + whenConditions.add(HtmlFormUtils.mapToString(whenTag.getAttributes(), "=", ",")); + j.remove(); + } + } + if (controlsTag.getChildTags().isEmpty() && controlsTag.getAttributes().isEmpty()) { + i.remove(); + } + } + } + dataRow.put("whenValueThen", String.join("|", whenConditions)); + + break; + } + + default: { + tagHandled = false; + log.debug("No specific handling found for tag: " + tag.getName()); + } + } + + if (!dataRow.isEmpty() || !unprocessedValues.isEmpty() || !tagHandled) { + if (shouldIncludeTag(tag)) { + dataRow.put("tagName", tag.getName()); + dataRow.put("additionalConfiguration", HtmlFormUtils.mapToString(unprocessedValues, "=", ",")); + processedTag.getDataRows().add(dataRow); + } + } + + log.trace("Processed tag: " + processedTag); + return processedTag; + } + + private boolean shouldIncludeTag(HtmlFormTag tag) { + for (Tags.Standard standardTag : Tags.Standard.values()) { + if (standardTag.getNodeName().equalsIgnoreCase(tag.getName())) { + return false; + } + } + return true; + } + + private String getDisplay(String messageCode, String messageText) { + if (StringUtils.isNotBlank(messageCode)) { + return messageCode; // TODO: Translate this? + } + return messageText == null ? "" : messageText; + } + + private void ignoreAttributes(Map attributes, String... keys) { + for (String key : keys) { + attributes.remove(key); + } + } + + public Set getColumns() { + return columns; + } + + public List> getRows() { + return rows; + } + + private static class ProcessedTag { + private boolean processChildren = true; + private List> dataRows = new ArrayList<>(); + private Map dataForChildren = new HashMap<>(); + + public boolean isProcessChildren() { + return processChildren; + } + + public void setProcessChildren(boolean processChildren) { + this.processChildren = processChildren; + } + + public List> getDataRows() { + return dataRows; + } + + public void setDataRows(List> dataRows) { + this.dataRows = dataRows; + } + + public Map getDataForChildren() { + return dataForChildren; + } + + public void setDataForChildren(Map dataForChildren) { + this.dataForChildren = dataForChildren; + } + } + + private static class ColumnComparator implements Comparator { + + @Override + public int compare(String s1, String s2) { + int index1 = AVAILABLE_COLUMNS.indexOf(s1); + int index2 = AVAILABLE_COLUMNS.indexOf(s2); + if (index1 == -1 && index2 != -1) { + return 1; + } + if (index1 != -1 && index2 == -1) { + return -1; + } + int ret = index1 - index2; + if (ret == 0) { + ret = s1.compareTo(s2); + } + return ret; + } + } + +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java new file mode 100644 index 000000000..d0d6427f1 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java @@ -0,0 +1,114 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.htmlformentry.HtmlFormEntryUtil; +import org.w3c.dom.CharacterData; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Stack; + +/** + * Loads and html form and parses it into tags + */ +public class HtmlFormParser { + + protected final Log log = LogFactory.getLog(getClass()); + + public static final String HTMLFORM_TAG_NAME = "htmlform"; + + private final HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + + protected final Stack tagStack = new Stack<>(); + + public HtmlFormTag parseFile(File formFile) { + try { + log.debug("Parsing htmlform file: " + formFile.getAbsolutePath()); + return parseXml(FileUtils.readFileToString(formFile, StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new IllegalArgumentException("Unable to load form from " + formFile.getName(), e); + } + } + + public HtmlFormTag parseXml(String formXml) { + try { + formXml = preprocessXml(formXml); + Document document = HtmlFormEntryUtil.stringToDocument(formXml); + Node htmlFormNode = HtmlFormEntryUtil.findChild(document, HTMLFORM_TAG_NAME); + HtmlFormTag htmlFormTag = processNode(htmlFormNode); + if (!tagStack.isEmpty()) { + throw new IllegalStateException("Tag Stack is not empty after processing. Found: " + tagStack); + } + return htmlFormTag; + } + catch (Exception e) { + throw new IllegalStateException("Unable to parse form xml", e); + } + } + + protected String preprocessXml(String xml) throws Exception { + return xml; + } + + protected HtmlFormTag processNode(Node node) { + HtmlFormTag tag = null; + if (node != null) { + tag = new HtmlFormTag(); + tag.setName(node.getNodeName()); + if (node instanceof CharacterData) { + CharacterData characterDataNode = (CharacterData) node; + tag.setData(characterDataNode.getData()); + } + NamedNodeMap attrs = node.getAttributes(); + if (attrs != null) { + for (int i = 0; i < attrs.getLength(); i++) { + Node attr = attrs.item(i); + tag.getAttributes().put(attr.getNodeName(), attr.getNodeValue()); + handleTagAttribute(tag, attr.getNodeName(), attr.getNodeValue()); + } + } + if (!tagStack.isEmpty()) { + tagStack.peek().getChildTags().add(tag); + } + handleTagStart(tag); + tagStack.push(tag); + NodeList childNodes = node.getChildNodes(); + for (int i=0; i getAllTagsByName(String name) { + List ret = new ArrayList<>(); + if (getName().equalsIgnoreCase(name)) { + ret.add(this); + } + for (HtmlFormTag childTag : getChildTags()) { + ret.addAll(childTag.getAllTagsByName(name)); + } + return ret; + } + + @Override + public String toString() { + return getName() + getAttributes(); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java new file mode 100644 index 000000000..044fdc99c --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java @@ -0,0 +1,118 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.openmrs.Concept; +import org.openmrs.module.htmlformentry.handler.AttributeDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +public class HtmlFormTagConverter { + + private final Logger log = LoggerFactory.getLogger(HtmlFormTagConverter.class); + + public HtmlFormTag removeTagsByName(HtmlFormTag tag, String tagName) { + HtmlFormTag convertedTag = tag.cloneWithoutChildren(); + for (HtmlFormTag childTag : tag.getChildTags()) { + childTag = removeTagsByName(childTag, tagName); + if (childTag.getName().equalsIgnoreCase(tagName)) { + if (!childTag.getChildTags().isEmpty()) { + convertedTag.getChildTags().addAll(childTag.getChildTags()); + } + } + else { + convertedTag.getChildTags().add(childTag); + } + } + return convertedTag; + } + + public HtmlFormTag removeIfModeTags(HtmlFormTag tag) { + HtmlFormTag convertedTag = tag.cloneWithoutChildren(); + for (HtmlFormTag childTag : tag.getChildTags()) { + childTag = removeIfModeTags(childTag); + if (childTag.getName().equalsIgnoreCase("ifMode")) { + boolean viewMode = childTag.hasAttribute("mode", "view"); + boolean include = !childTag.hasAttribute("include", "false"); + if ((viewMode && include) || (!viewMode && !include)) { + log.trace("Excluding content that is configured only for view mode: " + childTag); + } + else { + convertedTag.getChildTags().addAll(childTag.getChildTags()); + } + } + else { + convertedTag.getChildTags().add(childTag); + } + } + return convertedTag; + } + + public HtmlFormTag mergeTagIntoChildren(HtmlFormTag tag, String tagName) { + HtmlFormTag convertedTag = tag.cloneWithoutChildren(); + for (HtmlFormTag childTag : tag.getChildTags()) { + if (tag.getName().equalsIgnoreCase(tagName)) { + for (String attribute : tag.getAttributes().keySet()) { + childTag.addAttribute(attribute, tag.getAttributes().get(attribute), true); + } + } + convertedTag.getChildTags().add(mergeTagIntoChildren(childTag, tagName)); + } + return removeTagsByName(convertedTag, tagName); + } + + public HtmlFormTag mergeSiblings(HtmlFormTag tag, String tagName) { + HtmlFormTag convertedTag = tag.cloneWithoutChildren(); + HtmlFormTag matchingTag = null; + for (HtmlFormTag childTag : tag.getChildTags()) { + childTag = mergeSiblings(childTag, tagName); + if (childTag.getName().equalsIgnoreCase(tagName)) { + if (!childTag.getChildTags().isEmpty()) { + throw new IllegalStateException("You cannot merge siblings that have children"); + } + if (matchingTag == null) { + matchingTag = childTag; + convertedTag.getChildTags().add(matchingTag); + } + else { + Set attributes = new LinkedHashSet<>(matchingTag.getAttributes().keySet()); + attributes.addAll(childTag.getAttributes().keySet()); + for (String attribute : attributes) { + Set values = new HashSet<>(); + values.add(matchingTag.getAttributes().get(attribute)); + values.add(childTag.getAttributes().get(attribute)); + matchingTag.getAttributes().put(attribute, String.join(" || ", values)); + } + } + } + else { + convertedTag.getChildTags().add(childTag); + } + } + return convertedTag; + } + + public HtmlFormTag mergeObsGroupIntoChildren(HtmlFormTag tag) { + HtmlFormTag convertedTag = tag.cloneWithoutChildren(); + for (HtmlFormTag childTag : tag.getChildTags()) { + childTag = mergeObsGroupIntoChildren(childTag); + if (childTag.getName().equalsIgnoreCase("obsGroup")) { + String hiddenConceptId = childTag.getAttributes().remove("hiddenConceptId"); + String hiddenAnswerConceptId = childTag.getAttributes().remove("hiddenAnswerConceptId"); + if (hiddenConceptId != null || hiddenAnswerConceptId != null) { + HtmlFormTag hiddenTag = new HtmlFormTag(); + hiddenTag.setName("obs"); + hiddenTag.getAttributes().put("hidden", "true"); + hiddenTag.getAttributes().put("conceptId", hiddenConceptId); + hiddenTag.getAttributes().put("answerConceptId", hiddenAnswerConceptId); + childTag.getChildTags().add(hiddenTag); + } + } + convertedTag.getChildTags().add(childTag); + } + return mergeTagIntoChildren(convertedTag, "obsGroup"); + } + +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java new file mode 100644 index 000000000..1c6ea2115 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java @@ -0,0 +1,67 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import com.opencsv.CSVWriter; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class HtmlFormTagCounter { + + private final Map tagCounter = new TreeMap<>(); + private final Map attributeCounter = new TreeMap<>(); + + public HtmlFormTagCounter() { } + + public void analyze(File fileOrDirectory) { + HtmlFormParser parser = new HtmlFormParser() { + @Override + protected void handleTagStart(HtmlFormTag tag) { + super.handleTagStart(tag); + tagCounter.put(tag.getName(), tagCounter.getOrDefault(tag.getName(), 0) + 1); + } + @Override + protected void handleTagAttribute(HtmlFormTag tag, String attributeName, String attributeValue) { + super.handleTagAttribute(tag, attributeName, attributeValue); + String name = tag.getName() + "." + attributeName; + attributeCounter.put(name, attributeCounter.getOrDefault(name, 0) + 1); + } + }; + if (fileOrDirectory.isFile()) { + parser.parseFile(fileOrDirectory); + } + else if (fileOrDirectory.isDirectory()) { + List files = HtmlFormUtils.getNestedFilesBySuffix(fileOrDirectory, "xml"); + for (File file : files) { + parser.parseFile(file); + } + } + } + + public void writeToCsv(File outputFile) throws IOException { + try (FileWriter fileWriter = new FileWriter(outputFile)) { + try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { + String[] csvColumns = {"tagName", "attributeName", "count"}; + csvWriter.writeNext(csvColumns); + for (String element : tagCounter.keySet()) { + csvWriter.writeNext(new String[] {element, "", Integer.toString(tagCounter.get(element))}); + } + for (String element : attributeCounter.keySet()) { + String[] split = element.split("\\."); + csvWriter.writeNext(new String[] {split[0], split[1], Integer.toString(attributeCounter.get(element))}); + } + } + } + } + + public Map getTagCounter() { + return tagCounter; + } + + public Map getAttributeCounter() { + return attributeCounter; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java new file mode 100644 index 000000000..842edc84e --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java @@ -0,0 +1,46 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * A tag represents a specific xml node in a html form + */ +public class HtmlFormUtils { + + public static List getNestedFilesBySuffix(File directory, String suffix) { + List files = new ArrayList<>(); + if (directory.exists() && directory.isDirectory()) { + File[] filesInDirectory = directory.listFiles(); + if (filesInDirectory != null) { + for (File f : filesInDirectory) { + if (f.isDirectory()) { + files.addAll(getNestedFilesBySuffix(f, suffix)); + } else { + if (f.getName().endsWith(suffix)) { + files.add(f); + } + } + } + } + } + return files; + } + + public static String mapToString(Map m, String keyValueSeparator, String entrySeparator) { + List ret = new ArrayList<>(); + for (String key : m.keySet()) { + ret.add(key + keyValueSeparator + m.get(key)); + } + return String.join(entrySeparator, ret); + } + + public static void printTag(HtmlFormTag tag, String prefix) { + System.out.println(prefix + tag.toString()); + for (HtmlFormTag childTag : tag.getChildTags()) { + printTag(childTag, prefix + " "); + } + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java new file mode 100644 index 000000000..24a5a61a2 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java @@ -0,0 +1,35 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import lombok.Getter; + +public class Tags { + + public enum HFE { + appointments, causeOfDeathList, code, codedOrFreeTextObs, completeProgram, condition, controls, drugOrder, + encounterDate, encounterDiagnosesByObs, encounterDisposition, encounterLocation, encounterProvider, + encounterProviderAndRole, encounterType, encounterVoided, enrollInProgram, excludeIf, exitFromCare, + familyHistoryRelativeCheckboxes, htmlform, ifMode, immunization, includeIf, lookup, macro, macros, + markPatientDead, obs, obsgroup, obsreference, option, order, orderProperty, orderTemplate, + pastMedicalHistoryCheckbox, patient, postSubmissionAction, redirectOnSave, relationship, + render, repeat, section, submit, template, translations, uiInclude, uimessage, uimessages, + variant, when, workflowState + } + + @Getter + public enum Standard { + + a, b, br, button, center, comment("#comment"), dd, div, dl, dt, field, fieldset, font, + h1, h2, h3, h4, h5, h6, h7, hr, i, img, input, label, legend, li, p, script, select, sl, small, span, + strong, style, table, tbody, td, text("#text"), th, thead, tr, u, ul; + + private final String nodeName; + + Standard() { + nodeName = name(); + } + + Standard(String nodeName) { + this.nodeName = nodeName; + } + } +} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java new file mode 100644 index 000000000..990763ae2 --- /dev/null +++ b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java @@ -0,0 +1,101 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import com.opencsv.CSVWriter; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class HtmlFormCsvExporterTest { + + private static final Logger log = LoggerFactory.getLogger(HtmlFormNonMemoryTest.class); + + public File getHtmlFormsDirectory() { + return new File("/home/mseaton/code/github/pih/openmrs-config-pihsl/target/openmrs-packager-config/configuration/pih/htmlforms"); + } + + @Test + public void execute() throws Exception { + generateCsvForForm(new File(getHtmlFormsDirectory(), "mentalHealthFollowup.xml")); + } + + /* + public void generateTagSummary() throws Exception { + List htmlFormFiles = HtmlFormUtils.getNestedFilesBySuffix(getHtmlFormsDirectory(), ".xml"); + Map parsedForms = new TreeMap<>(); + for (File inputFile : htmlFormFiles) { + try { + HtmlFormCsvExporter parser = new HtmlFormCsvExporter(); + parser.parseFile(inputFile); + parsedForms.put(inputFile.getName(), parser); + } + catch (Exception e) { + log.error("Error parsing form: " + inputFile.getName()); + } + } + File outputFile = new File(getHtmlFormsDirectory(), "formSummary.csv"); + try (FileWriter fileWriter = new FileWriter(outputFile)) { + try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { + String[] csvColumns = {"form", "tag", "comments"}; + csvWriter.writeNext(csvColumns); + for (String formName : parsedForms.keySet()) { + HtmlFormCsvExporter parser = parsedForms.get(formName); + for (Map row : parser.getRows()) { + String[] data = new String[csvColumns.length]; + data[0] = formName; + data[1] = row.get("tagName"); + data[2] = row.get("additionalConfiguration"); + csvWriter.writeNext(data); + } + } + } + } + } + + */ + + public void generateCsvForAllForms() { + List htmlFormFiles = HtmlFormUtils.getNestedFilesBySuffix(getHtmlFormsDirectory(), ".xml"); + for (File inputFile : htmlFormFiles) { + try { + generateCsvForForm(inputFile); + } + catch (Exception e) { + log.error("Unable to generate CSV for form: " + inputFile.getName()); + } + } + } + + public void generateCsvForForm(File inputFile) throws Exception { + File outputFile = new File(inputFile.getAbsolutePath() + ".csv"); + log.info("Generating schema for " + inputFile.getName()); + log.info("TagRegister loaded for " + inputFile.getName()); + HtmlFormCsvExporter exporter = new HtmlFormCsvExporter(); + exporter.export(inputFile); + /* + String[] columns = parser.getColumns().toArray(new String[0]); + List> rows = parser.getRows(); + log.info("TagRegister parsed into " + columns.length + " columns and " + rows.size() + " rows"); + try (FileWriter fileWriter = new FileWriter(outputFile)) { + try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { + csvWriter.writeNext(columns); // Header + for (Map row : rows) { + String[] rowArray = new String[columns.length]; + for (int i=0; i tagHandlers = Context.getService(HtmlFormEntryService.class).getHandlers(); + Set nodeNames = new TreeSet<>(tagHandlers.keySet()); + for (String nodeName : nodeNames) { + System.out.println(nodeName); + } + } + + public static FormEntrySession getFormEntrySession(HtmlForm htmlForm) { + try { + Patient patient = HtmlFormEntryUtil.getFakePerson(); + HttpSession session = new MockHttpSession(); + session.setAttribute("emrContext.sessionLocationId", Context.getLocationService().getAllLocations().get(0)); + FormEntrySession s = new FormEntrySession(patient, null, FormEntryContext.Mode.ENTER, htmlForm, session); + s.getHtmlToDisplay(); + return s; + } + catch (Exception e) { + throw new IllegalStateException("Unable to get htmlform schema: ", e); + } + } + + public HtmlForm getHtmlFrom(String path) throws Exception { + HtmlForm htmlform = new HtmlForm(); + Form form = new Form(); + htmlform.setForm(form); + form.setEncounterType(new EncounterType()); + htmlform.setDateCreated(new Date()); + htmlform.setXmlData(FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8)); + return htmlform; + } +} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounterTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounterTest.java new file mode 100644 index 000000000..6b029132b --- /dev/null +++ b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounterTest.java @@ -0,0 +1,19 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.junit.jupiter.api.Test; + +import java.io.File; + +public class HtmlFormTagCounterTest { + + public File getHtmlFormsDirectory() { + return new File("/home/mseaton/code/github/pih/openmrs-config-pihsl/target/openmrs-packager-config/configuration/pih/htmlforms"); + } + + @Test + public void execute() throws Exception { + HtmlFormTagCounter analyzer = new HtmlFormTagCounter(); + analyzer.analyze(getHtmlFormsDirectory()); + analyzer.writeToCsv(new File(getHtmlFormsDirectory(), "tagAnalysis.csv")); + } +} diff --git a/pom.xml b/pom.xml index 0cb7de4e4..076a82997 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ 2.0.7 2.45.0-SNAPSHOT + 1.18.26 1.20.1 UTF-8 @@ -738,6 +739,13 @@ provided + + org.projectlombok + lombok + ${lombokVersion} + provided + + From e2a93b8485fb758e6b9728ea68d2aa197af6870d Mon Sep 17 00:00:00 2001 From: Michael Seaton Date: Sat, 31 Aug 2024 09:36:08 -0400 Subject: [PATCH 2/5] Move converter utilities into mappers and reducers --- .../analysis/HtmlFormCsvExporter.java | 65 +++++++--- .../analysis/HtmlFormTagConverter.java | 118 ------------------ .../analysis/HtmlFormTagCounter.java | 18 ++- .../analysis/mapper/IfModeTagRemover.java | 28 +++++ .../analysis/mapper/ObsGroupMapper.java | 31 +++++ .../analysis/mapper/ParentToChildMerger.java | 38 ++++++ .../analysis/mapper/SectionMerger.java | 25 ++++ .../analysis/mapper/SiblingMerger.java | 50 ++++++++ .../analysis/mapper/TagMapper.java | 7 ++ .../analysis/mapper/TagRemover.java | 32 +++++ .../analysis/reducer/TagCounter.java | 48 +++++++ .../analysis/reducer/TagReducer.java | 7 ++ 12 files changed, 330 insertions(+), 137 deletions(-) delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ParentToChildMerger.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java index 24e2f3a2b..70e692302 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java @@ -4,6 +4,14 @@ import org.openmrs.module.htmlformentry.FormEntryContext; import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; import org.openmrs.module.htmlformentry.Translator; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.IfModeTagRemover; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.ObsGroupMapper; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.ParentToChildMerger; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.SectionMerger; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.SiblingMerger; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.TagMapper; +import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.TagRemover; +import org.openmrs.module.pihcore.htmlformentry.analysis.reducer.TagCounter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,31 +33,56 @@ public class HtmlFormCsvExporter { public void export(File file) { HtmlFormTag tag = loadFormFromFile(file); - HtmlFormTagConverter converter = new HtmlFormTagConverter(); + + List mappers = new ArrayList<>(); + + // Remove any standard html tags for (Tags.Standard standardTag : Tags.Standard.values()) { - tag = converter.removeTagsByName(tag, standardTag.getNodeName()); + mappers.add(new TagRemover(standardTag.getNodeName())); } - tag = converter.removeTagsByName(tag, "uimessage"); - tag = converter.removeTagsByName(tag, "lookup"); - tag = converter.removeTagsByName(tag, "submit"); - tag = converter.removeIfModeTags(tag); - tag = converter.mergeTagIntoChildren(tag, "includeIf"); - tag = converter.mergeTagIntoChildren(tag, "excludeIf"); - // Handle obs group - tag = converter.mergeObsGroupIntoChildren(tag); + // Remove tags that do not impact the data model and cannot be easily used as context for other elements + mappers.add(new TagRemover("uimessage")); + mappers.add(new TagRemover("lookup")); + mappers.add(new TagRemover("submit")); + + // Remove any sections of the form that are added *just* for view mode + mappers.add(new IfModeTagRemover()); + + // Flatten any includeIf or excludeIf expressions down into the child tags + mappers.add(new ParentToChildMerger("includeIf")); + mappers.add(new ParentToChildMerger("excludeIf")); // Handle the complex web of velocity conditions for entry of date/location/provider - tag = converter.mergeSiblings(tag, "encounterDate"); - tag = converter.mergeSiblings(tag, "encounterLocation"); - tag = converter.mergeSiblings(tag, "encounterProviderAndRole"); + mappers.add(new SiblingMerger("encounterDate")); + mappers.add(new SiblingMerger("encounterLocation")); + mappers.add(new SiblingMerger("encounterProviderAndRole")); + + // Merge obs groups attributes down into children, and add any hidden obs + mappers.add(new ObsGroupMapper()); + + // Flatten sections down into the child tags + mappers.add(new SectionMerger()); // Handle when/then tags - tag = converter.mergeSiblings(tag, "when"); - tag = converter.removeTagsByName(tag,"controls"); + mappers.add(new SiblingMerger("when")); + tag = new TagRemover("controls").map(tag); + + for (TagMapper mapper : mappers) { + tag = mapper.map(tag); + } + TagCounter tagCounter = new TagCounter(); + Map> counters = tagCounter.reduce(tag); + for (String counter : counters.keySet()) { + Map counterValues = counters.get(counter); + for (String key : counterValues.keySet()) { + Integer count = counterValues.get(key); + System.out.println(counter + ", " + key + ", " + count); + } + } - HtmlFormUtils.printTag(tag, ""); + //HtmlFormUtils.printTag(tag, ""); } protected HtmlFormTag loadFormFromFile(File file) { diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java deleted file mode 100644 index 044fdc99c..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagConverter.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis; - -import org.openmrs.Concept; -import org.openmrs.module.htmlformentry.handler.AttributeDescriptor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.Set; - -public class HtmlFormTagConverter { - - private final Logger log = LoggerFactory.getLogger(HtmlFormTagConverter.class); - - public HtmlFormTag removeTagsByName(HtmlFormTag tag, String tagName) { - HtmlFormTag convertedTag = tag.cloneWithoutChildren(); - for (HtmlFormTag childTag : tag.getChildTags()) { - childTag = removeTagsByName(childTag, tagName); - if (childTag.getName().equalsIgnoreCase(tagName)) { - if (!childTag.getChildTags().isEmpty()) { - convertedTag.getChildTags().addAll(childTag.getChildTags()); - } - } - else { - convertedTag.getChildTags().add(childTag); - } - } - return convertedTag; - } - - public HtmlFormTag removeIfModeTags(HtmlFormTag tag) { - HtmlFormTag convertedTag = tag.cloneWithoutChildren(); - for (HtmlFormTag childTag : tag.getChildTags()) { - childTag = removeIfModeTags(childTag); - if (childTag.getName().equalsIgnoreCase("ifMode")) { - boolean viewMode = childTag.hasAttribute("mode", "view"); - boolean include = !childTag.hasAttribute("include", "false"); - if ((viewMode && include) || (!viewMode && !include)) { - log.trace("Excluding content that is configured only for view mode: " + childTag); - } - else { - convertedTag.getChildTags().addAll(childTag.getChildTags()); - } - } - else { - convertedTag.getChildTags().add(childTag); - } - } - return convertedTag; - } - - public HtmlFormTag mergeTagIntoChildren(HtmlFormTag tag, String tagName) { - HtmlFormTag convertedTag = tag.cloneWithoutChildren(); - for (HtmlFormTag childTag : tag.getChildTags()) { - if (tag.getName().equalsIgnoreCase(tagName)) { - for (String attribute : tag.getAttributes().keySet()) { - childTag.addAttribute(attribute, tag.getAttributes().get(attribute), true); - } - } - convertedTag.getChildTags().add(mergeTagIntoChildren(childTag, tagName)); - } - return removeTagsByName(convertedTag, tagName); - } - - public HtmlFormTag mergeSiblings(HtmlFormTag tag, String tagName) { - HtmlFormTag convertedTag = tag.cloneWithoutChildren(); - HtmlFormTag matchingTag = null; - for (HtmlFormTag childTag : tag.getChildTags()) { - childTag = mergeSiblings(childTag, tagName); - if (childTag.getName().equalsIgnoreCase(tagName)) { - if (!childTag.getChildTags().isEmpty()) { - throw new IllegalStateException("You cannot merge siblings that have children"); - } - if (matchingTag == null) { - matchingTag = childTag; - convertedTag.getChildTags().add(matchingTag); - } - else { - Set attributes = new LinkedHashSet<>(matchingTag.getAttributes().keySet()); - attributes.addAll(childTag.getAttributes().keySet()); - for (String attribute : attributes) { - Set values = new HashSet<>(); - values.add(matchingTag.getAttributes().get(attribute)); - values.add(childTag.getAttributes().get(attribute)); - matchingTag.getAttributes().put(attribute, String.join(" || ", values)); - } - } - } - else { - convertedTag.getChildTags().add(childTag); - } - } - return convertedTag; - } - - public HtmlFormTag mergeObsGroupIntoChildren(HtmlFormTag tag) { - HtmlFormTag convertedTag = tag.cloneWithoutChildren(); - for (HtmlFormTag childTag : tag.getChildTags()) { - childTag = mergeObsGroupIntoChildren(childTag); - if (childTag.getName().equalsIgnoreCase("obsGroup")) { - String hiddenConceptId = childTag.getAttributes().remove("hiddenConceptId"); - String hiddenAnswerConceptId = childTag.getAttributes().remove("hiddenAnswerConceptId"); - if (hiddenConceptId != null || hiddenAnswerConceptId != null) { - HtmlFormTag hiddenTag = new HtmlFormTag(); - hiddenTag.setName("obs"); - hiddenTag.getAttributes().put("hidden", "true"); - hiddenTag.getAttributes().put("conceptId", hiddenConceptId); - hiddenTag.getAttributes().put("answerConceptId", hiddenAnswerConceptId); - childTag.getChildTags().add(hiddenTag); - } - } - convertedTag.getChildTags().add(childTag); - } - return mergeTagIntoChildren(convertedTag, "obsGroup"); - } - -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java index 1c6ea2115..66cdee051 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -12,6 +13,7 @@ public class HtmlFormTagCounter { private final Map tagCounter = new TreeMap<>(); + private final Map flatTagCounter = new TreeMap<>(); private final Map attributeCounter = new TreeMap<>(); public HtmlFormTagCounter() { } @@ -22,6 +24,13 @@ public void analyze(File fileOrDirectory) { protected void handleTagStart(HtmlFormTag tag) { super.handleTagStart(tag); tagCounter.put(tag.getName(), tagCounter.getOrDefault(tag.getName(), 0) + 1); + List names = new ArrayList<>(); + for (HtmlFormTag tagInStack : tagStack) { + names.add(tagInStack.getName()); + } + names.add(tag.getName()); + String flatName = String.join(".", names); + flatTagCounter.put(flatName, tagCounter.getOrDefault(flatName, 0) + 1); } @Override protected void handleTagAttribute(HtmlFormTag tag, String attributeName, String attributeValue) { @@ -44,14 +53,17 @@ else if (fileOrDirectory.isDirectory()) { public void writeToCsv(File outputFile) throws IOException { try (FileWriter fileWriter = new FileWriter(outputFile)) { try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { - String[] csvColumns = {"tagName", "attributeName", "count"}; + String[] csvColumns = {"counter", "tagName", "attributeName", "count"}; csvWriter.writeNext(csvColumns); for (String element : tagCounter.keySet()) { - csvWriter.writeNext(new String[] {element, "", Integer.toString(tagCounter.get(element))}); + csvWriter.writeNext(new String[] {"tags", element, "", Integer.toString(tagCounter.get(element))}); + } + for (String element : flatTagCounter.keySet()) { + csvWriter.writeNext(new String[] {"flatTags", element, "", Integer.toString(flatTagCounter.get(element))}); } for (String element : attributeCounter.keySet()) { String[] split = element.split("\\."); - csvWriter.writeNext(new String[] {split[0], split[1], Integer.toString(attributeCounter.get(element))}); + csvWriter.writeNext(new String[] {"tagAttributes", split[0], split[1], Integer.toString(attributeCounter.get(element))}); } } } diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java new file mode 100644 index 000000000..9c1a8815d --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java @@ -0,0 +1,28 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +@Data +public class IfModeTagRemover implements TagMapper { + + @Override + public HtmlFormTag map(HtmlFormTag inputTag) { + HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + for (HtmlFormTag childTag : inputTag.getChildTags()) { + childTag = map(childTag); + if (childTag.getName().equalsIgnoreCase("ifMode")) { + boolean viewMode = childTag.hasAttribute("mode", "view"); + boolean include = !childTag.hasAttribute("include", "false"); + boolean viewOnlyNode = (viewMode && include) || (!viewMode && !include); + if (!viewOnlyNode) { + outputTag.getChildTags().addAll(childTag.getChildTags()); + } + } + else { + outputTag.getChildTags().add(childTag); + } + } + return outputTag; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java new file mode 100644 index 000000000..81bee783f --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java @@ -0,0 +1,31 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +@Data +public class ObsGroupMapper implements TagMapper { + + @Override + public HtmlFormTag map(HtmlFormTag inputTag) { + HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + for (HtmlFormTag childTag : inputTag.getChildTags()) { + childTag = map(childTag); + if (childTag.getName().equalsIgnoreCase("obsGroup")) { + String hiddenConceptId = childTag.getAttributes().remove("hiddenConceptId"); + String hiddenAnswerConceptId = childTag.getAttributes().remove("hiddenAnswerConceptId"); + if (hiddenConceptId != null || hiddenAnswerConceptId != null) { + HtmlFormTag hiddenTag = new HtmlFormTag(); + hiddenTag.setName("obs"); + hiddenTag.getAttributes().put("hidden", "true"); + hiddenTag.getAttributes().put("conceptId", hiddenConceptId); + hiddenTag.getAttributes().put("answerConceptId", hiddenAnswerConceptId); + childTag.getChildTags().add(hiddenTag); + } + } + outputTag.getChildTags().add(childTag); + } + ParentToChildMerger parentToChildMerger = new ParentToChildMerger("obsGroup"); + return parentToChildMerger.map(outputTag); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ParentToChildMerger.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ParentToChildMerger.java new file mode 100644 index 000000000..7a69bdf7a --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ParentToChildMerger.java @@ -0,0 +1,38 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class ParentToChildMerger implements TagMapper { + + private String tagName; + + public ParentToChildMerger(String tagName) { + this.tagName = tagName; + } + + @Override + public HtmlFormTag map(HtmlFormTag inputTag) { + HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + for (HtmlFormTag childTag : inputTag.getChildTags()) { + if (inputTag.getName().equalsIgnoreCase(tagName)) { + Map attributesToAdd = computeAttributesToMerge(new HashMap<>(inputTag.getAttributes())); + for (Map.Entry attribute : attributesToAdd.entrySet()) { + childTag.addAttribute(attribute.getKey(), attribute.getValue(), true); + } + } + childTag = map(childTag); + outputTag.getChildTags().add(childTag); + } + outputTag = new TagRemover(tagName).map(outputTag); + return outputTag; + } + + public Map computeAttributesToMerge(Map inputAttributes) { + return inputAttributes; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java new file mode 100644 index 000000000..4a0d4674a --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java @@ -0,0 +1,25 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import org.apache.commons.lang.StringUtils; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormUtils; + +import java.util.HashMap; +import java.util.Map; + +public class SectionMerger extends ParentToChildMerger { + + public SectionMerger() { + super("section"); + } + + @Override + public Map computeAttributesToMerge(Map inputAttributes) { + String section = inputAttributes.get("headerCode"); + if (StringUtils.isBlank(section)) { + section = HtmlFormUtils.mapToString(inputAttributes, "=", ","); + } + Map ret = new HashMap<>(); + ret.put("section", section); + return ret; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java new file mode 100644 index 000000000..57cdc9313 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java @@ -0,0 +1,50 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +@Data +public class SiblingMerger implements TagMapper { + + private String tagName; + + public SiblingMerger(String tagName) { + this.tagName = tagName; + } + + @Override + public HtmlFormTag map(HtmlFormTag inputTag) { + HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + HtmlFormTag matchingTag = null; + for (HtmlFormTag childTag : inputTag.getChildTags()) { + childTag = map(childTag); + if (childTag.getName().equalsIgnoreCase(tagName)) { + if (!childTag.getChildTags().isEmpty()) { + throw new IllegalStateException("You cannot merge siblings that have children"); + } + if (matchingTag == null) { + matchingTag = childTag; + outputTag.getChildTags().add(matchingTag); + } + else { + Set attributes = new LinkedHashSet<>(matchingTag.getAttributes().keySet()); + attributes.addAll(childTag.getAttributes().keySet()); + for (String attribute : attributes) { + Set values = new HashSet<>(); + values.add(matchingTag.getAttributes().get(attribute)); + values.add(childTag.getAttributes().get(attribute)); + matchingTag.getAttributes().put(attribute, String.join(" || ", values)); + } + } + } + else { + outputTag.getChildTags().add(childTag); + } + } + return outputTag; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java new file mode 100644 index 000000000..73f1506b6 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java @@ -0,0 +1,7 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +public interface TagMapper { + HtmlFormTag map(HtmlFormTag inputTag); +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java new file mode 100644 index 000000000..30249b981 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java @@ -0,0 +1,32 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +@Data +public class TagRemover implements TagMapper { + + private String tagName; + private boolean removeChildren; + + public TagRemover(String tagName) { + this.tagName = tagName; + } + + @Override + public HtmlFormTag map(HtmlFormTag inputTag) { + HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + for (HtmlFormTag childTag : inputTag.getChildTags()) { + childTag = map(childTag); + if (childTag.getName().equalsIgnoreCase(tagName)) { + if (!removeChildren) { + outputTag.getChildTags().addAll(childTag.getChildTags()); + } + } + else { + outputTag.getChildTags().add(childTag); + } + } + return outputTag; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java new file mode 100644 index 000000000..d4a49fba0 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java @@ -0,0 +1,48 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.reducer; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.TreeMap; + +@Data +public class TagCounter implements TagReducer>> { + + @Override + public Map> reduce(HtmlFormTag inputTag) { + Map> counters = new TreeMap<>(); + Stack stack = new Stack<>(); + reduce(inputTag, counters, stack); + return counters; + } + + private void reduce(HtmlFormTag inputTag, Map> counters, Stack stack) { + Map tagCounter = counters.computeIfAbsent("tags", k -> new TreeMap<>()); + tagCounter.put(inputTag.getName(), tagCounter.getOrDefault(inputTag.getName(), 0) + 1); + + List names = new ArrayList<>(); + for (HtmlFormTag tagInStack : stack) { + names.add(tagInStack.getName()); + } + names.add(inputTag.getName()); + String flatName = String.join(".", names); + Map flatTagCounter = counters.computeIfAbsent("flatTags", k -> new TreeMap<>()); + flatTagCounter.put(flatName, flatTagCounter.getOrDefault(flatName, 0) + 1); + + Map attributeCounter = counters.computeIfAbsent("attributes", k -> new TreeMap<>()); + for (String attributeName : inputTag.getAttributes().keySet()) { + String name = inputTag.getName() + "." + attributeName; + attributeCounter.put(name, attributeCounter.getOrDefault(name, 0) + 1); + } + + stack.push(inputTag); + for (HtmlFormTag childTag : inputTag.getChildTags()) { + reduce(childTag, counters, stack); + } + stack.pop(); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java new file mode 100644 index 000000000..18f4de779 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java @@ -0,0 +1,7 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.reducer; + +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +public interface TagReducer { + T reduce(HtmlFormTag inputTag); +} From 670abf7118e7d4f3a3f92ced3b07cb11053777af Mon Sep 17 00:00:00 2001 From: Michael Seaton Date: Tue, 3 Sep 2024 19:37:21 -0400 Subject: [PATCH 3/5] More refactoring --- .../htmlformentry/analysis/DataSet.java | 70 ++++ .../analysis/HtmlFormAnalyzer.java | 118 +++++++ .../analysis/HtmlFormCsvExporter.java | 317 ------------------ .../analysis/HtmlFormParser.java | 114 ------- .../htmlformentry/analysis/HtmlFormTag.java | 18 +- .../analysis/HtmlFormTagCounter.java | 79 ----- .../analysis/converter/Converter.java | 5 + .../converter/file/FileToStringConverter.java | 14 + .../tag/FlattenObsGroupsConverter.java} | 17 +- .../tag/FlattenOrExcludeIfModeConverter.java} | 13 +- .../tag/MergeParentToChildrenConverter.java | 41 +++ .../tag/MergeSectionToChildrenConverter.java} | 8 +- .../tag/MergeSiblingsConverter.java} | 15 +- .../analysis/converter/tag/ObsConverter.java | 21 ++ .../tag/RemoveTagByNameConverter.java} | 15 +- .../converter/xml/ApplyMacrosConverter.java | 15 + .../converter/xml/ApplyRepeatsConverter.java | 15 + .../xml/ApplyTranslationsConverter.java | 28 ++ .../converter/xml/StripCommentsConverter.java | 15 + .../converter/xml/XmlToTagConverter.java | 69 ++++ .../analysis/mapper/ParentToChildMerger.java | 38 --- .../analysis/mapper/TagMapper.java | 7 - .../{reducer => processor}/TagCounter.java | 28 +- .../analysis/processor/TagExporter.java | 107 ++++++ .../analysis/processor/TagProcessor.java | 10 + .../analysis/reducer/TagReducer.java | 7 - .../analysis/HtmlFormAnalyzerTest.java | 31 ++ .../analysis/HtmlFormCsvExporterTest.java | 101 ------ .../analysis/HtmlFormTagCounterTest.java | 19 -- 29 files changed, 627 insertions(+), 728 deletions(-) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/Converter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/file/FileToStringConverter.java rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{mapper/ObsGroupMapper.java => converter/tag/FlattenObsGroupsConverter.java} (60%) rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{mapper/IfModeTagRemover.java => converter/tag/FlattenOrExcludeIfModeConverter.java} (59%) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeParentToChildrenConverter.java rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{mapper/SectionMerger.java => converter/tag/MergeSectionToChildrenConverter.java} (71%) rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{mapper/SiblingMerger.java => converter/tag/MergeSiblingsConverter.java} (73%) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/ObsConverter.java rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{mapper/TagRemover.java => converter/tag/RemoveTagByNameConverter.java} (51%) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyMacrosConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyRepeatsConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyTranslationsConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/StripCommentsConverter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/XmlToTagConverter.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ParentToChildMerger.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java rename api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/{reducer => processor}/TagCounter.java (59%) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagExporter.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagProcessor.java delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java create mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java delete mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java delete mode 100644 api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormTagCounterTest.java diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java new file mode 100644 index 000000000..7b4d580fb --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java @@ -0,0 +1,70 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import com.opencsv.CSVWriter; +import lombok.Data; + +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Data +public class DataSet { + + private String name; + private List columns = new ArrayList<>(); + private List> rows = new ArrayList<>(); + + public DataSet() {} + + @Override + public String toString() { + return name + " (" + columns.size() + " x " + rows.size() + ")"; + } + + public void setHeaderRow(String... values) { + columns = Arrays.asList(values); + } + + public void addDataRow(String... values) { + if (values == null || columns == null || values.length != columns.size()) { + throw new IllegalArgumentException("Values and columns must have same length"); + } + Map row = new LinkedHashMap<>(); + for (int i=0; i row : rows) { + String[] r = new String[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + r[i] = row.get(columns.get(i)); + } + writer.writeNext(r); + } + } + } + } + + public void print(String separator) { + System.out.println(name); + System.out.println("===================================="); + System.out.println(String.join(separator, columns)); + for (Map row : rows) { + List values = new ArrayList<>(); + for (String c : columns) { + values.add(row.get(c)); + } + System.out.println(String.join(separator, values)); + } + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java new file mode 100644 index 000000000..ae05d029e --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java @@ -0,0 +1,118 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.file.FileToStringConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.FlattenObsGroupsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.FlattenOrExcludeIfModeConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.MergeParentToChildrenConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.MergeSectionToChildrenConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.MergeSiblingsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag.RemoveTagByNameConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.ApplyMacrosConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.ApplyRepeatsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.ApplyTranslationsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.StripCommentsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.XmlToTagConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class HtmlFormAnalyzer { + + private final Logger log = LoggerFactory.getLogger(HtmlFormAnalyzer.class); + + public DataSet analyze(File fileOrDirectory, TagProcessor processor) throws Exception { + log.info("Analysis started for " + fileOrDirectory.getAbsolutePath() + " with processor: " + processor); + List tags = new ArrayList<>(); + if (fileOrDirectory.exists()) { + if (fileOrDirectory.isFile()) { + tags.add(convertFileToTag(fileOrDirectory)); + } + else if (fileOrDirectory.isDirectory()) { + for (File file : HtmlFormUtils.getNestedFilesBySuffix(fileOrDirectory, "xml")) { + try { + tags.add(convertFileToTag(file)); + } + catch (Exception e) { + log.error("Unable to convert file to tag at: " + file, e); + } + } + } + } + log.info("Successfully converted " + tags.size() + " forms to tags. Executing processor..."); + return processor.process(tags); + } + + public HtmlFormTag convertFileToTag(File formFile) { + try { + log.debug("Processing htmlform file: " + formFile.getAbsolutePath()); + String xml = new FileToStringConverter().convert(formFile); + for (Converter converter : getXmlConverters()) { + log.debug("Converting xml: " + converter); + xml = converter.convert(xml); + } + log.debug("Converting xml to tags"); + HtmlFormTag tag = new XmlToTagConverter().convert(xml); + for (Converter converter : getTagConverters()) { + log.debug("Converting tag: " + converter); + tag = converter.convert(tag); + } + tag.setHtmlFormFile(formFile); + return tag; + } + catch (Exception e) { + throw new IllegalArgumentException("Unable to load form from " + formFile.getName(), e); + } + } + + public List> getXmlConverters() { + List> l = new ArrayList<>(); + l.add(new StripCommentsConverter()); + l.add(new ApplyMacrosConverter()); + l.add(new ApplyRepeatsConverter()); + l.add(new ApplyTranslationsConverter()); + return l; + } + + public List> getTagConverters() { + List> l = new ArrayList<>(); + + // Remove any standard html tags + for (Tags.Standard standardTag : Tags.Standard.values()) { + l.add(new RemoveTagByNameConverter(standardTag.getNodeName())); + } + + // Remove tags that do not impact the data model and cannot be easily used as context for other elements + l.add(new RemoveTagByNameConverter("uimessage")); + l.add(new RemoveTagByNameConverter("lookup")); + l.add(new RemoveTagByNameConverter("submit")); + + // Remove any sections of the form that are added *just* for view mode + l.add(new FlattenOrExcludeIfModeConverter()); + + // Flatten any includeIf or excludeIf expressions down into the child tags + l.add(new MergeParentToChildrenConverter("includeIf", " AND ")); + l.add(new MergeParentToChildrenConverter("excludeIf", " OR ")); + + // Handle the complex web of velocity conditions for entry of date/location/provider + l.add(new MergeSiblingsConverter("encounterDate")); + l.add(new MergeSiblingsConverter("encounterLocation")); + l.add(new MergeSiblingsConverter("encounterProviderAndRole")); + + // Merge obs groups attributes down into children, and add any hidden obs + l.add(new FlattenObsGroupsConverter()); + + // Flatten sections down into the child tags + l.add(new MergeSectionToChildrenConverter()); + + // Handle when/then tags + l.add(new MergeSiblingsConverter("when")); + l.add(new RemoveTagByNameConverter("controls")); + + return l; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java deleted file mode 100644 index 70e692302..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporter.java +++ /dev/null @@ -1,317 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis; - -import org.apache.commons.lang.StringUtils; -import org.openmrs.module.htmlformentry.FormEntryContext; -import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; -import org.openmrs.module.htmlformentry.Translator; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.IfModeTagRemover; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.ObsGroupMapper; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.ParentToChildMerger; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.SectionMerger; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.SiblingMerger; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.TagMapper; -import org.openmrs.module.pihcore.htmlformentry.analysis.mapper.TagRemover; -import org.openmrs.module.pihcore.htmlformentry.analysis.reducer.TagCounter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; -import java.util.TreeSet; - -public class HtmlFormCsvExporter { - - private final Logger log = LoggerFactory.getLogger(HtmlFormCsvExporter.class); - - public void export(File file) { - HtmlFormTag tag = loadFormFromFile(file); - - List mappers = new ArrayList<>(); - - // Remove any standard html tags - for (Tags.Standard standardTag : Tags.Standard.values()) { - mappers.add(new TagRemover(standardTag.getNodeName())); - } - - // Remove tags that do not impact the data model and cannot be easily used as context for other elements - mappers.add(new TagRemover("uimessage")); - mappers.add(new TagRemover("lookup")); - mappers.add(new TagRemover("submit")); - - // Remove any sections of the form that are added *just* for view mode - mappers.add(new IfModeTagRemover()); - - // Flatten any includeIf or excludeIf expressions down into the child tags - mappers.add(new ParentToChildMerger("includeIf")); - mappers.add(new ParentToChildMerger("excludeIf")); - - // Handle the complex web of velocity conditions for entry of date/location/provider - mappers.add(new SiblingMerger("encounterDate")); - mappers.add(new SiblingMerger("encounterLocation")); - mappers.add(new SiblingMerger("encounterProviderAndRole")); - - // Merge obs groups attributes down into children, and add any hidden obs - mappers.add(new ObsGroupMapper()); - - // Flatten sections down into the child tags - mappers.add(new SectionMerger()); - - // Handle when/then tags - mappers.add(new SiblingMerger("when")); - tag = new TagRemover("controls").map(tag); - - for (TagMapper mapper : mappers) { - tag = mapper.map(tag); - } - - TagCounter tagCounter = new TagCounter(); - Map> counters = tagCounter.reduce(tag); - for (String counter : counters.keySet()) { - Map counterValues = counters.get(counter); - for (String key : counterValues.keySet()) { - Integer count = counterValues.get(key); - System.out.println(counter + ", " + key + ", " + count); - } - } - - //HtmlFormUtils.printTag(tag, ""); - } - - protected HtmlFormTag loadFormFromFile(File file) { - HtmlFormParser parser = new HtmlFormParser() { - @Override - protected String preprocessXml(String xml) throws Exception { - HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); - xml = generator.stripComments(xml); - xml = generator.applyMacros(xml); - xml = generator.applyRepeats(xml); - xml = generator.applyTranslations(xml, new FormEntryContext(FormEntryContext.Mode.EDIT) { - @Override - public Translator getTranslator() { - return new Translator() { - @Override - public String translate(String localeStr, String key) { - String ret = getTranslations(localeStr).get(key); - return (ret == null ? key : ret); - } - }; - } - }); - return xml; - } - }; - return parser.parseFile(file); - } - - - - - - - - - - public static final List AVAILABLE_COLUMNS = Arrays.asList( - "sectionName", "tagName", "style", "groupingConceptId", "conceptId", "conceptIds", "valueCoded", - "answerConceptId", "answerConceptIds", "answers", "answerConceptSetIds", "answerClasses", - "answerDrugs", "answerLocationTags", "labelCode", "labelText", "conceptLabels", - "answerCode", "answerLabel", "answerCodes", "answerLabels", "required", - "obsCommentUsed", "showCommentField", "commentFieldLabel", "commentFieldCode", - "dateLabel", "allowFutureDates", "showTime", "defaultValue", "whenValueThen", "toggle" - ); - - private final Set columns = new TreeSet<>(new ColumnComparator()); - - private final List> rows = new ArrayList<>(); - - private final Stack processedTagStack = new Stack<>(); - - private void processTag(HtmlFormTag tag) { - ProcessedTag processedTag = getProcessedTag(tag); - if (!processedTag.getDataRows().isEmpty()) { - Map dataFromParents = new HashMap<>(); - for (ProcessedTag parentTag : processedTagStack) { - dataFromParents.putAll(parentTag.getDataForChildren()); - } - for (Map row : processedTag.getDataRows()) { - if (!row.isEmpty()) { - row.putAll(dataFromParents); - rows.add(row); - columns.addAll(row.keySet()); - } - } - } - if (processedTag.isProcessChildren()) { - processedTagStack.push(processedTag); - for (HtmlFormTag childTag : tag.getChildTags()) { - processTag(childTag); - } - processedTagStack.pop(); - } - } - - // This exists in the event that some tag needs to produce multiple rows, most will simply delegate to getDataRow - private ProcessedTag getProcessedTag(HtmlFormTag tag) { - ProcessedTag processedTag = new ProcessedTag(); - log.trace("Processing tag: " + tag.getName() + ": " + tag.getAttributes()); - Map dataRow = new HashMap<>(); - Map unprocessedValues = new HashMap<>(tag.getAttributes()); - boolean tagHandled = true; - switch (tag.getName()) { - - - // section: only contribute section as a column on child rows. only sectionName is relevant - case "section": { - ignoreAttributes(unprocessedValues, "id", "headerTag", "sectionTag", "headerStyle"); - String headerCode = unprocessedValues.remove("headerCode"); - String headerLabel = unprocessedValues.remove("headerLabel"); - processedTag.getDataForChildren().put("sectionName", getDisplay(headerCode, headerLabel)); - break; - } - - case "obs": { - - for (String obsField : AVAILABLE_COLUMNS) { - String value = unprocessedValues.remove(obsField); - if (StringUtils.isNotBlank(value)) { - dataRow.put(obsField, value); - } - } - - // There are situations where only an empty commentFieldLabel is configured to display the comment box, so we try to handle this - boolean obsCommentUsed = "true".equals(unprocessedValues.get("showCommentField")); - obsCommentUsed = obsCommentUsed || unprocessedValues.containsKey("commentFieldLabel"); - obsCommentUsed = obsCommentUsed || unprocessedValues.containsKey("commentFieldCode"); - if (obsCommentUsed) { - dataRow.put("obsCommentUsed", "true"); - } - - // Roll up any when tags - List whenConditions = new ArrayList<>(); - for (Iterator i = tag.getChildTags().iterator(); i.hasNext();) { - HtmlFormTag controlsTag = i.next(); - if (controlsTag.getName().equalsIgnoreCase("controls")) { - for (Iterator j = controlsTag.getChildTags().iterator(); j.hasNext();) { - HtmlFormTag whenTag = j.next(); - if (whenTag.getName().equalsIgnoreCase("when")) { - whenConditions.add(HtmlFormUtils.mapToString(whenTag.getAttributes(), "=", ",")); - j.remove(); - } - } - if (controlsTag.getChildTags().isEmpty() && controlsTag.getAttributes().isEmpty()) { - i.remove(); - } - } - } - dataRow.put("whenValueThen", String.join("|", whenConditions)); - - break; - } - - default: { - tagHandled = false; - log.debug("No specific handling found for tag: " + tag.getName()); - } - } - - if (!dataRow.isEmpty() || !unprocessedValues.isEmpty() || !tagHandled) { - if (shouldIncludeTag(tag)) { - dataRow.put("tagName", tag.getName()); - dataRow.put("additionalConfiguration", HtmlFormUtils.mapToString(unprocessedValues, "=", ",")); - processedTag.getDataRows().add(dataRow); - } - } - - log.trace("Processed tag: " + processedTag); - return processedTag; - } - - private boolean shouldIncludeTag(HtmlFormTag tag) { - for (Tags.Standard standardTag : Tags.Standard.values()) { - if (standardTag.getNodeName().equalsIgnoreCase(tag.getName())) { - return false; - } - } - return true; - } - - private String getDisplay(String messageCode, String messageText) { - if (StringUtils.isNotBlank(messageCode)) { - return messageCode; // TODO: Translate this? - } - return messageText == null ? "" : messageText; - } - - private void ignoreAttributes(Map attributes, String... keys) { - for (String key : keys) { - attributes.remove(key); - } - } - - public Set getColumns() { - return columns; - } - - public List> getRows() { - return rows; - } - - private static class ProcessedTag { - private boolean processChildren = true; - private List> dataRows = new ArrayList<>(); - private Map dataForChildren = new HashMap<>(); - - public boolean isProcessChildren() { - return processChildren; - } - - public void setProcessChildren(boolean processChildren) { - this.processChildren = processChildren; - } - - public List> getDataRows() { - return dataRows; - } - - public void setDataRows(List> dataRows) { - this.dataRows = dataRows; - } - - public Map getDataForChildren() { - return dataForChildren; - } - - public void setDataForChildren(Map dataForChildren) { - this.dataForChildren = dataForChildren; - } - } - - private static class ColumnComparator implements Comparator { - - @Override - public int compare(String s1, String s2) { - int index1 = AVAILABLE_COLUMNS.indexOf(s1); - int index2 = AVAILABLE_COLUMNS.indexOf(s2); - if (index1 == -1 && index2 != -1) { - return 1; - } - if (index1 != -1 && index2 == -1) { - return -1; - } - int ret = index1 - index2; - if (ret == 0) { - ret = s1.compareTo(s2); - } - return ret; - } - } - -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java deleted file mode 100644 index d0d6427f1..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormParser.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; -import org.openmrs.module.htmlformentry.HtmlFormEntryUtil; -import org.w3c.dom.CharacterData; -import org.w3c.dom.Document; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Stack; - -/** - * Loads and html form and parses it into tags - */ -public class HtmlFormParser { - - protected final Log log = LogFactory.getLog(getClass()); - - public static final String HTMLFORM_TAG_NAME = "htmlform"; - - private final HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); - - protected final Stack tagStack = new Stack<>(); - - public HtmlFormTag parseFile(File formFile) { - try { - log.debug("Parsing htmlform file: " + formFile.getAbsolutePath()); - return parseXml(FileUtils.readFileToString(formFile, StandardCharsets.UTF_8)); - } - catch (IOException e) { - throw new IllegalArgumentException("Unable to load form from " + formFile.getName(), e); - } - } - - public HtmlFormTag parseXml(String formXml) { - try { - formXml = preprocessXml(formXml); - Document document = HtmlFormEntryUtil.stringToDocument(formXml); - Node htmlFormNode = HtmlFormEntryUtil.findChild(document, HTMLFORM_TAG_NAME); - HtmlFormTag htmlFormTag = processNode(htmlFormNode); - if (!tagStack.isEmpty()) { - throw new IllegalStateException("Tag Stack is not empty after processing. Found: " + tagStack); - } - return htmlFormTag; - } - catch (Exception e) { - throw new IllegalStateException("Unable to parse form xml", e); - } - } - - protected String preprocessXml(String xml) throws Exception { - return xml; - } - - protected HtmlFormTag processNode(Node node) { - HtmlFormTag tag = null; - if (node != null) { - tag = new HtmlFormTag(); - tag.setName(node.getNodeName()); - if (node instanceof CharacterData) { - CharacterData characterDataNode = (CharacterData) node; - tag.setData(characterDataNode.getData()); - } - NamedNodeMap attrs = node.getAttributes(); - if (attrs != null) { - for (int i = 0; i < attrs.getLength(); i++) { - Node attr = attrs.item(i); - tag.getAttributes().put(attr.getNodeName(), attr.getNodeValue()); - handleTagAttribute(tag, attr.getNodeName(), attr.getNodeValue()); - } - } - if (!tagStack.isEmpty()) { - tagStack.peek().getChildTags().add(tag); - } - handleTagStart(tag); - tagStack.push(tag); - NodeList childNodes = node.getChildNodes(); - for (int i=0; i tagCounter = new TreeMap<>(); - private final Map flatTagCounter = new TreeMap<>(); - private final Map attributeCounter = new TreeMap<>(); - - public HtmlFormTagCounter() { } - - public void analyze(File fileOrDirectory) { - HtmlFormParser parser = new HtmlFormParser() { - @Override - protected void handleTagStart(HtmlFormTag tag) { - super.handleTagStart(tag); - tagCounter.put(tag.getName(), tagCounter.getOrDefault(tag.getName(), 0) + 1); - List names = new ArrayList<>(); - for (HtmlFormTag tagInStack : tagStack) { - names.add(tagInStack.getName()); - } - names.add(tag.getName()); - String flatName = String.join(".", names); - flatTagCounter.put(flatName, tagCounter.getOrDefault(flatName, 0) + 1); - } - @Override - protected void handleTagAttribute(HtmlFormTag tag, String attributeName, String attributeValue) { - super.handleTagAttribute(tag, attributeName, attributeValue); - String name = tag.getName() + "." + attributeName; - attributeCounter.put(name, attributeCounter.getOrDefault(name, 0) + 1); - } - }; - if (fileOrDirectory.isFile()) { - parser.parseFile(fileOrDirectory); - } - else if (fileOrDirectory.isDirectory()) { - List files = HtmlFormUtils.getNestedFilesBySuffix(fileOrDirectory, "xml"); - for (File file : files) { - parser.parseFile(file); - } - } - } - - public void writeToCsv(File outputFile) throws IOException { - try (FileWriter fileWriter = new FileWriter(outputFile)) { - try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { - String[] csvColumns = {"counter", "tagName", "attributeName", "count"}; - csvWriter.writeNext(csvColumns); - for (String element : tagCounter.keySet()) { - csvWriter.writeNext(new String[] {"tags", element, "", Integer.toString(tagCounter.get(element))}); - } - for (String element : flatTagCounter.keySet()) { - csvWriter.writeNext(new String[] {"flatTags", element, "", Integer.toString(flatTagCounter.get(element))}); - } - for (String element : attributeCounter.keySet()) { - String[] split = element.split("\\."); - csvWriter.writeNext(new String[] {"tagAttributes", split[0], split[1], Integer.toString(attributeCounter.get(element))}); - } - } - } - } - - public Map getTagCounter() { - return tagCounter; - } - - public Map getAttributeCounter() { - return attributeCounter; - } -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/Converter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/Converter.java new file mode 100644 index 000000000..0285008d5 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/Converter.java @@ -0,0 +1,5 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter; + +public interface Converter { + O convert(I input) throws Exception; +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/file/FileToStringConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/file/FileToStringConverter.java new file mode 100644 index 000000000..ff96632be --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/file/FileToStringConverter.java @@ -0,0 +1,14 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.file; + +import org.apache.commons.io.FileUtils; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +import java.io.File; +import java.nio.charset.StandardCharsets; + +public class FileToStringConverter implements Converter { + + public String convert(File input) throws Exception { + return FileUtils.readFileToString(input, StandardCharsets.UTF_8); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenObsGroupsConverter.java similarity index 60% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenObsGroupsConverter.java index 81bee783f..e642862e0 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/ObsGroupMapper.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenObsGroupsConverter.java @@ -1,16 +1,17 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; import lombok.Data; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; @Data -public class ObsGroupMapper implements TagMapper { +public class FlattenObsGroupsConverter implements Converter { @Override - public HtmlFormTag map(HtmlFormTag inputTag) { - HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); - for (HtmlFormTag childTag : inputTag.getChildTags()) { - childTag = map(childTag); + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + HtmlFormTag outputTag = input.cloneWithoutChildren(); + for (HtmlFormTag childTag : input.getChildTags()) { + childTag = convert(childTag); if (childTag.getName().equalsIgnoreCase("obsGroup")) { String hiddenConceptId = childTag.getAttributes().remove("hiddenConceptId"); String hiddenAnswerConceptId = childTag.getAttributes().remove("hiddenAnswerConceptId"); @@ -25,7 +26,7 @@ public HtmlFormTag map(HtmlFormTag inputTag) { } outputTag.getChildTags().add(childTag); } - ParentToChildMerger parentToChildMerger = new ParentToChildMerger("obsGroup"); - return parentToChildMerger.map(outputTag); + MergeParentToChildrenConverter parentToChildMerger = new MergeParentToChildrenConverter("obsGroup", " > "); + return parentToChildMerger.convert(outputTag); } } diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenOrExcludeIfModeConverter.java similarity index 59% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenOrExcludeIfModeConverter.java index 9c1a8815d..52ccca922 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/IfModeTagRemover.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/FlattenOrExcludeIfModeConverter.java @@ -1,16 +1,17 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; import lombok.Data; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; @Data -public class IfModeTagRemover implements TagMapper { +public class FlattenOrExcludeIfModeConverter implements Converter { @Override - public HtmlFormTag map(HtmlFormTag inputTag) { - HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); - for (HtmlFormTag childTag : inputTag.getChildTags()) { - childTag = map(childTag); + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + HtmlFormTag outputTag = input.cloneWithoutChildren(); + for (HtmlFormTag childTag : input.getChildTags()) { + childTag = convert(childTag); if (childTag.getName().equalsIgnoreCase("ifMode")) { boolean viewMode = childTag.hasAttribute("mode", "view"); boolean include = !childTag.hasAttribute("include", "false"); diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeParentToChildrenConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeParentToChildrenConverter.java new file mode 100644 index 000000000..22f761fb3 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeParentToChildrenConverter.java @@ -0,0 +1,41 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class MergeParentToChildrenConverter implements Converter { + + private String tagName; + private String multiValueSeparator = " AND "; + + public MergeParentToChildrenConverter(String tagName, String multiValueSeparator) { + this.tagName = tagName; + this.multiValueSeparator = multiValueSeparator; + } + + @Override + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + HtmlFormTag outputTag = input.cloneWithoutChildren(); + for (HtmlFormTag childTag : input.getChildTags()) { + if (input.getName().equalsIgnoreCase(tagName)) { + Map attributesToAdd = computeAttributesToMerge(new HashMap<>(input.getAttributes())); + for (Map.Entry attribute : attributesToAdd.entrySet()) { + childTag.mergeAttributeRecursively(attribute.getKey(), attribute.getValue(), multiValueSeparator); + } + } + childTag = convert(childTag); + outputTag.getChildTags().add(childTag); + } + outputTag = new RemoveTagByNameConverter(tagName).convert(outputTag); + return outputTag; + } + + public Map computeAttributesToMerge(Map inputAttributes) { + return inputAttributes; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSectionToChildrenConverter.java similarity index 71% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSectionToChildrenConverter.java index 4a0d4674a..93fe0ed68 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SectionMerger.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSectionToChildrenConverter.java @@ -1,4 +1,4 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; import org.apache.commons.lang.StringUtils; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormUtils; @@ -6,10 +6,10 @@ import java.util.HashMap; import java.util.Map; -public class SectionMerger extends ParentToChildMerger { +public class MergeSectionToChildrenConverter extends MergeParentToChildrenConverter { - public SectionMerger() { - super("section"); + public MergeSectionToChildrenConverter() { + super("section", " > "); } @Override diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSiblingsConverter.java similarity index 73% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSiblingsConverter.java index 57cdc9313..36b0a8cbe 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/SiblingMerger.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/MergeSiblingsConverter.java @@ -1,27 +1,28 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; import lombok.Data; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; @Data -public class SiblingMerger implements TagMapper { +public class MergeSiblingsConverter implements Converter { private String tagName; - public SiblingMerger(String tagName) { + public MergeSiblingsConverter(String tagName) { this.tagName = tagName; } @Override - public HtmlFormTag map(HtmlFormTag inputTag) { - HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + HtmlFormTag outputTag = input.cloneWithoutChildren(); HtmlFormTag matchingTag = null; - for (HtmlFormTag childTag : inputTag.getChildTags()) { - childTag = map(childTag); + for (HtmlFormTag childTag : input.getChildTags()) { + childTag = convert(childTag); if (childTag.getName().equalsIgnoreCase(tagName)) { if (!childTag.getChildTags().isEmpty()) { throw new IllegalStateException("You cannot merge siblings that have children"); diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/ObsConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/ObsConverter.java new file mode 100644 index 000000000..7862754f1 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/ObsConverter.java @@ -0,0 +1,21 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; + +import lombok.Data; +import org.apache.commons.lang.StringUtils; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class ObsConverter implements Converter { + + @Override + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + // Any of these will effectively show the comment field, so update that if needed + if (!input.hasAttribute("showCommentField", "true")) { + if (input.hasAttribute("commentFieldLabel") || input.hasAttribute("commentFieldCode")) { + input.addAttribute("showCommentField", "true"); + } + } + return input; + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/RemoveTagByNameConverter.java similarity index 51% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/RemoveTagByNameConverter.java index 30249b981..32245b375 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagRemover.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/tag/RemoveTagByNameConverter.java @@ -1,23 +1,24 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.tag; import lombok.Data; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; @Data -public class TagRemover implements TagMapper { +public class RemoveTagByNameConverter implements Converter { private String tagName; private boolean removeChildren; - public TagRemover(String tagName) { + public RemoveTagByNameConverter(String tagName) { this.tagName = tagName; } @Override - public HtmlFormTag map(HtmlFormTag inputTag) { - HtmlFormTag outputTag = inputTag.cloneWithoutChildren(); - for (HtmlFormTag childTag : inputTag.getChildTags()) { - childTag = map(childTag); + public HtmlFormTag convert(HtmlFormTag input) throws Exception { + HtmlFormTag outputTag = input.cloneWithoutChildren(); + for (HtmlFormTag childTag : input.getChildTags()) { + childTag = convert(childTag); if (childTag.getName().equalsIgnoreCase(tagName)) { if (!removeChildren) { outputTag.getChildTags().addAll(childTag.getChildTags()); diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyMacrosConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyMacrosConverter.java new file mode 100644 index 000000000..8be4918c8 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyMacrosConverter.java @@ -0,0 +1,15 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import lombok.Data; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class ApplyMacrosConverter implements Converter { + + @Override + public String convert(String input) throws Exception { + HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + return generator.applyMacros(input); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyRepeatsConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyRepeatsConverter.java new file mode 100644 index 000000000..6a1587d77 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyRepeatsConverter.java @@ -0,0 +1,15 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import lombok.Data; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class ApplyRepeatsConverter implements Converter { + + @Override + public String convert(String input) throws Exception { + HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + return generator.applyRepeats(input); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyTranslationsConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyTranslationsConverter.java new file mode 100644 index 000000000..b62469b59 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/ApplyTranslationsConverter.java @@ -0,0 +1,28 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import lombok.Data; +import org.openmrs.module.htmlformentry.FormEntryContext; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.htmlformentry.Translator; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class ApplyTranslationsConverter implements Converter { + + @Override + public String convert(String input) throws Exception { + HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + return generator.applyTranslations(input, new FormEntryContext(FormEntryContext.Mode.EDIT) { + @Override + public Translator getTranslator() { + return new Translator() { + @Override + public String translate(String localeStr, String key) { + String ret = getTranslations(localeStr).get(key); + return (ret == null ? key : ret); + } + }; + } + }); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/StripCommentsConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/StripCommentsConverter.java new file mode 100644 index 000000000..d27437fb5 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/StripCommentsConverter.java @@ -0,0 +1,15 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import lombok.Data; +import org.openmrs.module.htmlformentry.HtmlFormEntryGenerator; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class StripCommentsConverter implements Converter { + + @Override + public String convert(String input) throws Exception { + HtmlFormEntryGenerator generator = new HtmlFormEntryGenerator(); + return generator.stripComments(input); + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/XmlToTagConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/XmlToTagConverter.java new file mode 100644 index 000000000..543147590 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/XmlToTagConverter.java @@ -0,0 +1,69 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.openmrs.module.htmlformentry.HtmlFormEntryUtil; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; +import org.w3c.dom.CharacterData; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.Stack; + +/** + * Loads and html form and parses it into tags + */ +public class XmlToTagConverter implements Converter { + + protected final Log log = LogFactory.getLog(getClass()); + + public static final String HTMLFORM_TAG_NAME = "htmlform"; + + private final Stack tagStack = new Stack<>(); + + public HtmlFormTag convert(String input) throws Exception { + Document document = HtmlFormEntryUtil.stringToDocument(input); + Node htmlFormNode = HtmlFormEntryUtil.findChild(document, HTMLFORM_TAG_NAME); + HtmlFormTag htmlFormTag = processNode(htmlFormNode); + if (!tagStack.isEmpty()) { + throw new IllegalStateException("Tag Stack is not empty after processing. Found: " + tagStack); + } + return htmlFormTag; + } + + protected HtmlFormTag processNode(Node node) { + HtmlFormTag tag = null; + if (node != null) { + tag = new HtmlFormTag(); + tag.setName(node.getNodeName()); + if (node instanceof CharacterData) { + CharacterData characterDataNode = (CharacterData) node; + tag.setData(characterDataNode.getData()); + } + NamedNodeMap attrs = node.getAttributes(); + if (attrs != null) { + for (int i = 0; i < attrs.getLength(); i++) { + Node attr = attrs.item(i); + tag.getAttributes().put(attr.getNodeName(), attr.getNodeValue()); + log.trace("tagAttribute: " + attr.getNodeName() + " = " + attr.getNodeValue()); + } + } + if (!tagStack.isEmpty()) { + tagStack.peek().getChildTags().add(tag); + } + log.trace("startTag: " + tag.getName()); + tagStack.push(tag); + NodeList childNodes = node.getChildNodes(); + for (int i=0; i attributesToAdd = computeAttributesToMerge(new HashMap<>(inputTag.getAttributes())); - for (Map.Entry attribute : attributesToAdd.entrySet()) { - childTag.addAttribute(attribute.getKey(), attribute.getValue(), true); - } - } - childTag = map(childTag); - outputTag.getChildTags().add(childTag); - } - outputTag = new TagRemover(tagName).map(outputTag); - return outputTag; - } - - public Map computeAttributesToMerge(Map inputAttributes) { - return inputAttributes; - } -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java deleted file mode 100644 index 73f1506b6..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/mapper/TagMapper.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.mapper; - -import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; - -public interface TagMapper { - HtmlFormTag map(HtmlFormTag inputTag); -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagCounter.java similarity index 59% rename from api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java rename to api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagCounter.java index d4a49fba0..3977b236f 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagCounter.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagCounter.java @@ -1,6 +1,6 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.reducer; +package org.openmrs.module.pihcore.htmlformentry.analysis.processor; -import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.DataSet; import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; import java.util.ArrayList; @@ -9,18 +9,28 @@ import java.util.Stack; import java.util.TreeMap; -@Data -public class TagCounter implements TagReducer>> { +public class TagCounter implements TagProcessor { @Override - public Map> reduce(HtmlFormTag inputTag) { + public DataSet process(List tags) throws Exception { Map> counters = new TreeMap<>(); Stack stack = new Stack<>(); - reduce(inputTag, counters, stack); - return counters; + for (HtmlFormTag tag : tags) { + process(tag, counters, stack); + } + DataSet dataSet = new DataSet(); + dataSet.setHeaderRow("counter", "key", "value"); + for (String counter : counters.keySet()) { + Map counterValues = counters.get(counter); + for (String key : counterValues.keySet()) { + Integer count = counterValues.get(key); + dataSet.addDataRow(counter, key, Integer.toString(count)); + } + } + return dataSet; } - private void reduce(HtmlFormTag inputTag, Map> counters, Stack stack) { + private void process(HtmlFormTag inputTag, Map> counters, Stack stack) { Map tagCounter = counters.computeIfAbsent("tags", k -> new TreeMap<>()); tagCounter.put(inputTag.getName(), tagCounter.getOrDefault(inputTag.getName(), 0) + 1); @@ -41,7 +51,7 @@ private void reduce(HtmlFormTag inputTag, Map> coun stack.push(inputTag); for (HtmlFormTag childTag : inputTag.getChildTags()) { - reduce(childTag, counters, stack); + process(childTag, counters, stack); } stack.pop(); } diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagExporter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagExporter.java new file mode 100644 index 000000000..757f033a1 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagExporter.java @@ -0,0 +1,107 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.processor; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.DataSet; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +@Data +public class TagExporter implements TagProcessor { + + private File outputDirectory; + + private final Logger log = LoggerFactory.getLogger(TagExporter.class); + + public static final List COLUMNS = Arrays.asList( + "section", "tagName", "style", "groupingConceptId", "conceptId", "conceptIds", "valueCoded", + "answerConceptId", "answerConceptIds", "answers", "answerConceptSetIds", "answerClasses", + "answerDrugs", "answerLocationTags", "labelCode", "labelText", "conceptLabels", + "answerCode", "answerLabel", "answerCodes", "answerLabels", "required", + "obsCommentUsed", "showCommentField", "commentFieldLabel", "commentFieldCode", + "dateLabel", "allowFutureDates", "showTime", "defaultValue", "whenValueThen", "toggle", "additionalAttributes" + ); + + public DataSet process(List tags) throws Exception { + DataSet ret = new DataSet(); + ret.setHeaderRow("inputFile", "outputFile"); + for (HtmlFormTag tag : tags) { + try { + if (outputDirectory == null) { + outputDirectory = tag.getHtmlFormFile().getParentFile(); + } + File outputFile = new File(outputDirectory, tag.getHtmlFormFile().getName() + ".csv"); + log.info("Processing tag " + tag + " into " + outputFile); + List columns = new ArrayList<>(getAllColumns(Collections.singletonList(tag))); + DataSet formDataSet = new DataSet(); + formDataSet.setColumns(columns); + formDataSet.setRows(getAllRows(columns, tag)); + formDataSet.toCsv(outputFile, ','); + ret.addDataRow(tag.getHtmlFormFile().getAbsolutePath(), outputFile.getAbsolutePath()); + } + catch (Exception e) { + log.error("Error exporting tag " + tag, e); + } + } + return ret; + } + + private Set getAllColumns(List tags) { + Set ret = new TreeSet<>(new ColumnComparator()); + ret.add("tagName"); + for (HtmlFormTag tag : tags) { + ret.addAll(tag.getAttributes().keySet()); + ret.addAll(getAllColumns(tag.getChildTags())); + } + return ret; + } + + private List> getAllRows(List columns, HtmlFormTag tag) { + List> rows = new ArrayList<>(); + Map row = new HashMap<>(); + Map attributes = new HashMap<>(tag.getAttributes()); + attributes.put("tagName", tag.getName()); + for (String column : columns) { + String value = attributes.remove(column); + row.put(column, value == null ? "" : value); + } + row.put("additionalAttributes", HtmlFormUtils.mapToString(attributes, "=", ",")); + rows.add(row); + for (HtmlFormTag childTag : tag.getChildTags()) { + rows.addAll(getAllRows(columns, childTag)); + } + return rows; + } + + private static class ColumnComparator implements Comparator { + + @Override + public int compare(String s1, String s2) { + int index1 = COLUMNS.indexOf(s1); + int index2 = COLUMNS.indexOf(s2); + if (index1 == -1 && index2 != -1) { + return 1; + } + if (index1 != -1 && index2 == -1) { + return -1; + } + int ret = index1 - index2; + if (ret == 0) { + ret = s1.compareTo(s2); + } + return ret; + } + } +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagProcessor.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagProcessor.java new file mode 100644 index 000000000..5843fb515 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagProcessor.java @@ -0,0 +1,10 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.processor; + +import org.openmrs.module.pihcore.htmlformentry.analysis.DataSet; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; + +import java.util.List; + +public interface TagProcessor { + DataSet process(List tags) throws Exception; +} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java deleted file mode 100644 index 18f4de779..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/reducer/TagReducer.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis.reducer; - -import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; - -public interface TagReducer { - T reduce(HtmlFormTag inputTag); -} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java new file mode 100644 index 000000000..4c2381a20 --- /dev/null +++ b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java @@ -0,0 +1,31 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis; + +import org.junit.jupiter.api.Test; +import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagCounter; +import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagExporter; + +import javax.xml.crypto.Data; +import java.io.File; + +public class HtmlFormAnalyzerTest { + + public File getHtmlFormsDirectory() { + return new File("/home/mseaton/code/github/pih/openmrs-config-pihsl/target/openmrs-packager-config/configuration/pih/htmlforms"); + } + + @Test + public void generateCountsOfAllTags() throws Exception { + HtmlFormAnalyzer analyzer = new HtmlFormAnalyzer(); + DataSet dataSet = analyzer.analyze(getHtmlFormsDirectory(), new TagCounter()); + dataSet.toCsv(new File(getHtmlFormsDirectory(), "tagAnalysis.csv"), ','); + dataSet.print("\t"); + } + + @Test + public void generateCsvForForms() throws Exception { + HtmlFormAnalyzer analyzer = new HtmlFormAnalyzer(); + File htmlForm = new File(getHtmlFormsDirectory(), "mentalHealthFollowup.xml"); + DataSet dataSet = analyzer.analyze(htmlForm, new TagExporter()); + dataSet.print("\t"); + } +} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java deleted file mode 100644 index 990763ae2..000000000 --- a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormCsvExporterTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis; - -import com.opencsv.CSVWriter; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileWriter; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class HtmlFormCsvExporterTest { - - private static final Logger log = LoggerFactory.getLogger(HtmlFormNonMemoryTest.class); - - public File getHtmlFormsDirectory() { - return new File("/home/mseaton/code/github/pih/openmrs-config-pihsl/target/openmrs-packager-config/configuration/pih/htmlforms"); - } - - @Test - public void execute() throws Exception { - generateCsvForForm(new File(getHtmlFormsDirectory(), "mentalHealthFollowup.xml")); - } - - /* - public void generateTagSummary() throws Exception { - List htmlFormFiles = HtmlFormUtils.getNestedFilesBySuffix(getHtmlFormsDirectory(), ".xml"); - Map parsedForms = new TreeMap<>(); - for (File inputFile : htmlFormFiles) { - try { - HtmlFormCsvExporter parser = new HtmlFormCsvExporter(); - parser.parseFile(inputFile); - parsedForms.put(inputFile.getName(), parser); - } - catch (Exception e) { - log.error("Error parsing form: " + inputFile.getName()); - } - } - File outputFile = new File(getHtmlFormsDirectory(), "formSummary.csv"); - try (FileWriter fileWriter = new FileWriter(outputFile)) { - try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { - String[] csvColumns = {"form", "tag", "comments"}; - csvWriter.writeNext(csvColumns); - for (String formName : parsedForms.keySet()) { - HtmlFormCsvExporter parser = parsedForms.get(formName); - for (Map row : parser.getRows()) { - String[] data = new String[csvColumns.length]; - data[0] = formName; - data[1] = row.get("tagName"); - data[2] = row.get("additionalConfiguration"); - csvWriter.writeNext(data); - } - } - } - } - } - - */ - - public void generateCsvForAllForms() { - List htmlFormFiles = HtmlFormUtils.getNestedFilesBySuffix(getHtmlFormsDirectory(), ".xml"); - for (File inputFile : htmlFormFiles) { - try { - generateCsvForForm(inputFile); - } - catch (Exception e) { - log.error("Unable to generate CSV for form: " + inputFile.getName()); - } - } - } - - public void generateCsvForForm(File inputFile) throws Exception { - File outputFile = new File(inputFile.getAbsolutePath() + ".csv"); - log.info("Generating schema for " + inputFile.getName()); - log.info("TagRegister loaded for " + inputFile.getName()); - HtmlFormCsvExporter exporter = new HtmlFormCsvExporter(); - exporter.export(inputFile); - /* - String[] columns = parser.getColumns().toArray(new String[0]); - List> rows = parser.getRows(); - log.info("TagRegister parsed into " + columns.length + " columns and " + rows.size() + " rows"); - try (FileWriter fileWriter = new FileWriter(outputFile)) { - try (CSVWriter csvWriter = new CSVWriter(fileWriter)) { - csvWriter.writeNext(columns); // Header - for (Map row : rows) { - String[] rowArray = new String[columns.length]; - for (int i=0; i Date: Tue, 3 Sep 2024 20:40:28 -0400 Subject: [PATCH 4/5] More refactoring --- .../htmlformentry/analysis/DataSet.java | 6 ++- .../htmlformentry/analysis/HtmlFormUtils.java | 7 --- .../analysis/processor/TagWriter.java | 45 +++++++++++++++++++ .../analysis/HtmlFormAnalyzerTest.java | 17 ++++++- 4 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagWriter.java diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java index 7b4d580fb..ede4eea83 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/DataSet.java @@ -2,6 +2,7 @@ import com.opencsv.CSVWriter; import lombok.Data; +import org.apache.commons.lang.StringUtils; import java.io.File; import java.io.FileWriter; @@ -56,8 +57,9 @@ public void toCsv(File outputFile, char separator) throws Exception { } public void print(String separator) { - System.out.println(name); - System.out.println("===================================="); + if (StringUtils.isNotBlank(name)) { + System.out.println(name); + } System.out.println(String.join(separator, columns)); for (Map row : rows) { List values = new ArrayList<>(); diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java index 842edc84e..89f8f4f84 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormUtils.java @@ -36,11 +36,4 @@ public static String mapToString(Map m, String keyValueSeparator } return String.join(entrySeparator, ret); } - - public static void printTag(HtmlFormTag tag, String prefix) { - System.out.println(prefix + tag.toString()); - for (HtmlFormTag childTag : tag.getChildTags()) { - printTag(childTag, prefix + " "); - } - } } diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagWriter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagWriter.java new file mode 100644 index 000000000..82ade4df1 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/processor/TagWriter.java @@ -0,0 +1,45 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.processor; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.DataSet; +import org.openmrs.module.pihcore.htmlformentry.analysis.HtmlFormTag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.List; + +@Data +public class TagWriter implements TagProcessor { + + private File outputFile; + + private final Logger log = LoggerFactory.getLogger(TagWriter.class); + + public DataSet process(List tags) throws Exception { + DataSet ret = new DataSet(); + ret.setHeaderRow("inputFile", "outputData"); + for (HtmlFormTag tag : tags) { + try (StringWriter writer = new StringWriter()) { + write(tag, writer, " "); + String data = writer.toString(); + ret.addDataRow(tag.getHtmlFormFile().getName(), data); + } + } + return ret; + } + + public static void write(HtmlFormTag tag, Writer writer, String prefix) throws IOException { + write(tag, writer, "", prefix); + } + + private static void write(HtmlFormTag tag, Writer writer, String prefix, String prefixIncrement) throws IOException { + writer.write(prefix + tag + System.lineSeparator()); + for (HtmlFormTag childTag : tag.getChildTags()) { + write(childTag, writer, prefix + prefixIncrement, prefix); + } + } +} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java index 4c2381a20..278a180d0 100644 --- a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java +++ b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java @@ -3,9 +3,11 @@ import org.junit.jupiter.api.Test; import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagCounter; import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagExporter; +import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagWriter; -import javax.xml.crypto.Data; import java.io.File; +import java.io.FileWriter; +import java.util.Map; public class HtmlFormAnalyzerTest { @@ -28,4 +30,17 @@ public void generateCsvForForms() throws Exception { DataSet dataSet = analyzer.analyze(htmlForm, new TagExporter()); dataSet.print("\t"); } + + @Test + public void writeFormData() throws Exception { + HtmlFormAnalyzer analyzer = new HtmlFormAnalyzer(); + DataSet dataSet = analyzer.analyze(getHtmlFormsDirectory(), new TagWriter()); + try (FileWriter writer = new FileWriter(new File(getHtmlFormsDirectory(), "tagAnalysis.txt"))) { + for (Map rows : dataSet.getRows()) { + writer.write(rows.get("inputFile") + System.lineSeparator()); + writer.write("=============================" + System.lineSeparator()); + writer.write(rows.get("outputData") + System.lineSeparator()); + } + } + } } From f6eea0ebd6520582e320fcab5d4176c2c4cae9dc Mon Sep 17 00:00:00 2001 From: Michael Seaton Date: Tue, 3 Sep 2024 22:40:15 -0400 Subject: [PATCH 5/5] More refactoring --- .../analysis/HtmlFormAnalyzer.java | 55 +++++++++++++++++-- .../pihcore/htmlformentry/analysis/Tags.java | 35 ------------ .../analysis/converter/xml/TextConverter.java | 21 +++++++ .../analysis/HtmlFormAnalyzerTest.java | 12 ++++ 4 files changed, 83 insertions(+), 40 deletions(-) delete mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java create mode 100644 api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/TextConverter.java diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java index ae05d029e..91dc11660 100644 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzer.java @@ -12,6 +12,7 @@ import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.ApplyRepeatsConverter; import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.ApplyTranslationsConverter; import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.StripCommentsConverter; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.TextConverter; import org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml.XmlToTagConverter; import org.openmrs.module.pihcore.htmlformentry.analysis.processor.TagProcessor; import org.slf4j.Logger; @@ -75,6 +76,9 @@ public List> getXmlConverters() { l.add(new ApplyMacrosConverter()); l.add(new ApplyRepeatsConverter()); l.add(new ApplyTranslationsConverter()); + l.add(new TextConverter("(!$user.hasPrivilege('Task: emr.retroConsultNoteThisProviderOnly') and !$user.hasPrivilege('Task: emr.retroConsultNote')) or ($user.hasPrivilege('Task: emr.retroConsultNoteThisProviderOnly') and !$user.hasPrivilege('Task: emr.retroConsultNote') and $visit.open)", "retroCondition1")); + l.add(new TextConverter("$user.hasPrivilege('Task: emr.retroConsultNoteThisProviderOnly') and !($user.hasPrivilege('Task: emr.retroConsultNote')) and (!$visit.open)", "retroCondition2")); + l.add(new TextConverter("$user.hasPrivilege('Task: emr.retroConsultNote')", "retroCondition3")); return l; } @@ -82,9 +86,50 @@ public List> getTagConverters() { List> l = new ArrayList<>(); // Remove any standard html tags - for (Tags.Standard standardTag : Tags.Standard.values()) { - l.add(new RemoveTagByNameConverter(standardTag.getNodeName())); - } + l.add(new RemoveTagByNameConverter("#comment")); + l.add(new RemoveTagByNameConverter("#text")); + l.add(new RemoveTagByNameConverter("a")); + l.add(new RemoveTagByNameConverter("b")); + l.add(new RemoveTagByNameConverter("br")); + l.add(new RemoveTagByNameConverter("button")); + l.add(new RemoveTagByNameConverter("center")); + l.add(new RemoveTagByNameConverter("dd")); + l.add(new RemoveTagByNameConverter("div")); + l.add(new RemoveTagByNameConverter("dl")); + l.add(new RemoveTagByNameConverter("dt")); + l.add(new RemoveTagByNameConverter("field")); + l.add(new RemoveTagByNameConverter("fieldset")); + l.add(new RemoveTagByNameConverter("font")); + l.add(new RemoveTagByNameConverter("h1")); + l.add(new RemoveTagByNameConverter("h2")); + l.add(new RemoveTagByNameConverter("h3")); + l.add(new RemoveTagByNameConverter("h4")); + l.add(new RemoveTagByNameConverter("h5")); + l.add(new RemoveTagByNameConverter("h6")); + l.add(new RemoveTagByNameConverter("h7")); + l.add(new RemoveTagByNameConverter("hr")); + l.add(new RemoveTagByNameConverter("i")); + l.add(new RemoveTagByNameConverter("img")); + l.add(new RemoveTagByNameConverter("input")); + l.add(new RemoveTagByNameConverter("label")); + l.add(new RemoveTagByNameConverter("legend")); + l.add(new RemoveTagByNameConverter("li")); + l.add(new RemoveTagByNameConverter("p")); + l.add(new RemoveTagByNameConverter("script")); + l.add(new RemoveTagByNameConverter("select")); + l.add(new RemoveTagByNameConverter("sl")); + l.add(new RemoveTagByNameConverter("small")); + l.add(new RemoveTagByNameConverter("span")); + l.add(new RemoveTagByNameConverter("strong")); + l.add(new RemoveTagByNameConverter("style")); + l.add(new RemoveTagByNameConverter("table")); + l.add(new RemoveTagByNameConverter("tbody")); + l.add(new RemoveTagByNameConverter("td")); + l.add(new RemoveTagByNameConverter("th")); + l.add(new RemoveTagByNameConverter("thead")); + l.add(new RemoveTagByNameConverter("tr")); + l.add(new RemoveTagByNameConverter("u")); + l.add(new RemoveTagByNameConverter("ul")); // Remove tags that do not impact the data model and cannot be easily used as context for other elements l.add(new RemoveTagByNameConverter("uimessage")); @@ -95,8 +140,8 @@ public List> getTagConverters() { l.add(new FlattenOrExcludeIfModeConverter()); // Flatten any includeIf or excludeIf expressions down into the child tags - l.add(new MergeParentToChildrenConverter("includeIf", " AND ")); - l.add(new MergeParentToChildrenConverter("excludeIf", " OR ")); + l.add(new MergeParentToChildrenConverter("includeIf", " | ")); + l.add(new MergeParentToChildrenConverter("excludeIf", " | ")); // Handle the complex web of velocity conditions for entry of date/location/provider l.add(new MergeSiblingsConverter("encounterDate")); diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java deleted file mode 100644 index 24a5a61a2..000000000 --- a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/Tags.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.openmrs.module.pihcore.htmlformentry.analysis; - -import lombok.Getter; - -public class Tags { - - public enum HFE { - appointments, causeOfDeathList, code, codedOrFreeTextObs, completeProgram, condition, controls, drugOrder, - encounterDate, encounterDiagnosesByObs, encounterDisposition, encounterLocation, encounterProvider, - encounterProviderAndRole, encounterType, encounterVoided, enrollInProgram, excludeIf, exitFromCare, - familyHistoryRelativeCheckboxes, htmlform, ifMode, immunization, includeIf, lookup, macro, macros, - markPatientDead, obs, obsgroup, obsreference, option, order, orderProperty, orderTemplate, - pastMedicalHistoryCheckbox, patient, postSubmissionAction, redirectOnSave, relationship, - render, repeat, section, submit, template, translations, uiInclude, uimessage, uimessages, - variant, when, workflowState - } - - @Getter - public enum Standard { - - a, b, br, button, center, comment("#comment"), dd, div, dl, dt, field, fieldset, font, - h1, h2, h3, h4, h5, h6, h7, hr, i, img, input, label, legend, li, p, script, select, sl, small, span, - strong, style, table, tbody, td, text("#text"), th, thead, tr, u, ul; - - private final String nodeName; - - Standard() { - nodeName = name(); - } - - Standard(String nodeName) { - this.nodeName = nodeName; - } - } -} diff --git a/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/TextConverter.java b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/TextConverter.java new file mode 100644 index 000000000..c939aa181 --- /dev/null +++ b/api/src/main/java/org/openmrs/module/pihcore/htmlformentry/analysis/converter/xml/TextConverter.java @@ -0,0 +1,21 @@ +package org.openmrs.module.pihcore.htmlformentry.analysis.converter.xml; + +import lombok.Data; +import org.openmrs.module.pihcore.htmlformentry.analysis.converter.Converter; + +@Data +public class TextConverter implements Converter { + + private String find; + private String replace; + + public TextConverter(String find, String replace) { + this.find = find; + this.replace = replace; + } + + @Override + public String convert(String input) throws Exception { + return input.replace(find, replace); + } +} diff --git a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java index 278a180d0..fd9052ea4 100644 --- a/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java +++ b/api/src/test/java/org/openmrs/module/pihcore/htmlformentry/analysis/HtmlFormAnalyzerTest.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileWriter; +import java.io.PrintWriter; import java.util.Map; public class HtmlFormAnalyzerTest { @@ -43,4 +44,15 @@ public void writeFormData() throws Exception { } } } + + @Test + public void writeMentalHealthForm() throws Exception { + HtmlFormAnalyzer analyzer = new HtmlFormAnalyzer(); + DataSet dataSet = analyzer.analyze(new File(getHtmlFormsDirectory(), "admissionNote.xml"), new TagWriter()); + try (PrintWriter writer = new PrintWriter(System.out)) { + for (Map rows : dataSet.getRows()) { + writer.write(rows.get("outputData") + System.lineSeparator()); + } + } + } }