Changeset 628

Timestamp:

12/16/13 16:31:52 (11 years ago)

Author:

ngw8

Message:

Much better support for wrapped text when converting web pages, strips out all links and in-paragraph formatting and is a bit messy + slow currently

Location:

trunk/src/org/expeditee

Files:

• io/WebParser.java (modified) (4 diffs)
• reflection/JavaFX.java (modified) (2 diffs)

trunk/src/org/expeditee/io/WebParser.java

@@ -24 +24 @@
 import org.expeditee.items.Text;
 import org.expeditee.reflection.JavaFX;
+import org.w3c.dom.Element;
 import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
 import org.w3c.dom.html.HTMLBodyElement;
 
@@ -136 +138 @@
                                                                 new Object[] { "background-color" })));
 
+                                                // Following 2 functions taken from jQuery (Apache 2 License)
+                                        /*      JavaFX.WebEngineExecuteScript
+                                                                .invoke(webEngine,
+                                                                                "function getText ( elem ) { var i, node, nodeType = elem.nodeType, ret = ''; if ( nodeType ) { if ( nodeType === 1 || nodeType === 9 ) { if ( typeof elem.textContent === 'string' ) { return elem.textContent; } else if ( typeof elem.innerText === 'string' ) { return elem.innerText.replace( rReturn, '' ); } else { for ( elem = elem.firstChild; elem; elem = elem.nextSibling) { ret += getText( elem ); } } } else if ( nodeType === 3 || nodeType === 4 ) { return elem.nodeValue; } } else { for ( i = 0; (node = elem[i]); i++ ) { if ( node.nodeType !== 8 ) { ret += getText( node ); } } } return ret; };");
+                                                JavaFX.WebEngineExecuteScript
+                                                                .invoke(webEngine,
+                                                                                "function myText(elem) { var i, node, nodeType = elem.nodeType, ret = ''; if (nodeType) { if (nodeType === 1 || nodeType === 9 || nodeType === 11) { if (typeof elem.textContent === 'string') { return elem.textContent; } else if (typeof elem.innerText === 'string') { return elem.innerText.replace(rReturn, ''); } else { for (elem = elem.firstChild; elem; elem = elem.nextSibling) { ret += getText(elem); } } } else if (nodeType === 3 || nodeType === 4) { return elem.nodeValue; } } else { for (i = 0; (node = elem[i]); i++) { if (node.nodeType !== 8) { ret += getText(node); } } } return ret; }");
+                
+                                                JavaFX.WebEngineExecuteScript.invoke(webEngine, 
+                                                                "var ps = document.getElementsByTagName('p');"
+                                                                + "for(var i = 0; i < ps.length; i++) {"
+                                                                        + "ps[i].innerHTML = myText(ps[i]);"
+                                                                        + "var span = document.createElement('replacement');"
+                                                                        + "span.textContent = ps[i].textContent;"
+                                                                        + " ps[i].innerHTML = span.outerHTML;"
+                                                                + "}" );
+                                        */
+                                                
+                                                JavaFX.WebEngineExecuteScript
+                                                                .invoke(webEngine,
+                                                                                "function isContent(el) { "
+                                                                                + "             if (el.tagName === 'li' && window.getComputedStyle(el).getPropertyValue('list-style-type') !== 'none') { "
+                                                                                + "                     return true; "
+                                                                                + "             } "
+                                                                                + "             var text = el.textContent.replace(/(^\\s*)|(\\s*$)/, ''); "
+                                                                                + "             var w = text.split(/\\S\\s\\S/).length; var punct = text.split(/\\.|\\,|\\!/).length; "
+                                                                                + "             if (punct < 2 || w < 10) { "
+                                                                                + "                     return false;"
+                                                                                + "             } else { "
+                                                                                + "                     return true;"
+                                                                                + "             } "
+                                                                                + "} "
+                                                                                + "function addToSpan(text) {"
+                                                                                + "             span = document.createElement('wordSpan');"
+                                                                                + "             span.textContent = text + ' '; "
+                                                                                + "             para.appendChild(span); "
+                                                                                + "             if (prevSpan !== null && span.getBoundingClientRect().top > prevSpan.getBoundingClientRect().top) { "
+                                                                                + "                     span.textContent = '\\n' + span.textContent;"
+                                                                                + "                     if (prevPrevSpan !== null && prevPrevSpan.getBoundingClientRect().left == prevSpan.getBoundingClientRect().left) {"
+                                                                                + "                             prevPrevSpan.textContent = prevPrevSpan.textContent + prevSpan.textContent;"
+                                                                                + "                             para.removeChild(prevSpan);"
+                                                                                + "                     } else {"
+                                                                                + "                             prevPrevSpan = prevSpan;"
+                                                                                + "                     }"
+                                                                                + "                     prevSpan = span;"
+                                                                                
+                                                                                + "             } else {"
+                                                                                + "                     prevSpan.textContent = prevSpan.textContent + span.textContent;"
+                                                                                + "                     para.removeChild(span);"
+                                                                                + "             }"
+                                                                                + "}"   
+                                                );
+                                        
+                                                Object contentElements = JavaFX.WebEngineExecuteScript.invoke(webEngine, "document.querySelectorAll('p,li');");
+                                                int contentElementsLength = (Integer) JavaFX.JSObjectGetMember.invoke(contentElements, "length");
+
+                                                for (int i = 0; i < contentElementsLength; i++) {
+                                                        Element currentNode = (Element) JavaFX.JSObjectGetSlot.invoke(contentElements, i);
+                                                        // JavaFX.JSObjectCall.invoke(window, "addBreaks", new Object[] { currentNode });
+                                                        JavaFX.JSObjectSetMember.invoke(window, "para", currentNode);
+                                                        
+                                                        JavaFX.WebEngineExecuteScript.invoke(webEngine, ""
+                                                                        + "var exp = {};"
+                                                                        + "var span = document.createElement('wordSpan');"
+                                                                        + "var prevSpan = span;"
+                                                                        + "var prevPrevSpan = null;"
+                                                                        );
+                                                        
+                                                        if ((boolean) (JavaFX.WebEngineExecuteScript.invoke(webEngine, "isContent(para)"))) {
+                                                                Object paraWords = JavaFX.WebEngineExecuteScript.invoke(webEngine, "para.textContent.split(/\\s+/);");
+                                                                int paraWordsLength = (Integer) JavaFX.JSObjectGetMember.invoke(paraWords, "length");
+                                                                
+                                                                JavaFX.WebEngineExecuteScript.invoke(webEngine, ""
+                                                                                + "para.innerHTML = '';"
+                                                                                + "para.style.wordBreak = 'normal';"
+                                                                                );
+                                                                
+                                                                for (int j = 0; j < paraWordsLength; j++) {
+                                                                        Object currentWord = JavaFX.JSObjectGetSlot.invoke(paraWords, j);
+                                                                        JavaFX.JSObjectCall.invoke(window, "addToSpan", new Object[] { currentWord });
+                                                                }
+                                                                
+                                                                JavaFX.WebEngineExecuteScript.invoke(webEngine, ""
+                                                                                + "                     if (prevPrevSpan !== null && prevPrevSpan.getBoundingClientRect().left == prevSpan.getBoundingClientRect().left) {"
+                                                                                + "                             prevPrevSpan.textContent = prevPrevSpan.textContent + prevSpan.textContent;"
+                                                                                + "                             para.removeChild(prevSpan);"
+                                                                                + "                     }"
+                                                                                );
+                                                        }
+                                                }
+                                                
                                                 // Using Javascript to get an array of all the nodes in the document
                                                 Object nodes = JavaFX.WebEngineExecuteScript.invoke(webEngine,
@@ -151 +244 @@
                                                 for (int i = 0; i < nodesLength; i++) {
                                                         Node currentNode = (Node) JavaFX.JSObjectGetSlot.invoke(nodes, i);
-                                                        
+
                                                         if (currentNode.getNodeType() == Node.TEXT_NODE || currentNode.getNodeType() == Node.ELEMENT_NODE) {
 
@@ -251 +344 @@
                                                                                 }
 
+                                                                                Node siblingNode;
+                                                                                NodeList siblings = currentNode.getParentNode().getChildNodes();
+
+                                                                                String combined = currentNode.getTextContent();
+
+                                                                                for (int j = 0; j < siblings.getLength(); j++) {
+                                                                                        siblingNode = siblings.item(j);
+                                                                                        if (siblingNode.getNodeType() == Node.TEXT_NODE) {
+                                                                                                System.out.println("sibling is text");
+                                                                                                combined = combined + siblingNode.getTextContent();
+                                                                                        }
+                                                                                }
+
+                                                                                System.out.println("Combined: " + combined);
+
+
                                                                                 Text t;
 
-                                                                                t = frame.addText((int) x, (int) y, currentNode.getTextContent().replaceAll("\\s+", " ").trim(), null);
+                                                                                t = frame.addText((int) x, (int) y, currentNode.getTextContent().replaceAll("[^\\S\\n]+", " ").trim(), null);
 
                                                                                 t.setColor(rgbStringToColor(color));

trunk/src/org/expeditee/reflection/JavaFX.java

@@ -66 +66 @@
         public static Method JSObjectCall;
         public static Method JSObjectGetMember;
+        public static Method JSObjectSetMember;
         public static Method JSObjectGetSlot;
 
@@ -139 +140 @@
                         JSObjectCall = JSObject.getMethod("call", String.class, Object[].class);
                         JSObjectGetMember = JSObject.getMethod("getMember", String.class);
+                        JSObjectSetMember = JSObject.getMethod("setMember", String.class, Object.class);
                         JSObjectGetSlot = JSObject.getMethod("getSlot", int.class);