".$data."
"); 90 | break; 91 | case self::PAGEBREAK: 92 | $str->append("
"); 121 | for($i = 0, $len = sizeof($entries); $i < $len; $i++){ 122 | $entry = $entries[$i]; 123 | 124 | $str->append(" "); 127 | } 128 | $str->append(" append(">".$entry["title"]."
"; 115 | 116 | return $toc; 117 | } 118 | /** 119 | * Get the images (an array containing the jpeg data). Array entry 0 will 120 | * correspond to image record 0. 121 | * @return array 122 | */ 123 | public function getImages(){ 124 | return $this->images; 125 | } 126 | 127 | /** 128 | * Get the metadata in the form of a hashtable (for example, title or author). 129 | * @return array 130 | */ 131 | public function getMetaData(){ 132 | return $this->metadata; 133 | } 134 | 135 | } 136 | ?> 137 | -------------------------------------------------------------------------------- /MOBIClass/OnlineArticle.php: -------------------------------------------------------------------------------- 1 | init(); 23 | if(!isset($this->metadata["title"])){ 24 | $this->metadata["title"] = CharacterEntities::convert(strip_tags($r->getTitle()->innerHTML)); 25 | } 26 | if(!isset($this->metadata["author"])){ 27 | $parts = parse_url($url); 28 | $this->metadata["author"] = $parts["host"]; 29 | } 30 | 31 | $article = $r->getContent()->innerHTML; 32 | if(substr($article, 0, 5) == "
"; 109 | for($i = 0, $len = sizeof($this->toc); $i < $len; $i++){ 110 | $entry = $this->toc[$i]; 111 | $position = $entry["pos"]+$base; 112 | $toc .= " "; 113 | } 114 | $toc .= " ".($i+1).". ".$entry["title"]."
Para 1
Para 2
Para 1
Para 2
' 21 | * echo "\n\n"; 22 | * 23 | * // set innerHTML 24 | * $elem->innerHTML = 'FiveFilters.org'; 25 | * echo $elem->innerHTML; // prints 'FiveFilters.org' 26 | * echo "\n\n"; 27 | * 28 | * // print document (with our changes) 29 | * echo $doc->saveXML(); 30 | * @endcode 31 | * 32 | * @author Keyvan Minoukadeh - http://www.keyvan.net - keyvan@keyvan.net 33 | * @see http://fivefilters.org (the project this was written for) 34 | */ 35 | class JSLikeHTMLElement extends DOMElement 36 | { 37 | /** 38 | * Used for setting innerHTML like it's done in JavaScript: 39 | * @code 40 | * $div->innerHTML = 'The story begins...
'; 41 | * @endcode 42 | */ 43 | public function __set($name, $value) { 44 | if ($name == 'innerHTML') { 45 | // first, empty the element 46 | for ($x=$this->childNodes->length-1; $x>=0; $x--) { 47 | $this->removeChild($this->childNodes->item($x)); 48 | } 49 | // $value holds our new inner HTML 50 | if ($value != '') { 51 | $f = $this->ownerDocument->createDocumentFragment(); 52 | // appendXML() expects well-formed markup (XHTML) 53 | $result = @$f->appendXML($value); // @ to suppress PHP warnings 54 | if ($result) { 55 | if ($f->hasChildNodes()) $this->appendChild($f); 56 | } else { 57 | // $value is probably ill-formed 58 | $f = new DOMDocument(); 59 | $value = mb_convert_encoding($value, 'HTML-ENTITIES', 'UTF-8'); 60 | // Using', $html); 114 | $html = preg_replace($this->regexps['replaceFonts'], '<$1span>', $html); 115 | $html = mb_convert_encoding($html, 'HTML-ENTITIES', "UTF-8"); 116 | $this->dom = new DOMDocument(); 117 | $this->dom->preserveWhiteSpace = false; 118 | $this->dom->registerNodeClass('DOMElement', 'JSLikeHTMLElement'); 119 | @$this->dom->loadHTML($html); 120 | $this->url = $url; 121 | } 122 | 123 | /** 124 | * Get article title element 125 | * @return DOMElement 126 | */ 127 | public function getTitle() { 128 | return $this->articleTitle; 129 | } 130 | 131 | /** 132 | * Get article content element 133 | * @return DOMElement 134 | */ 135 | public function getContent() { 136 | return $this->articleContent; 137 | } 138 | 139 | /** 140 | * Runs readability. 141 | * 142 | * Workflow: 143 | * 1. Prep the document by removing script tags, css, etc. 144 | * 2. Build readability's DOM tree. 145 | * 3. Grab the article content from the current dom tree. 146 | * 4. Replace the current DOM tree with the new one. 147 | * 5. Read peacefully. 148 | * 149 | * @return boolean true if we found content, false otherwise 150 | **/ 151 | public function init() 152 | { 153 | $this->removeScripts($this->dom); 154 | 155 | // Assume successful outcome 156 | $this->success = true; 157 | 158 | $bodyElems = $this->dom->getElementsByTagName('body'); 159 | if ($bodyElems->length > 0) { 160 | if ($this->bodyCache == null) { 161 | $this->bodyCache = $bodyElems->item(0)->innerHTML; 162 | } 163 | if ($this->body == null) { 164 | $this->body = $bodyElems->item(0); 165 | } 166 | } 167 | 168 | $this->prepDocument(); 169 | 170 | //die($this->dom->documentElement->parentNode->nodeType); 171 | //$this->setInnerHTML($this->dom->documentElement, $this->getInnerHTML($this->dom->documentElement)); 172 | //die($this->getInnerHTML($this->dom->documentElement)); 173 | 174 | /* Build readability's DOM tree */ 175 | $overlay = $this->dom->createElement('div'); 176 | $innerDiv = $this->dom->createElement('div'); 177 | $articleTitle = $this->getArticleTitle(); 178 | $articleContent = $this->grabArticle(); 179 | 180 | if (!$articleContent) { 181 | $this->success = false; 182 | $articleContent = $this->dom->createElement('div'); 183 | $articleContent->setAttribute('id', 'readability-content'); 184 | $articleContent->innerHTML = '
Sorry, Readability was unable to parse this page for content.
'; 185 | } 186 | 187 | $overlay->setAttribute('id', 'readOverlay'); 188 | $innerDiv->setAttribute('id', 'readInner'); 189 | 190 | /* Glue the structure of our document together. */ 191 | $innerDiv->appendChild($articleTitle); 192 | $innerDiv->appendChild($articleContent); 193 | $overlay->appendChild($innerDiv); 194 | 195 | /* Clear the old HTML, insert the new content. */ 196 | $this->body->innerHTML = ''; 197 | $this->body->appendChild($overlay); 198 | //document.body.insertBefore(overlay, document.body.firstChild); 199 | $this->body->removeAttribute('style'); 200 | 201 | $this->postProcessContent($articleContent); 202 | 203 | // Set title and content instance variables 204 | $this->articleTitle = $articleTitle; 205 | $this->articleContent = $articleContent; 206 | 207 | return $this->success; 208 | } 209 | 210 | /** 211 | * Debug 212 | */ 213 | protected function dbg($msg) { 214 | if ($this->debug) echo '* ',$msg, '').replace(readability.regexps.replaceFonts, '<$1span>'); 311 | // We do this in the constructor for PHP as that's when we have raw HTML - before parsing it into a DOM tree. 312 | // Manipulating innerHTML as it's done in JS is not possible in PHP. 313 | } 314 | 315 | /** 316 | * For easier reading, convert this document to have footnotes at the bottom rather than inline links. 317 | * @see http://www.roughtype.com/archives/2010/05/experiments_in.php 318 | * 319 | * @return void 320 | **/ 321 | public function addFootnotes($articleContent) { 322 | $footnotesWrapper = $this->dom->createElement('div'); 323 | $footnotesWrapper->setAttribute('id', 'readability-footnotes'); 324 | $footnotesWrapper->innerHTML = '
tags, etc.
405 | *
406 | * @param DOMElement
407 | * @return void
408 | */
409 | function prepArticle($articleContent) {
410 | $this->cleanStyles($articleContent);
411 | $this->killBreaks($articleContent);
412 | if ($this->revertForcedParagraphElements) {
413 | $this->revertReadabilityStyledElements($articleContent);
414 | }
415 |
416 | /* Clean out junk from the article content */
417 | $this->cleanConditionally($articleContent, 'form');
418 | $this->clean($articleContent, 'object');
419 | $this->clean($articleContent, 'h1');
420 |
421 | /**
422 | * If there is only one h2, they are probably using it
423 | * as a header and not a subheader, so remove it since we already have a header.
424 | ***/
425 | if ($articleContent->getElementsByTagName('h2')->length == 1) {
426 | $this->clean($articleContent, 'h2');
427 | }
428 | $this->clean($articleContent, 'iframe');
429 |
430 | $this->cleanHeaders($articleContent);
431 |
432 | /* Do these last as the previous stuff may have removed junk that will affect these */
433 | $this->cleanConditionally($articleContent, 'table');
434 | $this->cleanConditionally($articleContent, 'ul');
435 | $this->cleanConditionally($articleContent, 'div');
436 |
437 | /* Remove extra paragraphs */
438 | $articleParagraphs = $articleContent->getElementsByTagName('p');
439 | for ($i = $articleParagraphs->length-1; $i >= 0; $i--)
440 | {
441 | $imgCount = $articleParagraphs->item($i)->getElementsByTagName('img')->length;
442 | $embedCount = $articleParagraphs->item($i)->getElementsByTagName('embed')->length;
443 | $objectCount = $articleParagraphs->item($i)->getElementsByTagName('object')->length;
444 |
445 | if ($imgCount === 0 && $embedCount === 0 && $objectCount === 0 && $this->getInnerText($articleParagraphs->item($i), false) == '')
446 | {
447 | $articleParagraphs->item($i)->parentNode->removeChild($articleParagraphs->item($i));
448 | }
449 | }
450 |
451 | try {
452 | $articleContent->innerHTML = preg_replace('/ innerHTML);
453 | //articleContent.innerHTML = articleContent.innerHTML.replace(/ dbg("Cleaning innerHTML of breaks failed. This is an IE strict-block-elements bug. Ignoring.: " . $e);
457 | }
458 | }
459 |
460 | /**
461 | * Initialize a node with the readability object. Also checks the
462 | * className/id for special names to add to its score.
463 | *
464 | * @param Element
465 | * @return void
466 | **/
467 | protected function initializeNode($node) {
468 | $readability = $this->dom->createAttribute('readability');
469 | $readability->value = 0; // this is our contentScore
470 | $node->setAttributeNode($readability);
471 |
472 | switch (strtoupper($node->tagName)) { // unsure if strtoupper is needed, but using it just in case
473 | case 'DIV':
474 | $readability->value += 5;
475 | break;
476 |
477 | case 'PRE':
478 | case 'TD':
479 | case 'BLOCKQUOTE':
480 | $readability->value += 3;
481 | break;
482 |
483 | case 'ADDRESS':
484 | case 'OL':
485 | case 'UL':
486 | case 'DL':
487 | case 'DD':
488 | case 'DT':
489 | case 'LI':
490 | case 'FORM':
491 | $readability->value -= 3;
492 | break;
493 |
494 | case 'H1':
495 | case 'H2':
496 | case 'H3':
497 | case 'H4':
498 | case 'H5':
499 | case 'H6':
500 | case 'TH':
501 | $readability->value -= 5;
502 | break;
503 | }
504 | $readability->value += $this->getClassWeight($node);
505 | }
506 |
507 | /***
508 | * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
509 | * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
510 | *
511 | * @return DOMElement
512 | **/
513 | protected function grabArticle($page=null) {
514 | $stripUnlikelyCandidates = $this->flagIsActive(self::FLAG_STRIP_UNLIKELYS);
515 | if (!$page) $page = $this->dom;
516 | $allElements = $page->getElementsByTagName('*');
517 | /**
518 | * First, node prepping. Trash nodes that look cruddy (like ones with the class name "comment", etc), and turn divs
519 | * into P tags where they have been used inappropriately (as in, where they contain no other block level elements.)
520 | *
521 | * Note: Assignment from index for performance. See http://www.peachpit.com/articles/article.aspx?p=31567&seqNum=5
522 | * TODO: Shouldn't this be a reverse traversal?
523 | **/
524 | $node = null;
525 | $nodesToScore = array();
526 | for ($nodeIndex = 0; ($node = $allElements->item($nodeIndex)); $nodeIndex++) {
527 | //for ($nodeIndex=$targetList->length-1; $nodeIndex >= 0; $nodeIndex--) {
528 | //$node = $targetList->item($nodeIndex);
529 | $tagName = strtoupper($node->tagName);
530 | /* Remove unlikely candidates */
531 | if ($stripUnlikelyCandidates) {
532 | $unlikelyMatchString = $node->getAttribute('class') . $node->getAttribute('id');
533 | if (
534 | preg_match($this->regexps['unlikelyCandidates'], $unlikelyMatchString) &&
535 | !preg_match($this->regexps['okMaybeItsACandidate'], $unlikelyMatchString) &&
536 | $tagName != 'BODY'
537 | )
538 | {
539 | $this->dbg('Removing unlikely candidate - ' . $unlikelyMatchString);
540 | //$nodesToRemove[] = $node;
541 | $node->parentNode->removeChild($node);
542 | $nodeIndex--;
543 | continue;
544 | }
545 | }
546 |
547 | if ($tagName == 'P' || $tagName == 'TD' || $tagName == 'PRE') {
548 | $nodesToScore[] = $node;
549 | }
550 |
551 | /* Turn all divs that don't have children block level elements into p's */
552 | if ($tagName == 'DIV') {
553 | if (!preg_match($this->regexps['divToPElements'], $node->innerHTML)) {
554 | //$this->dbg('Altering div to p');
555 | $newNode = $this->dom->createElement('p');
556 | try {
557 | $newNode->innerHTML = $node->innerHTML;
558 | //$nodesToReplace[] = array('new'=>$newNode, 'old'=>$node);
559 | $node->parentNode->replaceChild($newNode, $node);
560 | $nodeIndex--;
561 | $nodesToScore[] = $node; // or $newNode?
562 | }
563 | catch(Exception $e) {
564 | $this->dbg('Could not alter div to p, reverting back to div.: ' . $e);
565 | }
566 | }
567 | else
568 | {
569 | /* EXPERIMENTAL */
570 | // TODO: change these p elements back to text nodes after processing
571 | for ($i = 0, $il = $node->childNodes->length; $i < $il; $i++) {
572 | $childNode = $node->childNodes->item($i);
573 | if ($childNode->nodeType == 3) { // XML_TEXT_NODE
574 | //$this->dbg('replacing text node with a p tag with the same content.');
575 | $p = $this->dom->createElement('p');
576 | $p->innerHTML = $childNode->nodeValue;
577 | $p->setAttribute('style', 'display: inline;');
578 | $p->setAttribute('class', 'readability-styled');
579 | $childNode->parentNode->replaceChild($p, $childNode);
580 | }
581 | }
582 | }
583 | }
584 | }
585 |
586 | /**
587 | * Loop through all paragraphs, and assign a score to them based on how content-y they look.
588 | * Then add their score to their parent node.
589 | *
590 | * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
591 | **/
592 | $candidates = array();
593 | for ($pt=0; $pt < count($nodesToScore); $pt++) {
594 | $parentNode = $nodesToScore[$pt]->parentNode;
595 | // $grandParentNode = $parentNode ? $parentNode->parentNode : null;
596 | $grandParentNode = !$parentNode ? null : (($parentNode->parentNode instanceof DOMElement) ? $parentNode->parentNode : null);
597 | $innerText = $this->getInnerText($nodesToScore[$pt]);
598 |
599 | if (!$parentNode || !isset($parentNode->tagName)) {
600 | continue;
601 | }
602 |
603 | /* If this paragraph is less than 25 characters, don't even count it. */
604 | if(strlen($innerText) < 25) {
605 | continue;
606 | }
607 |
608 | /* Initialize readability data for the parent. */
609 | if (!$parentNode->hasAttribute('readability'))
610 | {
611 | $this->initializeNode($parentNode);
612 | $candidates[] = $parentNode;
613 | }
614 |
615 | /* Initialize readability data for the grandparent. */
616 | if ($grandParentNode && !$grandParentNode->hasAttribute('readability') && isset($grandParentNode->tagName))
617 | {
618 | $this->initializeNode($grandParentNode);
619 | $candidates[] = $grandParentNode;
620 | }
621 |
622 | $contentScore = 0;
623 |
624 | /* Add a point for the paragraph itself as a base. */
625 | $contentScore++;
626 |
627 | /* Add points for any commas within this paragraph */
628 | $contentScore += count(explode(',', $innerText));
629 |
630 | /* For every 100 characters in this paragraph, add another point. Up to 3 points. */
631 | $contentScore += min(floor(strlen($innerText) / 100), 3);
632 |
633 | /* Add the score to the parent. The grandparent gets half. */
634 | $parentNode->getAttributeNode('readability')->value += $contentScore;
635 |
636 | if ($grandParentNode) {
637 | $grandParentNode->getAttributeNode('readability')->value += $contentScore/2;
638 | }
639 | }
640 |
641 | /**
642 | * After we've calculated scores, loop through all of the possible candidate nodes we found
643 | * and find the one with the highest score.
644 | **/
645 | $topCandidate = null;
646 | for ($c=0, $cl=count($candidates); $c < $cl; $c++)
647 | {
648 | /**
649 | * Scale the final candidates score based on link density. Good content should have a
650 | * relatively small link density (5% or less) and be mostly unaffected by this operation.
651 | **/
652 | $readability = $candidates[$c]->getAttributeNode('readability');
653 | $readability->value = $readability->value * (1-$this->getLinkDensity($candidates[$c]));
654 |
655 | $this->dbg('Candidate: ' . $candidates[$c]->tagName . ' (' . $candidates[$c]->getAttribute('class') . ':' . $candidates[$c]->getAttribute('id') . ') with score ' . $readability->value);
656 |
657 | if (!$topCandidate || $readability->value > (int)$topCandidate->getAttribute('readability')) {
658 | $topCandidate = $candidates[$c];
659 | }
660 | }
661 |
662 | /**
663 | * If we still have no top candidate, just use the body as a last resort.
664 | * We also have to copy the body node so it is something we can modify.
665 | **/
666 | if ($topCandidate === null || strtoupper($topCandidate->tagName) == 'BODY')
667 | {
668 | $topCandidate = $this->dom->createElement('div');
669 | $topCandidate->innerHTML = ($page instanceof DOMDocument) ? $page->saveXML($page->documentElement) : $page->innerHTML;
670 | $page->innerHTML = '';
671 | $page->appendChild($topCandidate);
672 | $this->initializeNode($topCandidate);
673 | }
674 |
675 | /**
676 | * Now that we have the top candidate, look through its siblings for content that might also be related.
677 | * Things like preambles, content split by ads that we removed, etc.
678 | **/
679 | $articleContent = $this->dom->createElement('div');
680 | $articleContent->setAttribute('id', 'readability-content');
681 | $siblingScoreThreshold = max(10, ((int)$topCandidate->getAttribute('readability')) * 0.2);
682 | $siblingNodes = $topCandidate->parentNode->childNodes;
683 |
684 | for ($s=0, $sl=$siblingNodes->length; $s < $sl; $s++)
685 | {
686 | $siblingNode = $siblingNodes->item($s);
687 | $append = false;
688 |
689 | $this->dbg('Looking at sibling node: ' . $siblingNode->nodeName . (($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability')) ? (' with score ' . $siblingNode->getAttribute('readability')) : ''));
690 |
691 | //dbg('Sibling has score ' . ($siblingNode->readability ? siblingNode.readability.contentScore : 'Unknown'));
692 |
693 | if ($siblingNode === $topCandidate)
694 | // or if ($siblingNode->isSameNode($topCandidate))
695 | {
696 | $append = true;
697 | }
698 |
699 | $contentBonus = 0;
700 | /* Give a bonus if sibling nodes and top candidates have the example same classname */
701 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->getAttribute('class') == $topCandidate->getAttribute('class') && $topCandidate->getAttribute('class') != '') {
702 | $contentBonus += ((int)$topCandidate->getAttribute('readability')) * 0.2;
703 | }
704 |
705 | if ($siblingNode->nodeType === XML_ELEMENT_NODE && $siblingNode->hasAttribute('readability') && (((int)$siblingNode->getAttribute('readability')) + $contentBonus) >= $siblingScoreThreshold)
706 | {
707 | $append = true;
708 | }
709 |
710 | if (strtoupper($siblingNode->nodeName) == 'P') {
711 | $linkDensity = $this->getLinkDensity($siblingNode);
712 | $nodeContent = $this->getInnerText($siblingNode);
713 | $nodeLength = strlen($nodeContent);
714 |
715 | if ($nodeLength > 80 && $linkDensity < 0.25)
716 | {
717 | $append = true;
718 | }
719 | else if ($nodeLength < 80 && $linkDensity === 0 && preg_match('/\.( |$)/', $nodeContent))
720 | {
721 | $append = true;
722 | }
723 | }
724 |
725 | if ($append)
726 | {
727 | $this->dbg('Appending node: ' . $siblingNode->nodeName);
728 |
729 | $nodeToAppend = null;
730 | $sibNodeName = strtoupper($siblingNode->nodeName);
731 | if ($sibNodeName != 'DIV' && $sibNodeName != 'P') {
732 | /* We have a node that isn't a common block level element, like a form or td tag. Turn it into a div so it doesn't get filtered out later by accident. */
733 |
734 | $this->dbg('Altering siblingNode of ' . $sibNodeName . ' to div.');
735 | $nodeToAppend = $this->dom->createElement('div');
736 | try {
737 | $nodeToAppend->setAttribute('id', $siblingNode->getAttribute('id'));
738 | $nodeToAppend->innerHTML = $siblingNode->innerHTML;
739 | }
740 | catch(Exception $e)
741 | {
742 | $this->dbg('Could not alter siblingNode to div, reverting back to original.');
743 | $nodeToAppend = $siblingNode;
744 | $s--;
745 | $sl--;
746 | }
747 | } else {
748 | $nodeToAppend = $siblingNode;
749 | $s--;
750 | $sl--;
751 | }
752 |
753 | /* To ensure a node does not interfere with readability styles, remove its classnames */
754 | $nodeToAppend->removeAttribute('class');
755 |
756 | /* Append sibling and subtract from our list because it removes the node when you append to another node */
757 | $articleContent->appendChild($nodeToAppend);
758 | }
759 | }
760 |
761 | /**
762 | * So we have all of the content that we need. Now we clean it up for presentation.
763 | **/
764 | $this->prepArticle($articleContent);
765 |
766 | /**
767 | * Now that we've gone through the full algorithm, check to see if we got any meaningful content.
768 | * If we didn't, we may need to re-run grabArticle with different flags set. This gives us a higher
769 | * likelihood of finding the content, and the sieve approach gives us a higher likelihood of
770 | * finding the -right- content.
771 | **/
772 | if (strlen($this->getInnerText($articleContent, false)) < 250)
773 | {
774 | $this->body->innerHTML = $this->bodyCache;
775 |
776 | if ($this->flagIsActive(self::FLAG_STRIP_UNLIKELYS)) {
777 | $this->removeFlag(self::FLAG_STRIP_UNLIKELYS);
778 | return $this->grabArticle($this->body);
779 | }
780 | else if ($this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
781 | $this->removeFlag(self::FLAG_WEIGHT_CLASSES);
782 | return $this->grabArticle($this->body);
783 | }
784 | else if ($this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
785 | $this->removeFlag(self::FLAG_CLEAN_CONDITIONALLY);
786 | return $this->grabArticle($this->body);
787 | }
788 | else {
789 | return false;
790 | }
791 | }
792 | return $articleContent;
793 | }
794 |
795 | /**
796 | * Remove script tags from document
797 | *
798 | * @param DOMElement
799 | * @return void
800 | */
801 | public function removeScripts($doc) {
802 | $scripts = $doc->getElementsByTagName('script');
803 | for($i = $scripts->length-1; $i >= 0; $i--)
804 | {
805 | $scripts->item($i)->parentNode->removeChild($scripts->item($i));
806 | }
807 | }
808 |
809 | /**
810 | * Get the inner text of a node.
811 | * This also strips out any excess whitespace to be found.
812 | *
813 | * @param DOMElement $
814 | * @param boolean $normalizeSpaces (default: true)
815 | * @return string
816 | **/
817 | public function getInnerText($e, $normalizeSpaces=true) {
818 | $textContent = '';
819 |
820 | if (!isset($e->textContent) || $e->textContent == '') {
821 | return '';
822 | }
823 |
824 | $textContent = trim($e->textContent);
825 |
826 | if ($normalizeSpaces) {
827 | return preg_replace($this->regexps['normalize'], ' ', $textContent);
828 | } else {
829 | return $textContent;
830 | }
831 | }
832 |
833 | /**
834 | * Get the number of times a string $s appears in the node $e.
835 | *
836 | * @param DOMElement $e
837 | * @param string - what to count. Default is ","
838 | * @return number (integer)
839 | **/
840 | public function getCharCount($e, $s=',') {
841 | return substr_count($this->getInnerText($e), $s);
842 | }
843 |
844 | /**
845 | * Remove the style attribute on every $e and under.
846 | *
847 | * @param DOMElement $e
848 | * @return void
849 | */
850 | public function cleanStyles($e) {
851 | $elems = $e->getElementsByTagName('*');
852 | foreach ($elems as $elem) {
853 | $elem->removeAttribute('style');
854 | }
855 | }
856 |
857 | /**
858 | * Get the density of links as a percentage of the content
859 | * This is the amount of text that is inside a link divided by the total text in the node.
860 | *
861 | * @param DOMElement $e
862 | * @return number (float)
863 | */
864 | public function getLinkDensity($e) {
865 | $links = $e->getElementsByTagName('a');
866 | $textLength = strlen($this->getInnerText($e));
867 | $linkLength = 0;
868 | for ($i=0, $il=$links->length; $i < $il; $i++)
869 | {
870 | $linkLength += strlen($this->getInnerText($links->item($i)));
871 | }
872 | if ($textLength > 0) {
873 | return $linkLength / $textLength;
874 | } else {
875 | return 0;
876 | }
877 | }
878 |
879 | /**
880 | * Get an elements class/id weight. Uses regular expressions to tell if this
881 | * element looks good or bad.
882 | *
883 | * @param DOMElement $e
884 | * @return number (Integer)
885 | */
886 | public function getClassWeight($e) {
887 | if(!$this->flagIsActive(self::FLAG_WEIGHT_CLASSES)) {
888 | return 0;
889 | }
890 |
891 | $weight = 0;
892 |
893 | /* Look for a special classname */
894 | if ($e->hasAttribute('class') && $e->getAttribute('class') != '')
895 | {
896 | if (preg_match($this->regexps['negative'], $e->getAttribute('class'))) {
897 | $weight -= 25;
898 | }
899 | if (preg_match($this->regexps['positive'], $e->getAttribute('class'))) {
900 | $weight += 25;
901 | }
902 | }
903 |
904 | /* Look for a special ID */
905 | if ($e->hasAttribute('id') && $e->getAttribute('id') != '')
906 | {
907 | if (preg_match($this->regexps['negative'], $e->getAttribute('id'))) {
908 | $weight -= 25;
909 | }
910 | if (preg_match($this->regexps['positive'], $e->getAttribute('id'))) {
911 | $weight += 25;
912 | }
913 | }
914 | return $weight;
915 | }
916 |
917 | /**
918 | * Remove extraneous break tags from a node.
919 | *
920 | * @param DOMElement $node
921 | * @return void
922 | */
923 | public function killBreaks($node) {
924 | $html = $node->innerHTML;
925 | $html = preg_replace($this->regexps['killBreaks'], '
', $html);
926 | $node->innerHTML = $html;
927 | }
928 |
929 | /**
930 | * Clean a node of all elements of type "tag".
931 | * (Unless it's a youtube/vimeo video. People love movies.)
932 | *
933 | * @param DOMElement $e
934 | * @param string $tag
935 | * @return void
936 | */
937 | public function clean($e, $tag) {
938 | $targetList = $e->getElementsByTagName($tag);
939 | $isEmbed = ($tag == 'object' || $tag == 'embed');
940 |
941 | for ($y=$targetList->length-1; $y >= 0; $y--) {
942 | /* Allow youtube and vimeo videos through as people usually want to see those. */
943 | if ($isEmbed) {
944 | $attributeValues = '';
945 | for ($i=0, $il=$targetList->item($y)->attributes->length; $i < $il; $i++) {
946 | $attributeValues .= $targetList->item($y)->attributes->item($i)->value . '|'; // DOMAttr? (TODO: test)
947 | }
948 |
949 | /* First, check the elements attributes to see if any of them contain youtube or vimeo */
950 | if (preg_match($this->regexps['video'], $attributeValues)) {
951 | continue;
952 | }
953 |
954 | /* Then check the elements inside this element for the same. */
955 | if (preg_match($this->regexps['video'], $targetList->item($y)->innerHTML)) {
956 | continue;
957 | }
958 | }
959 | $targetList->item($y)->parentNode->removeChild($targetList->item($y));
960 | }
961 | }
962 |
963 | /**
964 | * Clean an element of all tags of type "tag" if they look fishy.
965 | * "Fishy" is an algorithm based on content length, classnames,
966 | * link density, number of images & embeds, etc.
967 | *
968 | * @param DOMElement $e
969 | * @param string $tag
970 | * @return void
971 | */
972 | public function cleanConditionally($e, $tag) {
973 | if (!$this->flagIsActive(self::FLAG_CLEAN_CONDITIONALLY)) {
974 | return;
975 | }
976 |
977 | $tagsList = $e->getElementsByTagName($tag);
978 | $curTagsLength = $tagsList->length;
979 |
980 | /**
981 | * Gather counts for other typical elements embedded within.
982 | * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
983 | *
984 | * TODO: Consider taking into account original contentScore here.
985 | */
986 | for ($i=$curTagsLength-1; $i >= 0; $i--) {
987 | $weight = $this->getClassWeight($tagsList->item($i));
988 | $contentScore = ($tagsList->item($i)->hasAttribute('readability')) ? (int)$tagsList->item($i)->getAttribute('readability') : 0;
989 |
990 | $this->dbg('Cleaning Conditionally ' . $tagsList->item($i)->tagName . ' (' . $tagsList->item($i)->getAttribute('class') . ':' . $tagsList->item($i)->getAttribute('id') . ')' . (($tagsList->item($i)->hasAttribute('readability')) ? (' with score ' . $tagsList->item($i)->getAttribute('readability')) : ''));
991 |
992 | if ($weight + $contentScore < 0) {
993 | $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
994 | }
995 | else if ( $this->getCharCount($tagsList->item($i), ',') < 10) {
996 | /**
997 | * If there are not very many commas, and the number of
998 | * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
999 | **/
1000 | $p = $tagsList->item($i)->getElementsByTagName('p')->length;
1001 | $img = $tagsList->item($i)->getElementsByTagName('img')->length;
1002 | $li = $tagsList->item($i)->getElementsByTagName('li')->length-100;
1003 | $input = $tagsList->item($i)->getElementsByTagName('input')->length;
1004 |
1005 | $embedCount = 0;
1006 | $embeds = $tagsList->item($i)->getElementsByTagName('embed');
1007 | for ($ei=0, $il=$embeds->length; $ei < $il; $ei++) {
1008 | if (preg_match($this->regexps['video'], $embeds->item($ei)->getAttribute('src'))) {
1009 | $embedCount++;
1010 | }
1011 | }
1012 |
1013 | $linkDensity = $this->getLinkDensity($tagsList->item($i));
1014 | $contentLength = strlen($this->getInnerText($tagsList->item($i)));
1015 | $toRemove = false;
1016 |
1017 | if ( $img > $p ) {
1018 | $toRemove = true;
1019 | } else if ($li > $p && $tag != 'ul' && $tag != 'ol') {
1020 | $toRemove = true;
1021 | } else if ( $input > floor($p/3) ) {
1022 | $toRemove = true;
1023 | } else if ($contentLength < 25 && ($img === 0 || $img > 2) ) {
1024 | $toRemove = true;
1025 | } else if($weight < 25 && $linkDensity > 0.2) {
1026 | $toRemove = true;
1027 | } else if($weight >= 25 && $linkDensity > 0.5) {
1028 | $toRemove = true;
1029 | } else if(($embedCount == 1 && $contentLength < 75) || $embedCount > 1) {
1030 | $toRemove = true;
1031 | }
1032 |
1033 | if ($toRemove) {
1034 | $tagsList->item($i)->parentNode->removeChild($tagsList->item($i));
1035 | }
1036 | }
1037 | }
1038 | }
1039 |
1040 | /**
1041 | * Clean out spurious headers from an Element. Checks things like classnames and link density.
1042 | *
1043 | * @param DOMElement $e
1044 | * @return void
1045 | */
1046 | public function cleanHeaders($e) {
1047 | for ($headerIndex = 1; $headerIndex < 3; $headerIndex++) {
1048 | $headers = $e->getElementsByTagName('h' . $headerIndex);
1049 | for ($i=$headers->length-1; $i >=0; $i--) {
1050 | if ($this->getClassWeight($headers->item($i)) < 0 || $this->getLinkDensity($headers->item($i)) > 0.33) {
1051 | $headers->item($i)->parentNode->removeChild($headers->item($i));
1052 | }
1053 | }
1054 | }
1055 | }
1056 |
1057 | public function flagIsActive($flag) {
1058 | return ($this->flags & $flag) > 0;
1059 | }
1060 |
1061 | public function addFlag($flag) {
1062 | $this->flags = $this->flags | $flag;
1063 | }
1064 |
1065 | public function removeFlag($flag) {
1066 | $this->flags = $this->flags & ~$flag;
1067 | }
1068 | }
1069 | ?>
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | phpMobi file generator
2 | ======================
3 |
4 | phpMobi is a php script that can generate .mobi files from valid html
5 | files. While this was meant as an experiment, this tool works quite
6 | well and can be used to generate mobipocket files from most news articles.
7 |
8 | IMPORTANT: Do NOT use this on a public web server: most of it was coded in
9 | a weekend, with no testing and no special attention to security. Also, as no official
10 | documentation for the MOBI file format is available, there will be some bugs/problems
11 | in the generated files, but it works for relatively simple documents on the Kindle
12 | previewer and the Kindle 3.
13 |
14 | MobiPocket is an eBook format created by Mobipocket SA. This tool also
15 | uses a php readability port made by [Keyvan Minoukadeh](http://www.keyvan.net/2010/08/php-readability/).
16 |
17 | Code sample
18 | ------------
19 |
20 | See index.php for an example of using this program.
21 |
22 | Sending an online article as a download:
23 |
24 | //Create the MOBI object
25 | $mobi = new MOBI();
26 |
27 | //Set the content provider
28 | $content = new OnlineArticle("URL");
29 | $mobi->setContentProvider($content);
30 |
31 | //Get title and make it a 12 character long url-safe filename
32 | $title = $mobi->getTitle();
33 | if($title === false)
34 | $title = "file";
35 |
36 | $title = urlencode(str_replace(" ", "_", strtolower(substr($title, 0, 12))));
37 |
38 | //Send the mobi file as download
39 | $mobi->download($title.".mobi");
40 |
41 | Using a previously generated/downloaded html file (will not download any images!):
42 |
43 | $data = "...";
44 | $options = array(
45 | "title" => "Local document",
46 | "author" => "Author name",
47 | "subject" => "Subject"
48 | );
49 |
50 | //Create the MOBI object
51 | $mobi = new MOBI();
52 |
53 | //Set the data
54 | $mobi->setData($data);
55 | $mobi->setOptions($options);
56 |
57 | //Save the mobi file locally
58 | $mobi->save($options["title"].".mobi");
59 |
60 | Implementation
61 | --------------
62 |
63 | This code was implemented while reverse-engineering the MobiPocket format.
64 | Therefore this code absolutely isn't optimized for speed, but rather for
65 | easy changes, as getting it to produce valid files was quite fiddly.
66 |
67 | Features
68 | --------
69 |
70 | Modular content provider system:
71 | Adding a new data source can be done by extending the ContentProvider
72 | class. See the OnlineArticle class for a simple but complete
73 | implementation of such a system.
74 |
75 | Image support:
76 | By default, the online article downloader (and any other content
77 | provider that supports images) will download images and integrate them
78 | into the mobi file.
79 |
80 | Partial UTF-8 support:
81 | In practice UTF-8 just works, but there are some unhandled corner
82 | cases (see missing features).
83 |
84 | Missing Features
85 | ----------------
86 |
87 | Compression:
88 | This won't be implemented (or if it is, only to serve as a
89 | reference of the format).
90 |
91 | Different eBook types:
92 | MobiPocket supports other formats/layouts, such as newspaper-like
93 | formats. At the moment only the book layout has been implemented.
94 |
95 | Full UTF-8 support:
96 | UTF-8 should work most of the time (it worked every time I
97 | tested it), but there might be some problems when the character
98 | is split over two "records".
99 |
100 | License
101 | -------
102 | This code is released under the Apache license (version 2.0)
103 |
--------------------------------------------------------------------------------
/index.php:
--------------------------------------------------------------------------------
1 | setContentProvider($content);
29 |
30 | //Get title and make it a 12 character long filename
31 | $title = $mobi->getTitle();
32 | if($title === false) $title = "file";
33 | $title = urlencode(str_replace(" ", "_", strtolower(substr($title, 0, 12))));
34 |
35 | //Send the mobi file as download
36 | $mobi->download($title.".mobi");
37 | die;
38 | }else{
39 | //Create the mobi object
40 | $mobi = new MOBI();
41 |
42 | $content = new MOBIFile();
43 |
44 | $content->set("title", "My first eBook");
45 | $content->set("author", "Me");
46 |
47 | $content->appendChapterTitle("Introduction");
48 | for($i = 0, $lenI = rand(5, 10); $i < $lenI; $i++){
49 | $content->appendParagraph("P".($i+1));
50 | }
51 |
52 |
53 | //Based on PHP's imagecreatetruecolor help paage
54 | $im = imagecreatetruecolor(220, 200);
55 | $text_color = imagecolorallocate($im, 233, 14, 91);
56 | imagestring($im, 10, 5, 5, 'A Simple Text String', $text_color);
57 | imagestring($im, 5, 15, 75, 'A Simple Text String', $text_color);
58 | imagestring($im, 3, 25, 125, 'A Simple Text String', $text_color);
59 | imagestring($im, 2, 10, 155, 'A Simple Text String', $text_color);
60 | $content->appendImage($im);
61 | imagedestroy($im);
62 |
63 | $content->appendPageBreak();
64 |
65 | for($i = 0, $lenI = rand(10, 15); $i < $lenI; $i++){
66 | $content->appendChapterTitle(($i+1).". Chapter ".($i+1));
67 |
68 | for($j = 0, $lenJ = rand(20, 40); $j < $lenJ; $j++){
69 | $content->appendParagraph("P".($i+1).".".($j+1)." TEXT TEXT TEXT");
70 | }
71 |
72 | $content->appendPageBreak();
73 | }
74 |
75 | $mobi->setContentProvider($content);
76 |
77 | //Get title and make it a 12 character long filename
78 | $title = $mobi->getTitle();
79 | if($title === false) $title = "file";
80 | $title = urlencode(str_replace(" ", "_", strtolower(substr($title, 0, 12))));
81 |
82 | //Send the mobi file as download
83 | $mobi->download($title.".mobi");
84 | die;
85 | }
86 | }
87 | ?>
88 |
89 |
90 |