\nx\n>\n
\U0001d4b5 \U0001d538
𝒵 𝔸
foo <bad>bar</bad> baz
foo bar baz
Hello World! -------------------------------------------------------------------------------- /lib/html5lib/tests/utf-8-bom.html: -------------------------------------------------------------------------------- 1 | 2 |
Hello World! © -------------------------------------------------------------------------------- /lib/html5lib/treebuilders/__init__.py: -------------------------------------------------------------------------------- 1 | """A collection of modules for building different kinds of tree from 2 | HTML documents. 3 | 4 | To create a treebuilder for a new type of tree, you need to do 5 | implement several things: 6 | 7 | 1) A set of classes for various types of elements: Document, Doctype, 8 | Comment, Element. These must implement the interface of 9 | _base.treebuilders.Node (although comment nodes have a different 10 | signature for their constructor, see treebuilders.simpletree.Comment) 11 | Textual content may also be implemented as another node type, or not, as 12 | your tree implementation requires. 13 | 14 | 2) A treebuilder object (called TreeBuilder by convention) that 15 | inherits from treebuilders._base.TreeBuilder. This has 4 required attributes: 16 | documentClass - the class to use for the bottommost node of a document 17 | elementClass - the class to use for HTML Elements 18 | commentClass - the class to use for comments 19 | doctypeClass - the class to use for doctypes 20 | It also has one required method: 21 | getDocument - Returns the root node of the complete document tree 22 | 23 | 3) If you wish to run the unit tests, you must also create a 24 | testSerializer method on your treebuilder which accepts a node and 25 | returns a string containing Node and its children serialized according 26 | to the format used in the unittests 27 | 28 | The supplied simpletree module provides a python-only implementation 29 | of a full treebuilder and is a useful reference for the semantics of 30 | the various methods. 31 | """ 32 | 33 | treeBuilderCache = {} 34 | 35 | import sys 36 | 37 | def getTreeBuilder(treeType, implementation=None, **kwargs): 38 | """Get a TreeBuilder class for various types of tree with built-in support 39 | 40 | treeType - the name of the tree type required (case-insensitive). Supported 41 | values are "simpletree", "dom", "etree" and "beautifulsoup" 42 | 43 | "simpletree" - a built-in DOM-ish tree type with support for some 44 | more pythonic idioms. 45 | "dom" - A generic builder for DOM implementations, defaulting to 46 | a xml.dom.minidom based implementation for the sake of 47 | backwards compatibility (as releases up until 0.10 had a 48 | builder called "dom" that was a minidom implemenation). 49 | "etree" - A generic builder for tree implementations exposing an 50 | elementtree-like interface (known to work with 51 | ElementTree, cElementTree and lxml.etree). 52 | "beautifulsoup" - Beautiful soup (if installed) 53 | 54 | implementation - (Currently applies to the "etree" and "dom" tree types). A 55 | module implementing the tree type e.g. 56 | xml.etree.ElementTree or lxml.etree.""" 57 | 58 | treeType = treeType.lower() 59 | if treeType not in treeBuilderCache: 60 | if treeType == "dom": 61 | import dom 62 | # XXX: Keep backwards compatibility by using minidom if no implementation is given 63 | if implementation == None: 64 | from xml.dom import minidom 65 | implementation = minidom 66 | # XXX: NEVER cache here, caching is done in the dom submodule 67 | return dom.getDomModule(implementation, **kwargs).TreeBuilder 68 | elif treeType == "simpletree": 69 | import simpletree 70 | treeBuilderCache[treeType] = simpletree.TreeBuilder 71 | elif treeType == "beautifulsoup": 72 | import soup 73 | treeBuilderCache[treeType] = soup.TreeBuilder 74 | elif treeType == "lxml": 75 | import etree_lxml 76 | treeBuilderCache[treeType] = etree_lxml.TreeBuilder 77 | elif treeType == "etree": 78 | # Come up with a sane default 79 | if implementation == None: 80 | try: 81 | import xml.etree.cElementTree as ET 82 | except ImportError: 83 | try: 84 | import xml.etree.ElementTree as ET 85 | except ImportError: 86 | try: 87 | import cElementTree as ET 88 | except ImportError: 89 | import elementtree.ElementTree as ET 90 | implementation = ET 91 | import etree 92 | # NEVER cache here, caching is done in the etree submodule 93 | return etree.getETreeModule(implementation, **kwargs).TreeBuilder 94 | else: 95 | raise ValueError("""Unrecognised treebuilder "%s" """%treeType) 96 | return treeBuilderCache.get(treeType) 97 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/__init__.py: -------------------------------------------------------------------------------- 1 | """A collection of modules for iterating through different kinds of 2 | tree, generating tokens identical to those produced by the tokenizer 3 | module. 4 | 5 | To create a tree walker for a new type of tree, you need to do 6 | implement a tree walker object (called TreeWalker by convention) that 7 | implements a 'serialize' method taking a tree as sole argument and 8 | returning an iterator generating tokens. 9 | """ 10 | 11 | treeWalkerCache = {} 12 | 13 | def getTreeWalker(treeType, implementation=None, **kwargs): 14 | """Get a TreeWalker class for various types of tree with built-in support 15 | 16 | treeType - the name of the tree type required (case-insensitive). Supported 17 | values are "simpletree", "dom", "etree" and "beautifulsoup" 18 | 19 | "simpletree" - a built-in DOM-ish tree type with support for some 20 | more pythonic idioms. 21 | "dom" - The xml.dom.minidom DOM implementation 22 | "pulldom" - The xml.dom.pulldom event stream 23 | "etree" - A generic walker for tree implementations exposing an 24 | elementtree-like interface (known to work with 25 | ElementTree, cElementTree and lxml.etree). 26 | "lxml" - Optimized walker for lxml.etree 27 | "beautifulsoup" - Beautiful soup (if installed) 28 | "genshi" - a Genshi stream 29 | 30 | implementation - (Currently applies to the "etree" tree type only). A module 31 | implementing the tree type e.g. xml.etree.ElementTree or 32 | cElementTree.""" 33 | 34 | treeType = treeType.lower() 35 | if treeType not in treeWalkerCache: 36 | if treeType in ("dom", "pulldom", "simpletree"): 37 | mod = __import__(treeType, globals()) 38 | treeWalkerCache[treeType] = mod.TreeWalker 39 | elif treeType == "genshi": 40 | import genshistream 41 | treeWalkerCache[treeType] = genshistream.TreeWalker 42 | elif treeType == "beautifulsoup": 43 | import soup 44 | treeWalkerCache[treeType] = soup.TreeWalker 45 | elif treeType == "lxml": 46 | import lxmletree 47 | treeWalkerCache[treeType] = lxmletree.TreeWalker 48 | elif treeType == "etree": 49 | import etree 50 | # XXX: NEVER cache here, caching is done in the etree submodule 51 | return etree.getETreeModule(implementation, **kwargs).TreeWalker 52 | return treeWalkerCache.get(treeType) 53 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/_base.py: -------------------------------------------------------------------------------- 1 | import gettext 2 | _ = gettext.gettext 3 | 4 | from html5lib.constants import voidElements, spaceCharacters 5 | spaceCharacters = u"".join(spaceCharacters) 6 | 7 | class TreeWalker(object): 8 | def __init__(self, tree): 9 | self.tree = tree 10 | 11 | def __iter__(self): 12 | raise NotImplementedError 13 | 14 | def error(self, msg): 15 | return {"type": "SerializeError", "data": msg} 16 | 17 | def normalizeAttrs(self, attrs): 18 | newattrs = {} 19 | if attrs: 20 | #TODO: treewalkers should always have attrs 21 | for (namespace,name),value in attrs.iteritems(): 22 | namespace = unicode(namespace) if namespace else None 23 | name = unicode(name) 24 | value = unicode(value) 25 | newattrs[(namespace,name)] = value 26 | return newattrs 27 | 28 | def emptyTag(self, namespace, name, attrs, hasChildren=False): 29 | yield {"type": "EmptyTag", "name": unicode(name), 30 | "namespace":unicode(namespace), 31 | "data": self.normalizeAttrs(attrs)} 32 | if hasChildren: 33 | yield self.error(_("Void element has children")) 34 | 35 | def startTag(self, namespace, name, attrs): 36 | return {"type": "StartTag", 37 | "name": unicode(name), 38 | "namespace":unicode(namespace), 39 | "data": self.normalizeAttrs(attrs)} 40 | 41 | def endTag(self, namespace, name): 42 | return {"type": "EndTag", 43 | "name": unicode(name), 44 | "namespace":unicode(namespace), 45 | "data": {}} 46 | 47 | def text(self, data): 48 | data = unicode(data) 49 | middle = data.lstrip(spaceCharacters) 50 | left = data[:len(data)-len(middle)] 51 | if left: 52 | yield {"type": "SpaceCharacters", "data": left} 53 | data = middle 54 | middle = data.rstrip(spaceCharacters) 55 | right = data[len(middle):] 56 | if middle: 57 | yield {"type": "Characters", "data": middle} 58 | if right: 59 | yield {"type": "SpaceCharacters", "data": right} 60 | 61 | def comment(self, data): 62 | return {"type": "Comment", "data": unicode(data)} 63 | 64 | def doctype(self, name, publicId=None, systemId=None, correct=True): 65 | return {"type": "Doctype", 66 | "name": name is not None and unicode(name) or u"", 67 | "publicId": publicId, 68 | "systemId": systemId, 69 | "correct": correct} 70 | 71 | def entity(self, name): 72 | return {"type": "Entity", "name": unicode(name)} 73 | 74 | def unknown(self, nodeType): 75 | return self.error(_("Unknown node type: ") + nodeType) 76 | 77 | class RecursiveTreeWalker(TreeWalker): 78 | def walkChildren(self, node): 79 | raise NodeImplementedError 80 | 81 | def element(self, node, namespace, name, attrs, hasChildren): 82 | if name in voidElements: 83 | for token in self.emptyTag(namespace, name, attrs, hasChildren): 84 | yield token 85 | else: 86 | yield self.startTag(name, attrs) 87 | if hasChildren: 88 | for token in self.walkChildren(node): 89 | yield token 90 | yield self.endTag(name) 91 | 92 | from xml.dom import Node 93 | 94 | DOCUMENT = Node.DOCUMENT_NODE 95 | DOCTYPE = Node.DOCUMENT_TYPE_NODE 96 | TEXT = Node.TEXT_NODE 97 | ELEMENT = Node.ELEMENT_NODE 98 | COMMENT = Node.COMMENT_NODE 99 | ENTITY = Node.ENTITY_NODE 100 | UNKNOWN = "<#UNKNOWN#>" 101 | 102 | class NonRecursiveTreeWalker(TreeWalker): 103 | def getNodeDetails(self, node): 104 | raise NotImplementedError 105 | 106 | def getFirstChild(self, node): 107 | raise NotImplementedError 108 | 109 | def getNextSibling(self, node): 110 | raise NotImplementedError 111 | 112 | def getParentNode(self, node): 113 | raise NotImplementedError 114 | 115 | def __iter__(self): 116 | currentNode = self.tree 117 | while currentNode is not None: 118 | details = self.getNodeDetails(currentNode) 119 | type, details = details[0], details[1:] 120 | hasChildren = False 121 | endTag = None 122 | 123 | if type == DOCTYPE: 124 | yield self.doctype(*details) 125 | 126 | elif type == TEXT: 127 | for token in self.text(*details): 128 | yield token 129 | 130 | elif type == ELEMENT: 131 | namespace, name, attributes, hasChildren = details 132 | if name in voidElements: 133 | for token in self.emptyTag(namespace, name, attributes, 134 | hasChildren): 135 | yield token 136 | hasChildren = False 137 | else: 138 | endTag = name 139 | yield self.startTag(namespace, name, attributes) 140 | 141 | elif type == COMMENT: 142 | yield self.comment(details[0]) 143 | 144 | elif type == ENTITY: 145 | yield self.entity(details[0]) 146 | 147 | elif type == DOCUMENT: 148 | hasChildren = True 149 | 150 | else: 151 | yield self.unknown(details[0]) 152 | 153 | if hasChildren: 154 | firstChild = self.getFirstChild(currentNode) 155 | else: 156 | firstChild = None 157 | 158 | if firstChild is not None: 159 | currentNode = firstChild 160 | else: 161 | while currentNode is not None: 162 | details = self.getNodeDetails(currentNode) 163 | type, details = details[0], details[1:] 164 | if type == ELEMENT: 165 | namespace, name, attributes, hasChildren = details 166 | if name not in voidElements: 167 | yield self.endTag(namespace, name) 168 | if self.tree is currentNode: 169 | currentNode = None 170 | break 171 | nextSibling = self.getNextSibling(currentNode) 172 | if nextSibling is not None: 173 | currentNode = nextSibling 174 | break 175 | else: 176 | currentNode = self.getParentNode(currentNode) 177 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/dom.py: -------------------------------------------------------------------------------- 1 | from xml.dom import Node 2 | 3 | import gettext 4 | _ = gettext.gettext 5 | 6 | import _base 7 | from html5lib.constants import voidElements 8 | 9 | class TreeWalker(_base.NonRecursiveTreeWalker): 10 | def getNodeDetails(self, node): 11 | if node.nodeType == Node.DOCUMENT_TYPE_NODE: 12 | return _base.DOCTYPE, node.name, node.publicId, node.systemId 13 | 14 | elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): 15 | return _base.TEXT, node.nodeValue 16 | 17 | elif node.nodeType == Node.ELEMENT_NODE: 18 | attrs = {} 19 | for attr in node.attributes.keys(): 20 | attr = node.getAttributeNode(attr) 21 | attrs[(attr.namespaceURI,attr.localName)] = attr.value 22 | return (_base.ELEMENT, node.namespaceURI, node.nodeName, 23 | attrs, node.hasChildNodes()) 24 | 25 | elif node.nodeType == Node.COMMENT_NODE: 26 | return _base.COMMENT, node.nodeValue 27 | 28 | elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE): 29 | return (_base.DOCUMENT,) 30 | 31 | else: 32 | return _base.UNKNOWN, node.nodeType 33 | 34 | def getFirstChild(self, node): 35 | return node.firstChild 36 | 37 | def getNextSibling(self, node): 38 | return node.nextSibling 39 | 40 | def getParentNode(self, node): 41 | return node.parentNode 42 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/etree.py: -------------------------------------------------------------------------------- 1 | import gettext 2 | _ = gettext.gettext 3 | 4 | import new 5 | import copy 6 | import re 7 | 8 | import _base 9 | from html5lib.constants import voidElements 10 | 11 | tag_regexp = re.compile("{([^}]*)}(.*)") 12 | 13 | moduleCache = {} 14 | 15 | def getETreeModule(ElementTreeImplementation): 16 | name = "_" + ElementTreeImplementation.__name__+"builder" 17 | if name in moduleCache: 18 | return moduleCache[name] 19 | else: 20 | mod = new.module("_" + ElementTreeImplementation.__name__+"builder") 21 | objs = getETreeBuilder(ElementTreeImplementation) 22 | mod.__dict__.update(objs) 23 | moduleCache[name] = mod 24 | return mod 25 | 26 | def getETreeBuilder(ElementTreeImplementation): 27 | ElementTree = ElementTreeImplementation 28 | 29 | class TreeWalker(_base.NonRecursiveTreeWalker): 30 | """Given the particular ElementTree representation, this implementation, 31 | to avoid using recursion, returns "nodes" as tuples with the following 32 | content: 33 | 34 | 1. The current element 35 | 36 | 2. The index of the element relative to its parent 37 | 38 | 3. A stack of ancestor elements 39 | 40 | 4. A flag "text", "tail" or None to indicate if the current node is a 41 | text node; either the text or tail of the current element (1) 42 | """ 43 | def getNodeDetails(self, node): 44 | if isinstance(node, tuple): # It might be the root Element 45 | elt, key, parents, flag = node 46 | if flag in ("text", "tail"): 47 | return _base.TEXT, getattr(elt, flag) 48 | else: 49 | node = elt 50 | 51 | if not(hasattr(node, "tag")): 52 | node = node.getroot() 53 | 54 | if node.tag in ("", ""): 55 | return (_base.DOCUMENT,) 56 | 57 | elif node.tag == "": 58 | return (_base.DOCTYPE, node.text, 59 | node.get("publicId"), node.get("systemId")) 60 | 61 | elif type(node.tag) == type(ElementTree.Comment): 62 | return _base.COMMENT, node.text 63 | 64 | else: 65 | #This is assumed to be an ordinary element 66 | match = tag_regexp.match(node.tag) 67 | if match: 68 | namespace, tag = match.groups() 69 | else: 70 | namespace = None 71 | tag = node.tag 72 | attrs = {} 73 | for name, value in node.attrib.items(): 74 | match = tag_regexp.match(name) 75 | if match: 76 | attrs[(match.group(1),match.group(2))] = value 77 | else: 78 | attrs[(None,name)] = value 79 | return (_base.ELEMENT, namespace, tag, 80 | attrs, len(node) or node.text) 81 | 82 | def getFirstChild(self, node): 83 | if isinstance(node, tuple): 84 | element, key, parents, flag = node 85 | else: 86 | element, key, parents, flag = node, None, [], None 87 | 88 | if flag in ("text", "tail"): 89 | return None 90 | else: 91 | if element.text: 92 | return element, key, parents, "text" 93 | elif len(element): 94 | parents.append(element) 95 | return element[0], 0, parents, None 96 | else: 97 | return None 98 | 99 | def getNextSibling(self, node): 100 | if isinstance(node, tuple): 101 | element, key, parents, flag = node 102 | else: 103 | return None 104 | 105 | if flag == "text": 106 | if len(element): 107 | parents.append(element) 108 | return element[0], 0, parents, None 109 | else: 110 | return None 111 | else: 112 | if element.tail and flag != "tail": 113 | return element, key, parents, "tail" 114 | elif key < len(parents[-1]) - 1: 115 | return parents[-1][key+1], key+1, parents, None 116 | else: 117 | return None 118 | 119 | def getParentNode(self, node): 120 | if isinstance(node, tuple): 121 | element, key, parents, flag = node 122 | else: 123 | return None 124 | 125 | if flag == "text": 126 | if not parents: 127 | return element 128 | else: 129 | return element, key, parents, None 130 | else: 131 | parent = parents.pop() 132 | if not parents: 133 | return parent 134 | else: 135 | return parent, list(parents[-1]).index(parent), parents, None 136 | 137 | return locals() 138 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/genshistream.py: -------------------------------------------------------------------------------- 1 | from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT 2 | from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT 3 | from genshi.output import NamespaceFlattener 4 | 5 | import _base 6 | 7 | from html5lib.constants import voidElements 8 | 9 | class TreeWalker(_base.TreeWalker): 10 | def __iter__(self): 11 | depth = 0 12 | ignore_until = None 13 | previous = None 14 | for event in self.tree: 15 | if previous is not None: 16 | if previous[0] == START: 17 | depth += 1 18 | if ignore_until <= depth: 19 | ignore_until = None 20 | if ignore_until is None: 21 | for token in self.tokens(previous, event): 22 | yield token 23 | if token["type"] == "EmptyTag": 24 | ignore_until = depth 25 | if previous[0] == END: 26 | depth -= 1 27 | previous = event 28 | if previous is not None: 29 | if ignore_until is None or ignore_until <= depth: 30 | for token in self.tokens(previous, None): 31 | yield token 32 | elif ignore_until is not None: 33 | raise ValueError("Illformed DOM event stream: void element without END_ELEMENT") 34 | 35 | def tokens(self, event, next): 36 | kind, data, pos = event 37 | if kind == START: 38 | tag, attrib = data 39 | name = tag.localname 40 | namespace = tag.namespace 41 | if tag in voidElements: 42 | for token in self.emptyTag(namespace, name, list(attrib), 43 | not next or next[0] != END 44 | or next[1] != tag): 45 | yield token 46 | else: 47 | yield self.startTag(namespace, name, list(attrib)) 48 | 49 | elif kind == END: 50 | name = data.localname 51 | namespace = data.namespace 52 | if name not in voidElements: 53 | yield self.endTag(namespace, name) 54 | 55 | elif kind == COMMENT: 56 | yield self.comment(data) 57 | 58 | elif kind == TEXT: 59 | for token in self.text(data): 60 | yield token 61 | 62 | elif kind == DOCTYPE: 63 | yield self.doctype(*data) 64 | 65 | elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \ 66 | START_CDATA, END_CDATA, PI): 67 | pass 68 | 69 | else: 70 | yield self.unknown(kind) 71 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/lxmletree.py: -------------------------------------------------------------------------------- 1 | from lxml import etree 2 | from html5lib.treebuilders.etree import tag_regexp 3 | 4 | from gettext import gettext 5 | _ = gettext 6 | 7 | import _base 8 | 9 | from html5lib.constants import voidElements 10 | from html5lib import ihatexml 11 | 12 | class Root(object): 13 | def __init__(self, et): 14 | self.elementtree = et 15 | self.children = [] 16 | if et.docinfo.internalDTD: 17 | self.children.append(Doctype(self, et.docinfo.root_name, 18 | et.docinfo.public_id, 19 | et.docinfo.system_url)) 20 | root = et.getroot() 21 | node = root 22 | 23 | while node.getprevious() is not None: 24 | node = node.getprevious() 25 | while node is not None: 26 | self.children.append(node) 27 | node = node.getnext() 28 | 29 | self.text = None 30 | self.tail = None 31 | 32 | def __getitem__(self, key): 33 | return self.children[key] 34 | 35 | def getnext(self): 36 | return None 37 | 38 | def __len__(self): 39 | return 1 40 | 41 | class Doctype(object): 42 | def __init__(self, root_node, name, public_id, system_id): 43 | self.root_node = root_node 44 | self.name = name 45 | self.public_id = public_id 46 | self.system_id = system_id 47 | 48 | self.text = None 49 | self.tail = None 50 | 51 | def getnext(self): 52 | return self.root_node.children[1] 53 | 54 | class FragmentRoot(Root): 55 | def __init__(self, children): 56 | self.children = [FragmentWrapper(self, child) for child in children] 57 | self.text = self.tail = None 58 | 59 | def getnext(self): 60 | return None 61 | 62 | class FragmentWrapper(object): 63 | def __init__(self, fragment_root, obj): 64 | self.root_node = fragment_root 65 | self.obj = obj 66 | if hasattr(self.obj, 'text'): 67 | self.text = self.obj.text 68 | else: 69 | self.text = None 70 | if hasattr(self.obj, 'tail'): 71 | self.tail = self.obj.tail 72 | else: 73 | self.tail = None 74 | self.isstring = isinstance(obj, basestring) 75 | 76 | def __getattr__(self, name): 77 | return getattr(self.obj, name) 78 | 79 | def getnext(self): 80 | siblings = self.root_node.children 81 | idx = siblings.index(self) 82 | if idx < len(siblings) - 1: 83 | return siblings[idx + 1] 84 | else: 85 | return None 86 | 87 | def __getitem__(self, key): 88 | return self.obj[key] 89 | 90 | def __nonzero__(self): 91 | return bool(self.obj) 92 | 93 | def getparent(self): 94 | return None 95 | 96 | def __str__(self): 97 | return str(self.obj) 98 | 99 | def __unicode__(self): 100 | return unicode(self.obj) 101 | 102 | def __len__(self): 103 | return len(self.obj) 104 | 105 | 106 | class TreeWalker(_base.NonRecursiveTreeWalker): 107 | def __init__(self, tree): 108 | if hasattr(tree, "getroot"): 109 | tree = Root(tree) 110 | elif isinstance(tree, list): 111 | tree = FragmentRoot(tree) 112 | _base.NonRecursiveTreeWalker.__init__(self, tree) 113 | self.filter = ihatexml.InfosetFilter() 114 | def getNodeDetails(self, node): 115 | if isinstance(node, tuple): # Text node 116 | node, key = node 117 | assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key 118 | return _base.TEXT, getattr(node, key) 119 | 120 | elif isinstance(node, Root): 121 | return (_base.DOCUMENT,) 122 | 123 | elif isinstance(node, Doctype): 124 | return _base.DOCTYPE, node.name, node.public_id, node.system_id 125 | 126 | elif isinstance(node, FragmentWrapper) and node.isstring: 127 | return _base.TEXT, node 128 | 129 | elif node.tag == etree.Comment: 130 | return _base.COMMENT, node.text 131 | 132 | elif node.tag == etree.Entity: 133 | return _base.ENTITY, node.text[1:-1] # strip &; 134 | 135 | else: 136 | #This is assumed to be an ordinary element 137 | match = tag_regexp.match(node.tag) 138 | if match: 139 | namespace, tag = match.groups() 140 | else: 141 | namespace = None 142 | tag = node.tag 143 | attrs = {} 144 | for name, value in node.attrib.items(): 145 | match = tag_regexp.match(name) 146 | if match: 147 | attrs[(match.group(1),match.group(2))] = value 148 | else: 149 | attrs[(None,name)] = value 150 | return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), 151 | attrs, len(node) > 0 or node.text) 152 | 153 | def getFirstChild(self, node): 154 | assert not isinstance(node, tuple), _("Text nodes have no children") 155 | 156 | assert len(node) or node.text, "Node has no children" 157 | if node.text: 158 | return (node, "text") 159 | else: 160 | return node[0] 161 | 162 | def getNextSibling(self, node): 163 | if isinstance(node, tuple): # Text node 164 | node, key = node 165 | assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key 166 | if key == "text": 167 | # XXX: we cannot use a "bool(node) and node[0] or None" construct here 168 | # because node[0] might evaluate to False if it has no child element 169 | if len(node): 170 | return node[0] 171 | else: 172 | return None 173 | else: # tail 174 | return node.getnext() 175 | 176 | return node.tail and (node, "tail") or node.getnext() 177 | 178 | def getParentNode(self, node): 179 | if isinstance(node, tuple): # Text node 180 | node, key = node 181 | assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key 182 | if key == "text": 183 | return node 184 | # else: fallback to "normal" processing 185 | 186 | return node.getparent() 187 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/pulldom.py: -------------------------------------------------------------------------------- 1 | from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \ 2 | COMMENT, IGNORABLE_WHITESPACE, CHARACTERS 3 | 4 | import _base 5 | 6 | from html5lib.constants import voidElements 7 | 8 | class TreeWalker(_base.TreeWalker): 9 | def __iter__(self): 10 | ignore_until = None 11 | previous = None 12 | for event in self.tree: 13 | if previous is not None and \ 14 | (ignore_until is None or previous[1] is ignore_until): 15 | if previous[1] is ignore_until: 16 | ignore_until = None 17 | for token in self.tokens(previous, event): 18 | yield token 19 | if token["type"] == "EmptyTag": 20 | ignore_until = previous[1] 21 | previous = event 22 | if ignore_until is None or previous[1] is ignore_until: 23 | for token in self.tokens(previous, None): 24 | yield token 25 | elif ignore_until is not None: 26 | raise ValueError("Illformed DOM event stream: void element without END_ELEMENT") 27 | 28 | def tokens(self, event, next): 29 | type, node = event 30 | if type == START_ELEMENT: 31 | name = node.nodeName 32 | namespace = node.namespaceURI 33 | attrs = {} 34 | for attr in node.attributes.keys(): 35 | attr = node.getAttributeNode(attr) 36 | attrs[(attr.namespaceURI,attr.localName)] = attr.value 37 | if name in voidElements: 38 | for token in self.emptyTag(namespace, 39 | name, 40 | attrs, 41 | not next or next[1] is not node): 42 | yield token 43 | else: 44 | yield self.startTag(namespace, name, attrs) 45 | 46 | elif type == END_ELEMENT: 47 | name = node.nodeName 48 | namespace = node.namespaceURI 49 | if name not in voidElements: 50 | yield self.endTag(namespace, name) 51 | 52 | elif type == COMMENT: 53 | yield self.comment(node.nodeValue) 54 | 55 | elif type in (IGNORABLE_WHITESPACE, CHARACTERS): 56 | for token in self.text(node.nodeValue): 57 | yield token 58 | 59 | else: 60 | yield self.unknown(type) 61 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/simpletree.py: -------------------------------------------------------------------------------- 1 | import gettext 2 | _ = gettext.gettext 3 | 4 | import _base 5 | 6 | class TreeWalker(_base.NonRecursiveTreeWalker): 7 | """Given that simpletree has no performant way of getting a node's 8 | next sibling, this implementation returns "nodes" as tuples with the 9 | following content: 10 | 11 | 1. The parent Node (Element, Document or DocumentFragment) 12 | 13 | 2. The child index of the current node in its parent's children list 14 | 15 | 3. A list used as a stack of all ancestors. It is a pair tuple whose 16 | first item is a parent Node and second item is a child index. 17 | """ 18 | 19 | def getNodeDetails(self, node): 20 | if isinstance(node, tuple): # It might be the root Node 21 | parent, idx, parents = node 22 | node = parent.childNodes[idx] 23 | 24 | # testing node.type allows us not to import treebuilders.simpletree 25 | if node.type in (1, 2): # Document or DocumentFragment 26 | return (_base.DOCUMENT,) 27 | 28 | elif node.type == 3: # DocumentType 29 | return _base.DOCTYPE, node.name, node.publicId, node.systemId 30 | 31 | elif node.type == 4: # TextNode 32 | return _base.TEXT, node.value 33 | 34 | elif node.type == 5: # Element 35 | attrs = {} 36 | for name, value in node.attributes.items(): 37 | if isinstance(name, tuple): 38 | attrs[(name[2],name[1])] = value 39 | else: 40 | attrs[(None,name)] = value 41 | return (_base.ELEMENT, node.namespace, node.name, 42 | attrs, node.hasContent()) 43 | 44 | elif node.type == 6: # CommentNode 45 | return _base.COMMENT, node.data 46 | 47 | else: 48 | return _node.UNKNOWN, node.type 49 | 50 | def getFirstChild(self, node): 51 | if isinstance(node, tuple): # It might be the root Node 52 | parent, idx, parents = node 53 | parents.append((parent, idx)) 54 | node = parent.childNodes[idx] 55 | else: 56 | parents = [] 57 | 58 | assert node.hasContent(), "Node has no children" 59 | return (node, 0, parents) 60 | 61 | def getNextSibling(self, node): 62 | assert isinstance(node, tuple), "Node is not a tuple: " + str(node) 63 | parent, idx, parents = node 64 | idx += 1 65 | if len(parent.childNodes) > idx: 66 | return (parent, idx, parents) 67 | else: 68 | return None 69 | 70 | def getParentNode(self, node): 71 | assert isinstance(node, tuple) 72 | parent, idx, parents = node 73 | if parents: 74 | parent, idx = parents.pop() 75 | return parent, idx, parents 76 | else: 77 | # HACK: We could return ``parent`` but None will stop the algorithm the same way 78 | return None 79 | -------------------------------------------------------------------------------- /lib/html5lib/treewalkers/soup.py: -------------------------------------------------------------------------------- 1 | import re 2 | import gettext 3 | _ = gettext.gettext 4 | 5 | from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag 6 | from html5lib.constants import namespaces 7 | import _base 8 | 9 | class TreeWalker(_base.NonRecursiveTreeWalker): 10 | doctype_regexp = re.compile( 11 | r'DOCTYPE\s+(?P[^\s]*)(\s*PUBLIC\s*"(?P.*)"\s*"(?P.*)"|\s*SYSTEM\s*"(?P.*)")?') 12 | def getNodeDetails(self, node): 13 | if isinstance(node, BeautifulSoup): # Document or DocumentFragment 14 | return (_base.DOCUMENT,) 15 | 16 | elif isinstance(node, Declaration): # DocumentType 17 | string = unicode(node.string) 18 | #Slice needed to remove markup added during unicode conversion, 19 | #but only in some versions of BeautifulSoup/Python 20 | if string.startswith(''): 21 | string = string[2:-1] 22 | m = self.doctype_regexp.match(string) 23 | #This regexp approach seems wrong and fragile 24 | #but beautiful soup stores the doctype as a single thing and we want the seperate bits 25 | #It should work as long as the tree is created by html5lib itself but may be wrong if it's 26 | #been modified at all 27 | #We could just feed to it a html5lib tokenizer, I guess... 28 | assert m is not None, "DOCTYPE did not match expected format" 29 | 30 | name = m.group('name') 31 | publicId = m.group('publicId') 32 | if publicId is not None: 33 | systemId = m.group('systemId1') 34 | else: 35 | systemId = m.group('systemId2') 36 | return _base.DOCTYPE, name, publicId or "", systemId or "" 37 | 38 | elif isinstance(node, Comment): 39 | string = unicode(node.string) 40 | if string.startswith(''): 41 | string = string[4:-3] 42 | return _base.COMMENT, string 43 | 44 | elif isinstance(node, unicode): # TextNode 45 | return _base.TEXT, node 46 | 47 | elif isinstance(node, Tag): # Element 48 | return (_base.ELEMENT, namespaces["html"], node.name, 49 | dict(node.attrs).items(), node.contents) 50 | else: 51 | return _base.UNKNOWN, node.__class__.__name__ 52 | 53 | def getFirstChild(self, node): 54 | return node.contents[0] 55 | 56 | def getNextSibling(self, node): 57 | return node.nextSibling 58 | 59 | def getParentNode(self, node): 60 | return node.parent 61 | -------------------------------------------------------------------------------- /lib/html5lib/utils.py: -------------------------------------------------------------------------------- 1 | try: 2 | frozenset 3 | except NameError: 4 | #Import from the sets module for python 2.3 5 | from sets import Set as set 6 | from sets import ImmutableSet as frozenset 7 | 8 | class MethodDispatcher(dict): 9 | """Dict with 2 special properties: 10 | 11 | On initiation, keys that are lists, sets or tuples are converted to 12 | multiple keys so accessing any one of the items in the original 13 | list-like object returns the matching value 14 | 15 | md = MethodDispatcher({("foo", "bar"):"baz"}) 16 | md["foo"] == "baz" 17 | 18 | A default value which can be set through the default attribute. 19 | """ 20 | 21 | def __init__(self, items=()): 22 | # Using _dictEntries instead of directly assigning to self is about 23 | # twice as fast. Please do careful performance testing before changing 24 | # anything here. 25 | _dictEntries = [] 26 | for name,value in items: 27 | if type(name) in (list, tuple, frozenset, set): 28 | for item in name: 29 | _dictEntries.append((item, value)) 30 | else: 31 | _dictEntries.append((name, value)) 32 | dict.__init__(self, _dictEntries) 33 | self.default = None 34 | 35 | def __getitem__(self, key): 36 | return dict.get(self, key, self.default) 37 | 38 | #Pure python implementation of deque taken from the ASPN Python Cookbook 39 | #Original code by Raymond Hettinger 40 | 41 | class deque(object): 42 | 43 | def __init__(self, iterable=(), maxsize=-1): 44 | if not hasattr(self, 'data'): 45 | self.left = self.right = 0 46 | self.data = {} 47 | self.maxsize = maxsize 48 | self.extend(iterable) 49 | 50 | def append(self, x): 51 | self.data[self.right] = x 52 | self.right += 1 53 | if self.maxsize != -1 and len(self) > self.maxsize: 54 | self.popleft() 55 | 56 | def appendleft(self, x): 57 | self.left -= 1 58 | self.data[self.left] = x 59 | if self.maxsize != -1 and len(self) > self.maxsize: 60 | self.pop() 61 | 62 | def pop(self): 63 | if self.left == self.right: 64 | raise IndexError('cannot pop from empty deque') 65 | self.right -= 1 66 | elem = self.data[self.right] 67 | del self.data[self.right] 68 | return elem 69 | 70 | def popleft(self): 71 | if self.left == self.right: 72 | raise IndexError('cannot pop from empty deque') 73 | elem = self.data[self.left] 74 | del self.data[self.left] 75 | self.left += 1 76 | return elem 77 | 78 | def clear(self): 79 | self.data.clear() 80 | self.left = self.right = 0 81 | 82 | def extend(self, iterable): 83 | for elem in iterable: 84 | self.append(elem) 85 | 86 | def extendleft(self, iterable): 87 | for elem in iterable: 88 | self.appendleft(elem) 89 | 90 | def rotate(self, n=1): 91 | if self: 92 | n %= len(self) 93 | for i in xrange(n): 94 | self.appendleft(self.pop()) 95 | 96 | def __getitem__(self, i): 97 | if i < 0: 98 | i += len(self) 99 | try: 100 | return self.data[i + self.left] 101 | except KeyError: 102 | raise IndexError 103 | 104 | def __setitem__(self, i, value): 105 | if i < 0: 106 | i += len(self) 107 | try: 108 | self.data[i + self.left] = value 109 | except KeyError: 110 | raise IndexError 111 | 112 | def __delitem__(self, i): 113 | size = len(self) 114 | if not (-size <= i < size): 115 | raise IndexError 116 | data = self.data 117 | if i < 0: 118 | i += size 119 | for j in xrange(self.left+i, self.right-1): 120 | data[j] = data[j+1] 121 | self.pop() 122 | 123 | def __len__(self): 124 | return self.right - self.left 125 | 126 | def __cmp__(self, other): 127 | if type(self) != type(other): 128 | return cmp(type(self), type(other)) 129 | return cmp(list(self), list(other)) 130 | 131 | def __repr__(self, _track=[]): 132 | if id(self) in _track: 133 | return '...' 134 | _track.append(id(self)) 135 | r = 'deque(%r)' % (list(self),) 136 | _track.remove(id(self)) 137 | return r 138 | 139 | def __getstate__(self): 140 | return (tuple(self),) 141 | 142 | def __setstate__(self, s): 143 | self.__init__(s[0]) 144 | 145 | def __hash__(self): 146 | raise TypeError 147 | 148 | def __copy__(self): 149 | return self.__class__(self) 150 | 151 | def __deepcopy__(self, memo={}): 152 | from copy import deepcopy 153 | result = self.__class__() 154 | memo[id(self)] = result 155 | result.__init__(deepcopy(tuple(self), memo)) 156 | return result 157 | 158 | #Some utility functions to dal with weirdness around UCS2 vs UCS4 159 | #python builds 160 | 161 | def encodingType(): 162 | if len() == 2: 163 | return "UCS2" 164 | else: 165 | return "UCS4" 166 | 167 | def isSurrogatePair(data): 168 | return (len(data) == 2 and 169 | ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and 170 | ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF) 171 | 172 | def surrogatePairToCodepoint(data): 173 | char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + 174 | (ord(data[1]) - 0xDC00)) 175 | return char_val 176 | -------------------------------------------------------------------------------- /lib/rdfextras/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | """ 3 | The RDFExtras namespace package. 4 | """ 5 | 6 | __author__ = "Niklas Lindström" 7 | __version__ = "0.1" 8 | 9 | # This is a namespace package. 10 | try: 11 | import pkg_resources 12 | pkg_resources.declare_namespace(__name__) 13 | except ImportError: 14 | import pkgutil 15 | __path__ = pkgutil.extend_path(__path__, __name__) -------------------------------------------------------------------------------- /lib/rdfextras/sparql/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # 4 | # $Date: 2005/11/04 14:06:36 $, by $Author: ivan $, $Revision: 1.1 $ 5 | # The documentation of the module, hence the convention for the documentation of methods and classes, 6 | # is based on the epydoc tool. This tool parses Python source files 7 | # and generates API descriptions XHTML. 8 | # The latest release of epydoc (version 2.0) can be 9 | # downloaded from the SourceForge 10 | # download page. 11 | # 12 | # 13 | """ 14 | TODO: merge this first bit from sparql.sparql.py into rest of doc... 15 | updating all along the way. 16 | 17 | SPARQL implementation on top of RDFLib 18 | 19 | Implementation of the `W3C SPARQL `_ 20 | language (version April 2005). The basic class here is supposed to be a 21 | superclass of rdflib.sparql.graph; it has been separated only for 22 | a better maintainability. 23 | 24 | There is a separate 25 | `description `_ 26 | for the functionalities. 27 | 28 | 29 | For a general description of the SPARQL API, see the separate, more complete 30 | `description `_. 31 | 32 | Variables, Imports 33 | ================== 34 | 35 | The top level (__init__.py) module of the Package imports the 36 | important classes. In other words, the user may choose to use the 37 | following imports only:: 38 | 39 | from rdflibUtils import myTripleStore 40 | from rdflibUtils import retrieveRDFFiles 41 | from rdflibUtils import SPARQLError 42 | from rdflibUtils import GraphPattern 43 | 44 | The module imports and/or creates some frequently used Namespaces, and 45 | these can then be imported by the user like:: 46 | 47 | from rdflibUtils import ns_rdf 48 | 49 | Finally, the package also has a set of convenience string defines for 50 | XML Schema datatypes (ie, the URI-s of the datatypes); ie, one can 51 | use:: 52 | 53 | from rdflibUtils import type_string 54 | from rdflibUtils import type_integer 55 | from rdflibUtils import type_long 56 | from rdflibUtils import type_double 57 | from rdflibUtils import type_float 58 | from rdflibUtils import type_decimal 59 | from rdflibUtils import type_dateTime 60 | from rdflibUtils import type_date 61 | from rdflibUtils import type_time 62 | from rdflibUtils import type_duration 63 | 64 | These are used, for example, in the sparql-p implementation. 65 | 66 | The three most important classes in RDFLib for the average user are 67 | Namespace, URIRef and Literal; these are also imported, so the user 68 | can also use, eg:: 69 | 70 | from rdflibUtils import Namespace, URIRef, Literal 71 | 72 | History 73 | ======= 74 | 75 | - Version 1.0: based on an earlier version of the SPARQL, first released implementation 76 | 77 | - Version 2.0: version based on the March 2005 SPARQL document, 78 | also a major change of the core code (introduction of the separate 79 | ``GraphPattern`` :class:`rdflibUtils.graph.GraphPattern` class, etc). 80 | 81 | - Version 2.01: minor changes only: - switch to epydoc as a documentation tool, 82 | it gives a much better overview of the classes - addition of the 83 | SELECT * feature to sparql-p 84 | 85 | - Version 2.02: - added some methods to 86 | ``myTripleStore`` :class:`rdflibUtils.myTripleStore.myTripleStore` to handle 87 | ``Alt`` and ``Bag`` the same way as ``Seq`` - added also methods to 88 | :meth:`add` collections and containers to the triple store, not only 89 | retrieve them 90 | 91 | - Version 2.1: adapted to the inclusion of the code into rdflib, thanks to Michel Pelletier 92 | 93 | - Version 2.2: added the sorting possibilities; introduced the Unbound class and have a better 94 | interface to patterns using this (in the BasicGraphPattern class) 95 | 96 | @author: `Ivan Herman `_ 97 | 98 | @license: This software is available for use under the 99 | `W3C Software License `_ 100 | 101 | @contact: Ivan Herman, ivan@ivan-herman.net 102 | 103 | @version: 2.2 104 | 105 | """ 106 | from rdflib import URIRef 107 | 108 | DESCRIBE=URIRef('http://www.w3.org/TR/rdf-sparql-query/#describe') 109 | 110 | __version__ = "2.2" 111 | 112 | Debug = False 113 | 114 | # Note that the SPARQL draft allows the usage of a different query character, but I decided not to be that 115 | # generous, and keep to one only. I put it into a separate variable to avoid problems if the group decides 116 | # to change the syntax on that detail... 117 | _questChar = "?" 118 | 119 | 120 | def generateCollectionConstraint(triplets,collection,item) : 121 | """ 122 | 123 | Generate a function that can then be used as a global constaint in 124 | sparql to check whether the 'item' is an element of the 125 | 'collection' (a.k.a. list). Both collection and item can be a real 126 | resource or a query string. Furthermore, item might be a plain 127 | string, that is then turned into a literal run-time. 128 | 129 | The function returns an adapted filter method that can then be 130 | plugged into a sparql request. 131 | 132 | @param triplets: the 133 | L{SPARQLGraph} instance 134 | 135 | @param collection: is either a query string (that has to be bound 136 | by the query) or an RDFLib Resource representing the collection 137 | 138 | @param item: is either a query string (that has to be bound by the 139 | query) or an RDFLib Resource that must be tested to be part of the 140 | collection 141 | 142 | @rtype: a function suitable as a sparql filter 143 | 144 | @raises SPARQLError: if the collection or the 145 | item parameters are illegal (not valid resources for a collection 146 | or an object) 147 | """ 148 | return isOnCollection(collection,item, triples) 149 | 150 | ############################################################################################ 151 | 152 | class Processor(object): 153 | 154 | def __init__(self, graph): 155 | pass 156 | 157 | def query(self, strOrQuery, initBindings={}, initNs={}, DEBUG=False): 158 | pass 159 | 160 | from rdflib.exceptions import Error 161 | 162 | ## 163 | # SPARQL Error Exception (subclass of the RDFLib Exceptions) 164 | class SPARQLError(Error) : 165 | """Am SPARQL error has been detected""" 166 | def __init__(self,msg): 167 | Error.__init__(self, ("SPARQL Error: %s." % msg)) 168 | -------------------------------------------------------------------------------- /lib/rdfextras/sparql/processor.py: -------------------------------------------------------------------------------- 1 | import sparql 2 | import sparql.parser 3 | from sparql.algebra import TopEvaluate 4 | from rdflib.namespace import RDFS, RDF, OWL 5 | from sparql.components import Query, Prolog 6 | 7 | class Processor(sparql.Processor): 8 | 9 | def __init__(self, graph): 10 | self.graph = graph 11 | 12 | def query(self, 13 | strOrQuery, 14 | initBindings={}, 15 | initNs={}, 16 | DEBUG=False, 17 | PARSE_DEBUG=False, 18 | dataSetBase=None, 19 | extensionFunctions={}, 20 | USE_PYPARSING=False): 21 | 22 | initNs.update({u'rdfs':RDFS.uri, u'owl':OWL.uri, u'rdf':RDF.uri}) 23 | 24 | assert isinstance(strOrQuery, (basestring, Query)),"%s must be a string or an rdfextras.sparql.components.Query instance"%strOrQuery 25 | if isinstance(strOrQuery, basestring): 26 | strOrQuery = sparql.parser.parse(strOrQuery) 27 | if not strOrQuery.prolog: 28 | strOrQuery.prolog = Prolog(None, []) 29 | strOrQuery.prolog.prefixBindings.update(initNs) 30 | else: 31 | for prefix, nsInst in initNs.items(): 32 | if prefix not in strOrQuery.prolog.prefixBindings: 33 | strOrQuery.prolog.prefixBindings[prefix] = nsInst 34 | 35 | global prolog 36 | prolog = strOrQuery.prolog 37 | 38 | return TopEvaluate(strOrQuery, 39 | self.graph, 40 | initBindings, 41 | DEBUG=DEBUG, 42 | dataSetBase=dataSetBase, 43 | extensionFunctions=extensionFunctions) 44 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/CSVWriter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class CSVWriter(object): 5 | 6 | def __init__(self): 7 | self.f = None 8 | self.Columns = None 9 | 10 | def Open(self,columns, filename): 11 | self.Columns = columns 12 | self.f = open(filename,'a') 13 | # header 14 | if self.Columns != None: 15 | for k in self.Columns: 16 | self.f.write(k + '\t') 17 | self.f.write('\n') 18 | self.f.flush() 19 | 20 | def WriteLine(self,line): 21 | self.f.write(line + '\n') 22 | self.f.flush() 23 | 24 | def WriteListEntry(self,list): 25 | for k in list: 26 | if k != None: 27 | val = str(k).replace('\n',' ').replace('\t', ' ') 28 | else: 29 | val = '' 30 | self.f.write(val + '\t') 31 | self.f.write('\n') 32 | self.f.flush() 33 | 34 | def Write(self,list): 35 | # write data rows 36 | for d in list: 37 | self.WriteEntry(d) 38 | 39 | def WriteEntry(self,d): 40 | for k in self.Columns: 41 | if d.has_key(k): 42 | val = str(d[k]).replace('\n',' ').replace('\t', ' ') 43 | else: 44 | val = '' 45 | self.f.write(val + '\t') 46 | self.f.write('\n') 47 | self.f.flush() 48 | 49 | 50 | def Close(self): 51 | self.f.close() 52 | self.f = None 53 | 54 | def WriteAllResults(filename, columns, list): 55 | w = CSVWriter() 56 | w.Open(columns, filename) 57 | w.Write(list) 58 | w.Close() 59 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/EARLPlugin.py: -------------------------------------------------------------------------------- 1 | """ A Nose Plugin for EARL. 2 | 3 | See Also: 4 | http://nose.python-hosting.com/ 5 | http://www.w3.org/TR/EARL10-Schema/ 6 | 7 | """ 8 | 9 | import logging 10 | 11 | from nose.plugins import Plugin 12 | 13 | from rdflib.term import URIRef, BNode, Literal 14 | from rdflib.namespace import RDF, RDFS 15 | from rdflib.graph import Graph 16 | from rdflib.namespace import NamespaceDict as Namespace 17 | from rdflib.util import date_time 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | EARL = Namespace("http://www.w3.org/ns/earl#") 22 | 23 | 24 | class EARLPlugin(Plugin): 25 | """ 26 | Activate the EARL plugin to generate a report of the test results 27 | using EARL. 28 | """ 29 | name = 'EARL' 30 | 31 | def begin(self): 32 | self.graph = Graph() 33 | self.graph.bind("earl", EARL.uri) 34 | 35 | def finalize(self, result): 36 | # TODO: add plugin options for specifying where to send 37 | # output. 38 | self.graph.serialize("file:results-%s.rdf" % date_time(), format="pretty-xml") 39 | 40 | def addDeprecated(self, test): 41 | print "Deprecated: %s" % test 42 | 43 | def addError(self, test, err, capt): 44 | print "Error: %s" % test 45 | 46 | def addFailure(self, test, err, capt, tb_info): 47 | print "Failure: %s" % test 48 | 49 | def addSkip(self, test): 50 | print "Skip: %s" % test 51 | 52 | def addSuccess(self, test, capt): 53 | result = BNode() # TODO: coin URIRef 54 | self.graph.add((result, RDFS.label, Literal(test))) 55 | self.graph.add((result, RDFS.comment, Literal(type(test)))) 56 | self.graph.add((result, RDF.type, EARL.TestResult)) 57 | self.graph.add((result, EARL.outcome, EARL["pass"])) 58 | # etc 59 | 60 | """ 61 | 62 | 63 | Invalid Markup (code #353) 64 | 65 | 66 | The table element is not allowed to appear 67 | inside a p element 68 | 69 | 70 | 2006-08-13 71 | 72 | 73 | 74 | It seems the p element has not been closed 75 | 76 | 77 | 78 | """ 79 | 80 | 81 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/FixTypeViews.py: -------------------------------------------------------------------------------- 1 | import rdflib 2 | from rdflib.Graph import ConjunctiveGraph 3 | from rdflib import plugin 4 | from rdflib.store import Store, NO_STORE, VALID_STORE 5 | 6 | try: 7 | import MySQLdb 8 | except ImportError: 9 | import warnings 10 | warnings.warn("MySQLdb is not installed") 11 | __test__=False 12 | 13 | def FixViewsString(configStr, storeName='rdfstore'): 14 | # Get the mysql plugin. You may have to install the python mysql libraries 15 | store = plugin.get('MySQL', Store)(storeName,debug=False,perfLog=True) 16 | 17 | # Open previously created store, or create it if it doesn't exist yet 18 | rt = store.open(configStr,create=False) 19 | if rt == NO_STORE: 20 | #There is no underlying MySQL infrastructure, create it 21 | #store.open(configStr,create=True) 22 | 23 | #TODO: could create store & load appropriate data here 24 | assert False, "'%s' store '%s' not found using config string '%s!'" % (storeType, storeName, configStr) 25 | else: 26 | assert rt == VALID_STORE,"There underlying store is corrupted" 27 | 28 | #There is a store, use it; use ConjunctiveGraph to see everything! 29 | graph = ConjunctiveGraph(store, identifier = URIRef(graphUri)) 30 | 31 | FixViewsGraph(graph) 32 | 33 | def FixViewsGraph(graph): 34 | FixViews(graph.store._db.cursor(), graph.store._internedId) 35 | 36 | def FixViews(cursor, storeInteredId): 37 | print "Fixing view %s_all."%(storeInteredId) 38 | 39 | cursor.execute("""ALTER view %(store)s_all as select subject as subject, subject_term 40 | as subject_term, predicate as predicate, predicate_term as 41 | predicate_term, object as object, 'L' as object_term, context as 42 | context, context_term as context_term, data_type, language from 43 | %(store)s_literalProperties 44 | union all 45 | select subject as subject, subject_term as subject_term, predicate as 46 | predicate, predicate_term as predicate_term, object as object, 47 | object_term as object_term, context as context, context_term as 48 | context_term, NULL as data_type, NULL as language from 49 | %(store)s_relations 50 | union all 51 | select member as subject, member_term as subject_term, 52 | CONVERT('3732454415692939113', UNSIGNED INTEGER) as predicate, 'U' as predicate_term, class as 53 | object, class_term as object_term, context as context, context_term as 54 | context_term, NULL as data_type, NULL as language from 55 | %(store)s_associativeBox"""%dict(store=storeInteredId)) 56 | 57 | print "Fixing view %s_relation_or_associativeBox."%(storeInteredId) 58 | 59 | cursor.execute("""ALTER view %(store)s_relation_or_associativeBox as select * from 60 | %(store)s_relations 61 | union all 62 | select member as subject, member_term as subject_term, 63 | CONVERT('3732454415692939113', UNSIGNED INTEGER) as predicate, 'U' as predicate_term, class as 64 | object, class_term as object_term, context as context, context_term as 65 | context_term from %(store)s_associativeBox"""%dict(store=storeInteredId)) 66 | 67 | cursor.close() 68 | 69 | print "Done." 70 | 71 | 72 | if __name__ == '__main__': 73 | from optparse import OptionParser 74 | usage = '''usage: %prog [options] configStr storeName''' 75 | op = OptionParser(usage=usage) 76 | #op.add_option('--buildOWL',action="store_true",default=True, 77 | # help = 'Build OWL from components') 78 | #op.add_option('--sql',action='store_true',default=False, dest='sql', help='Generate SQL from SPARQL.') 79 | (options, args) = op.parse_args() 80 | FixViewsString(args[0], args[1], options) 81 | 82 | 83 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/QueryStats.py: -------------------------------------------------------------------------------- 1 | import rdflib 2 | from rdflib.sparql.bison import Parse 3 | from rdflib.sparql.bison.Query import * 4 | from rdflib.sparql.bison.GraphPattern import * 5 | from rdflib.sparql.bison.Operators import * 6 | from rdflib.sparql.bison.FunctionLibrary import * 7 | from rdflib.sparql.bison.Util import ListRedirect 8 | from rdflib.sparql.bison.Expression import * 9 | from rdflib.Variable import Variable 10 | from rdflib.Literal import Literal 11 | 12 | def QueryStats(queryString, log): 13 | 14 | log['queryString'] = queryString 15 | 16 | # use the SPARQL parser 17 | try: 18 | q = Parse(queryString) 19 | log['ParseError'] = 0 20 | except Exception, e: 21 | print 'PARSE ERROR: %s' % e 22 | q = None 23 | log['ParseError'] = 1 24 | return False 25 | 26 | GetQueryInfo(q, log) 27 | return True 28 | 29 | def GetQueryInfo(q,log): 30 | if isinstance(q.query, SelectQuery): 31 | log['queryType'] = 'Select' 32 | elif isinstance(q.query, AskQuery): 33 | log['queryType'] = 'Ask' 34 | elif isinstance(q.query, ConstructQuery): 35 | log['queryType'] = 'Construct' 36 | elif isinstance(q.query, DescribeQuery): 37 | log['queryType'] = 'Describe' 38 | 39 | log['distinct'] = 0 40 | if q.query.distinct: 41 | log['distinct'] = 1 42 | 43 | log['limit'] = 0 44 | log['offset'] = 0 45 | log['order'] = 0 46 | 47 | vars = dict() 48 | 49 | if q.query.solutionModifier.limitClause: 50 | log['limit'] = 1 # q.query.solutionModifier.limitClause 51 | if q.query.solutionModifier.offsetClause: 52 | log['offset'] = 1 #q.query.solutionModifier.offsetClause 53 | if q.query.solutionModifier.orderClause: 54 | log['order'] = 1 #q.query.solutionModifier.orderClause 55 | 56 | log['triple'] = 0 57 | log['group'] = 0 58 | log['optional'] = 0 59 | log['union'] = 0 60 | log['graph'] = 0 61 | log['filter'] = 0 62 | 63 | # FROM? FROM NAMED? 64 | 65 | if q.query.whereClause: 66 | GetQueryWhereInfo(q.query.whereClause.parsedGraphPattern,None,vars,log) 67 | 68 | log['selectVars'] = len(q.query.variables) 69 | log['allVars'] = len(vars) 70 | if len(q.query.variables) < 1: 71 | log['selectVars'] = len(vars) # SELECT * 72 | 73 | def GetQueryWhereInfo(pat, context, vars, log): 74 | foundType = False 75 | 76 | if isinstance(pat, ParsedGraphGraphPattern): 77 | foundType = True 78 | if isinstance(pat.name, Variable): 79 | context = '?' 80 | else: 81 | context = 'c' 82 | log['graph'] = log['graph'] + 1 83 | 84 | if isinstance(pat, ParsedAlternativeGraphPattern): # not a group 85 | foundType = True 86 | log['union'] = log['union'] + 1 87 | for a in pat.alternativePatterns: 88 | GetQueryWhereInfo(a, context, vars, log) 89 | 90 | if isinstance(pat, ParsedOptionalGraphPattern): 91 | foundType = True 92 | log['optional'] = log['optional'] + 1 93 | 94 | if isinstance(pat, ParsedGroupGraphPattern): 95 | foundType = True 96 | log['group'] = log['group'] + 1 97 | for p in pat.graphPatterns: 98 | GetQueryWhereInfo(p, context, vars, log) 99 | 100 | if isinstance(pat, GraphPattern): 101 | foundType = True 102 | for s in pat.triples: 103 | for p in s.propVals: 104 | for o in p.objects: 105 | GetTripleInfo(s.identifier, p.property, o, context, vars, log) 106 | if pat.nonTripleGraphPattern: 107 | GetQueryWhereInfo(pat.nonTripleGraphPattern, context, vars, log) 108 | if pat.filter: 109 | log['filter'] = log['filter'] + 1 110 | GetQueryFilterInfo(pat.filter.filter, log) 111 | 112 | if not foundType: 113 | print 'Unexpected type: %s' % pat.type 114 | raise Exception('Unexpected type: %s' % pat.type) 115 | 116 | 117 | def GetTripleInfo(s,p,o,context,vars,log): 118 | log['triple'] = log['triple'] + 1 119 | si = 's' 120 | pi = 'p' 121 | oi = 'o' 122 | if isinstance(s, Variable): 123 | si = '?' 124 | GetVarInfo(s, vars) 125 | if isinstance(p, Variable): 126 | pi = '?' 127 | GetVarInfo(p, vars) 128 | if isinstance(o, Variable): 129 | oi = '?' 130 | GetVarInfo(o, vars) 131 | 132 | if context: 133 | quadPat = 'AP(%s,%s,%s,%s)' % (si,pi,oi,context) 134 | if not log.has_key(quadPat): 135 | log[quadPat] = 1 136 | else: 137 | log[quadPat] = log[quadPat] + 1 138 | else: 139 | triplePat = 'AP(%s,%s,%s)' % (si,pi,oi) 140 | if not log.has_key(triplePat): 141 | log[triplePat] = 1 142 | else: 143 | log[triplePat] = log[triplePat] + 1 144 | 145 | def GetVarInfo(v, vars): 146 | if not vars.has_key(v): 147 | vars[v] = 1 148 | else: 149 | vars[v] = vars[v] + 1 150 | 151 | def GetQueryFilterInfo(filter, log): 152 | op = None 153 | if filter == None: 154 | return 155 | if isinstance(filter, Variable): 156 | return 157 | if isinstance(filter, Literal): 158 | return 159 | 160 | if isinstance(filter, BinaryOperator): 161 | op = 'OP-B' + filter.NAME 162 | GetQueryFilterInfo(filter.left, log) 163 | GetQueryFilterInfo(filter.right, log) 164 | if isinstance(filter, UnaryOperator): 165 | op = 'OP-U' + filter.NAME 166 | GetQueryFilterInfo(filter.argument, log) 167 | 168 | if isinstance(filter, ParsedREGEXInvocation): 169 | op = 'OP-REGEX' 170 | GetQueryFilterInfo(filter.arg1, log) 171 | GetQueryFilterInfo(filter.arg2, log) 172 | GetQueryFilterInfo(filter.arg3, log) 173 | 174 | if isinstance(filter, ParsedDatatypedLiteral): 175 | op = 'OP-TLit-' + filter.dataType 176 | 177 | if isinstance(filter, ListRedirect): 178 | for f in filter: 179 | GetQueryFilterInfo(f, log) 180 | if isinstance(filter, ParsedConditionalAndExpressionList): 181 | op = 'OP-OR' 182 | if isinstance(filter, ParsedRelationalExpressionList): 183 | op = 'OP-AND' 184 | if isinstance(filter, ParsedPrefixedMultiplicativeExpressionList): 185 | op = 'OP' + filter.prefix 186 | 187 | if not op: 188 | print 'Unknown filter operator: %s' % filter 189 | raise Exception('Unknown filter operator: %s' % filter) 190 | 191 | if not log.has_key(op): 192 | log[op] = 1 193 | else: 194 | log[op] = log[op] + 1 195 | 196 | def GetQueryStats(queryEntries): 197 | colDict = dict() 198 | cols = [] 199 | entries = [] 200 | entryCount = 0 201 | distinctQueries = dict() 202 | for log in queryEntries: 203 | entryCount = entryCount + 1 204 | #print "=== Entry %s (%s) ===" % (entryCount, log['lineNum']) 205 | if QueryStats(log['queryString'], log): 206 | if not distinctQueries.has_key(log['queryString'].strip()): 207 | distinctQueries[log['queryString'].strip()] = len(distinctQueries) + 1 208 | 209 | log['queryNum'] = distinctQueries[log['queryString'].strip()] 210 | entries.append(log) 211 | # get column names 212 | for k in log: 213 | if not colDict.has_key(k): 214 | colDict[k] = k 215 | cols.append(k) 216 | 217 | cols.sort() 218 | return (cols, entries) 219 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mhausenblas/web.instata/ccebeb2d51c42e849152363ce75b22def579f4a3/lib/rdfextras/tools/__init__.py -------------------------------------------------------------------------------- /lib/rdfextras/tools/pathutils.py: -------------------------------------------------------------------------------- 1 | """ 2 | RDF- and RDFlib-centric file and URL path utilities. 3 | """ 4 | 5 | from os.path import splitext 6 | 7 | 8 | def uri_leaf(uri): 9 | """ 10 | Get the "leaf" - fragment id or last segment - of a URI. Useful e.g. for 11 | getting a term from a "namespace like" URI. Examples: 12 | 13 | >>> uri_leaf('http://example.org/ns/things#item') 14 | 'item' 15 | >>> uri_leaf('http://example.org/ns/stuff/item') 16 | 'item' 17 | >>> uri_leaf('http://example.org/ns/stuff/') 18 | '' 19 | """ 20 | return uri.rsplit('/', 1)[-1].rsplit('#', 1)[-1] 21 | 22 | 23 | SUFFIX_FORMAT_MAP = { 24 | 'rdf': 'xml', 25 | 'rdfs': 'xml', 26 | 'owl': 'xml', 27 | 'n3': 'n3', 28 | 'ttl': 'n3', 29 | 'nt': 'nt', 30 | 'trix': 'trix', 31 | 'xhtml': 'rdfa', 32 | 'html': 'rdfa', 33 | 'svg': 'rdfa', 34 | } 35 | 36 | def guess_format(fpath, fmap=None): 37 | """ 38 | Guess RDF serialization based on file suffix. Uses 39 | ``SUFFIX_FORMAT_MAP`` unless ``fmap`` is provided. Examples: 40 | 41 | >>> guess_format('path/to/file.rdf') 42 | 'xml' 43 | >>> guess_format('path/to/file.owl') 44 | 'xml' 45 | >>> guess_format('path/to/file.ttl') 46 | 'n3' 47 | >>> guess_format('path/to/file.xhtml') 48 | 'rdfa' 49 | >>> guess_format('path/to/file.svg') 50 | 'rdfa' 51 | >>> guess_format('path/to/file.xhtml', {'xhtml': 'grddl'}) 52 | 'grddl' 53 | 54 | This also works with just the suffixes, with or without leading dot, and 55 | regardless of letter case:: 56 | 57 | >>> guess_format('.rdf') 58 | 'xml' 59 | >>> guess_format('rdf') 60 | 'xml' 61 | >>> guess_format('RDF') 62 | 'xml' 63 | """ 64 | fmap = fmap or SUFFIX_FORMAT_MAP 65 | return fmap.get(_get_ext(fpath)) or fmap.get(fpath.lower()) 66 | 67 | 68 | def _get_ext(fpath, lower=True): 69 | """ 70 | Gets the file extension from a file(path); stripped of leading '.' and in 71 | lower case. Examples: 72 | 73 | >>> _get_ext("path/to/file.txt") 74 | 'txt' 75 | >>> _get_ext("OTHER.PDF") 76 | 'pdf' 77 | >>> _get_ext("noext") 78 | '' 79 | """ 80 | ext = splitext(fpath)[-1] 81 | if lower: 82 | ext = ext.lower() 83 | if ext.startswith('.'): 84 | ext = ext[1:] 85 | return ext 86 | 87 | 88 | -------------------------------------------------------------------------------- /lib/rdfextras/tools/rdfpipe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | """ 4 | A commandline tool for parsing RDF in different formats and serializing the 5 | resulting graph to a chosen format. 6 | """ 7 | 8 | import sys 9 | from optparse import OptionParser 10 | import logging 11 | 12 | import rdflib 13 | from rdflib import plugin 14 | from rdflib.store import Store 15 | from rdflib.graph import Graph 16 | from rdflib.namespace import Namespace, RDF, RDFS, OWL, XSD 17 | from rdflib.parser import Parser 18 | from rdflib.serializer import Serializer 19 | 20 | from rdfextras.tools.pathutils import guess_format 21 | 22 | 23 | STORE_CONNECTION = '' 24 | STORE_TYPE = 'IOMemory' 25 | 26 | DEFAULT_INPUT_FORMAT = 'xml' 27 | DEFAULT_OUTPUT_FORMAT = 'n3' 28 | 29 | NS_BINDINGS = { 30 | 'rdf': RDF, 31 | 'rdfs': RDFS, 32 | 'owl': OWL, 33 | 'xsd': XSD, 34 | 'dc': "http://purl.org/dc/elements/1.1/", 35 | 'dct': "http://purl.org/dc/terms/", 36 | 'foaf': "http://xmlns.com/foaf/0.1/", 37 | 'wot': "http://xmlns.com/wot/0.1/" 38 | } 39 | 40 | 41 | def parse_and_serialize(input_files, input_format, guess, 42 | outfile, output_format, ns_bindings, 43 | store_conn=STORE_CONNECTION, store_type=STORE_TYPE): 44 | 45 | store = plugin.get(store_type, Store)() 46 | store.open(store_conn) 47 | graph = Graph(store) 48 | 49 | for prefix, uri in ns_bindings.items(): 50 | graph.namespace_manager.bind(prefix, uri, override=False) 51 | 52 | for fpath in input_files: 53 | use_format, kws = _format_and_kws(input_format) 54 | if fpath == '-': 55 | fpath = sys.stdin 56 | elif not input_format and guess: 57 | use_format = guess_format(fpath) or DEFAULT_INPUT_FORMAT 58 | graph.parse(fpath, format=use_format, **kws) 59 | 60 | if outfile: 61 | output_format, kws = _format_and_kws(output_format) 62 | graph.serialize(destination=outfile, format=output_format, base=None, **kws) 63 | store.rollback() 64 | 65 | def _format_and_kws(fmt): 66 | """ 67 | >>> _format_and_kws("fmt") 68 | ('fmt', {}) 69 | >>> _format_and_kws("fmt:+a") 70 | ('fmt', {'a': True}) 71 | >>> _format_and_kws("fmt:a") 72 | ('fmt', {'a': True}) 73 | >>> _format_and_kws("fmt:+a,-b") 74 | ('fmt', {'a': True, 'b': False}) 75 | >>> _format_and_kws("fmt:c=d") 76 | ('fmt', {'c': 'd'}) 77 | """ 78 | fmt, kws = fmt, {} 79 | if fmt and ':' in fmt: 80 | fmt, kwrepr = fmt.split(':') 81 | for kw in kwrepr.split(','): 82 | if '=' in kw: 83 | k, v = kw.split('=') 84 | kws[k] = v 85 | elif kw.startswith('-'): 86 | kws[kw[1:]] = False 87 | elif kw.startswith('+'): 88 | kws[kw[1:]] = True 89 | else: # same as "+" 90 | kws[kw] = True 91 | return fmt, kws 92 | 93 | 94 | def make_option_parser(): 95 | parser_names = _get_plugin_names(Parser) 96 | serializer_names = _get_plugin_names(Serializer) 97 | kw_example = "FORMAT:(+)KW1,-KW2,KW3=VALUE" 98 | 99 | oparser = OptionParser( 100 | "%prog [-h] [-i INPUT_FORMAT] [-o OUTPUT_FORMAT] [--ns=PFX=NS ...] [-] [FILE ...]", 101 | description=__doc__.strip() + ( 102 | " Reads file system paths, URLs or from stdin if '-' is given." 103 | " The result is serialized to stdout."), 104 | version="%prog " + "(using rdflib %s)" % rdflib.__version__) 105 | 106 | oparser.add_option('-i', '--input-format', 107 | type=str, #default=DEFAULT_INPUT_FORMAT, 108 | help="Format of the input document(s)." 109 | " Available input formats are: %s." % parser_names + 110 | " If no format is given, it will be guessed from the file name extension." 111 | " Keywords to parser can be given after format like: %s." % kw_example 112 | , 113 | metavar="INPUT_FORMAT") 114 | 115 | oparser.add_option('-o', '--output-format', 116 | type=str, default=DEFAULT_OUTPUT_FORMAT, 117 | help="Format of the graph serialization." 118 | " Available output formats are: %s." 119 | % serializer_names + 120 | " Default format is: '%default'." + 121 | " Keywords to serializer can be given after format like: %s." % kw_example 122 | , 123 | metavar="OUTPUT_FORMAT") 124 | 125 | oparser.add_option('--ns', 126 | action="append", type=str, 127 | help="Register a namespace binding (QName prefix to a base URI). " 128 | "This can be used more than once.", 129 | metavar="PREFIX=NAMESPACE") 130 | 131 | oparser.add_option('--no-guess', dest='guess', 132 | action='store_false', default=True, 133 | help="Don't guess format based on file suffix.") 134 | 135 | oparser.add_option('--no-out', 136 | action='store_true', default=False, 137 | help="Don't output the resulting graph (useful for checking validity of input).") 138 | 139 | oparser.add_option('-w', '--warn', 140 | action='store_true', default=False, 141 | help="Output warnings to stderr (by default only critical errors).") 142 | 143 | return oparser 144 | 145 | _get_plugin_names = lambda kind: ", ".join(p.name for p in plugin.plugins(kind=kind)) 146 | 147 | 148 | def main(): 149 | oparser = make_option_parser() 150 | opts, args = oparser.parse_args() 151 | if len(args) < 1: 152 | oparser.print_usage() 153 | oparser.exit() 154 | 155 | if opts.warn: 156 | loglevel = logging.WARNING 157 | else: 158 | loglevel = logging.CRITICAL 159 | logging.basicConfig(level=loglevel) 160 | 161 | ns_bindings = dict(NS_BINDINGS) 162 | if opts.ns: 163 | for ns_kw in opts.ns: 164 | pfx, uri = ns_kw.split('=') 165 | ns_bindings[pfx] = uri 166 | 167 | outfile = sys.stdout 168 | if opts.no_out: 169 | outfile = None 170 | 171 | parse_and_serialize(args, opts.input_format, opts.guess, 172 | outfile, opts.output_format, ns_bindings) 173 | 174 | 175 | if __name__ == "__main__": 176 | main() 177 | 178 | -------------------------------------------------------------------------------- /lib/rdflib/LICENSE: -------------------------------------------------------------------------------- 1 | LICENSE AGREEMENT FOR RDFLIB 0.9.0 THROUGH 2.5.0 2 | ------------------------------------------------ 3 | Copyright (c) 2002-2009, Daniel Krech, http://eikeon.com/ 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following 15 | disclaimer in the documentation and/or other materials provided 16 | with the distribution. 17 | 18 | * Neither the name of Daniel Krech nor the names of its 19 | contributors may be used to endorse or promote products derived 20 | from this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /lib/rdflib/__init__.py: -------------------------------------------------------------------------------- 1 | """\ 2 | A pure Python package providing the core RDF constructs. 3 | 4 | The packages is intended to provide the core RDF types and interfaces 5 | for working with RDF. The package defines a plugin interface for 6 | parsers, stores, and serializers that other packages can use to 7 | implement parsers, stores, and serializers that will plug into the 8 | rdflib package. 9 | 10 | The primary interface `rdflib` exposes to work with RDF is 11 | `rdflib.graph.Graph`. 12 | 13 | A tiny example: 14 | 15 | >>> import rdflib 16 | 17 | >>> g = rdflib.Graph() 18 | >>> result = g.parse("http://eikeon.com/foaf.rdf") 19 | 20 | >>> print "graph has %s statements." % len(g) 21 | graph has 34 statements. 22 | >>> 23 | >>> for s, p, o in g: 24 | ... if (s, p, o) not in g: 25 | ... raise Exception("It better be!") 26 | 27 | >>> s = g.serialize(format='n3') 28 | 29 | """ 30 | __docformat__ = "restructuredtext en" 31 | 32 | __version__ = "3.1.0" 33 | __date__ = "2011/03/17" 34 | 35 | __all__ = [ 36 | 'URIRef', 37 | 'BNode', 38 | 'Literal', 39 | 'Variable', 40 | 41 | 'Namespace', 42 | 43 | 'Graph', 44 | 'ConjunctiveGraph', 45 | 46 | 'RDF', 47 | 'RDFS', 48 | 'OWL', 49 | 'XSD', 50 | 51 | 'util', 52 | ] 53 | 54 | import sys 55 | # generator expressions require 2.4 56 | assert sys.version_info >= (2, 4, 0), "rdflib requires Python 2.4 or higher" 57 | del sys 58 | 59 | import logging 60 | _LOGGER = logging.getLogger("rdflib") 61 | _LOGGER.info("version: %s" % __version__) 62 | 63 | 64 | from rdflib.term import URIRef, BNode, Literal, Variable 65 | 66 | from rdflib.namespace import Namespace 67 | 68 | from rdflib.graph import Graph, ConjunctiveGraph 69 | 70 | from rdflib.namespace import RDF, RDFS, OWL, XSD 71 | 72 | from rdflib import plugin 73 | from rdflib import query 74 | 75 | from rdflib import util 76 | 77 | -------------------------------------------------------------------------------- /lib/rdflib/events.py: -------------------------------------------------------------------------------- 1 | 2 | __doc__ = """ 3 | Dirt Simple Events 4 | 5 | A Dispatcher (or a subclass of Dispatcher) stores event handlers that 6 | are 'fired' simple event objects when interesting things happen. 7 | 8 | Create a dispatcher: 9 | 10 | >>> d = Dispatcher() 11 | 12 | Now create a handler for the event and subscribe it to the dispatcher 13 | to handle Event events. A handler is a simple function or method that 14 | accepts the event as an argument: 15 | 16 | >>> def handler1(event): print `event` 17 | >>> d.subscribe(Event, handler1) 18 | 19 | Now dispatch a new event into the dispatcher, and see handler1 get 20 | fired: 21 | 22 | >>> d.dispatch(Event(foo='bar', data='yours', used_by='the event handlers')) 23 | 24 | """ 25 | 26 | class Event(object): 27 | """ 28 | An event is a container for attributes. The source of an event 29 | creates this object, or a subclass, gives it any kind of data that 30 | the events handlers need to handle the event, and then calls 31 | notify(event). 32 | 33 | The target of an event registers a function to handle the event it 34 | is interested with subscribe(). When a sources calls 35 | notify(event), each subscriber to that even will be called i no 36 | particular order. 37 | """ 38 | 39 | def __init__(self, **kw): 40 | self.__dict__.update(kw) 41 | 42 | def __repr__(self): 43 | attrs = self.__dict__.keys() 44 | attrs.sort() 45 | return '' % ([a for a in attrs],) 46 | 47 | 48 | class Dispatcher(object): 49 | """ 50 | An object that can dispatch events to a privately managed group of 51 | subscribers. 52 | """ 53 | 54 | _dispatch_map = None 55 | 56 | def set_map(self, amap): 57 | self._dispatch_map = amap 58 | 59 | def get_map(self): 60 | return self._dispatch_map 61 | 62 | def subscribe(self, event_type, handler): 63 | """ Subscribe the given handler to an event_type. Handlers 64 | are called in the order they are subscribed. 65 | """ 66 | if self._dispatch_map is None: 67 | self.set_map({}) 68 | lst = self._dispatch_map.get(event_type, None) 69 | if lst is None: 70 | lst = [handler] 71 | else: 72 | lst.append(handler) 73 | self._dispatch_map[event_type] = lst 74 | 75 | def dispatch(self, event): 76 | """ Dispatch the given event to the subscribed handlers for 77 | the event's type""" 78 | if self._dispatch_map is not None: 79 | lst = self._dispatch_map.get(type(event), None) 80 | if lst is None: 81 | raise ValueError("unknown event type: %s" % type(event)) 82 | for l in lst: 83 | l(event) 84 | 85 | def test(): 86 | import doctest 87 | doctest.testmod() 88 | 89 | if __name__ == '__main__': 90 | test() 91 | -------------------------------------------------------------------------------- /lib/rdflib/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | TODO: 3 | """ 4 | 5 | class Error(Exception): 6 | """Base class for rdflib exceptions.""" 7 | def __init__(self, msg=None): 8 | Exception.__init__(self, msg) 9 | self.msg = msg 10 | 11 | 12 | class TypeCheckError(Error): 13 | """Parts of assertions are subject to type checks.""" 14 | 15 | def __init__(self, node): 16 | Error.__init__(self, node) 17 | self.type = type(node) 18 | self.node = node 19 | 20 | 21 | class SubjectTypeError(TypeCheckError): 22 | """Subject of an assertion must be an instance of URIRef.""" 23 | def __init__(self, node): 24 | TypeCheckError.__init__(self, node) 25 | self.msg = "Subject must be instance of URIRef or BNode: %s(%s)" \ 26 | % (self.node, self.type) 27 | 28 | 29 | class PredicateTypeError(TypeCheckError): 30 | """Predicate of an assertion must be an instance of URIRef.""" 31 | def __init__(self, node): 32 | TypeCheckError.__init__(self, node) 33 | self.msg = "Predicate must be a URIRef instance: %s(%s)" \ 34 | % (self.node, self.type) 35 | 36 | 37 | class ObjectTypeError(TypeCheckError): 38 | """Object of an assertion must be an instance of URIRef, Literal, 39 | or BNode.""" 40 | def __init__(self, node): 41 | TypeCheckError.__init__(self, node) 42 | self.msg = "\ 43 | Object must be instance of URIRef, Literal, or BNode: %s(%s)" % \ 44 | (self.node, self.type) 45 | 46 | class ContextTypeError(TypeCheckError): 47 | """Context of an assertion must be an instance of URIRef.""" 48 | def __init__(self, node): 49 | TypeCheckError.__init__(self, node) 50 | self.msg = "Context must be instance of URIRef or BNode: %s(%s)" \ 51 | % (self.node, self.type) 52 | 53 | class ParserError(Error): 54 | """RDF Parser error.""" 55 | def __init__(self, msg): 56 | Error.__init__(self, msg) 57 | self.msg = msg 58 | 59 | def __str__(self): 60 | return self.msg 61 | 62 | 63 | class UniquenessError(Error) : 64 | """A uniqueness assumption was made in the context, and that is not true""" 65 | def __init__(self, values): 66 | Error.__init__(self, "\ 67 | Uniqueness assumption is not fulfilled. Multiple values are: %s" % values) 68 | 69 | 70 | -------------------------------------------------------------------------------- /lib/rdflib/parser.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parser plugin interface. 3 | 4 | This module defines the parser plugin interface and contains other 5 | related parser support code. 6 | 7 | The module is mainly useful for those wanting to write a parser that 8 | can plugin to rdflib. If you are wanting to invoke a parser you likely 9 | want to do so through the Graph class parse method. 10 | 11 | """ 12 | 13 | import os 14 | import __builtin__ 15 | import warnings 16 | from urllib import pathname2url, url2pathname 17 | from urllib2 import urlopen, Request, HTTPError 18 | from urlparse import urljoin 19 | from StringIO import StringIO 20 | from xml.sax import xmlreader 21 | from xml.sax.saxutils import prepare_input_source 22 | import types 23 | try: 24 | _StringTypes = (types.StringType, types.UnicodeType) 25 | except AttributeError: 26 | _StringTypes = (types.StringType,) 27 | 28 | from rdflib import __version__ 29 | from rdflib.term import URIRef 30 | from rdflib.namespace import Namespace 31 | 32 | 33 | class Parser(object): 34 | 35 | def __init__(self): 36 | pass 37 | 38 | def parse(self, source, sink): 39 | pass 40 | 41 | 42 | class InputSource(xmlreader.InputSource, object): 43 | """ 44 | TODO: 45 | """ 46 | 47 | def __init__(self, system_id=None): 48 | xmlreader.InputSource.__init__(self, system_id=system_id) 49 | self.content_type = None 50 | 51 | 52 | class StringInputSource(InputSource): 53 | """ 54 | TODO: 55 | """ 56 | 57 | def __init__(self, value, system_id=None): 58 | super(StringInputSource, self).__init__(system_id) 59 | stream = StringIO(value) 60 | self.setByteStream(stream) 61 | # TODO: 62 | # encoding = value.encoding 63 | # self.setEncoding(encoding) 64 | 65 | 66 | headers = { 67 | 'User-agent': 'rdflib-%s (http://rdflib.net/; eikeon@eikeon.com)' % __version__ 68 | } 69 | 70 | 71 | class URLInputSource(InputSource): 72 | """ 73 | TODO: 74 | """ 75 | 76 | def __init__(self, system_id=None, format=None): 77 | super(URLInputSource, self).__init__(system_id) 78 | self.url = system_id 79 | 80 | # copy headers to change 81 | myheaders=dict(headers) 82 | if format=='application/rdf+xml': 83 | myheaders['Accept']='application/rdf+xml, */*;q=0.1' 84 | elif format=='n3': 85 | myheaders['Accept']='text/n3, */*;q=0.1' 86 | elif format=='nt': 87 | myheaders['Accept']='text/plain, */*;q=0.1' 88 | else: 89 | myheaders['Accept']='application/rdf+xml,text/rdf+n3;q=0.9,application/xhtml+xml;q=0.5, */*;q=0.1' 90 | 91 | req = Request(system_id, None, myheaders) 92 | try: 93 | file = urlopen(req) 94 | except HTTPError, e: 95 | # TODO: 96 | raise Exception('"%s" while trying to open "%s"' % (e, self.url)) 97 | self.content_type = file.info().get('content-type') 98 | self.content_type = self.content_type.split(";", 1)[0] 99 | self.setByteStream(file) 100 | # TODO: self.setEncoding(encoding) 101 | 102 | def __repr__(self): 103 | return self.url 104 | 105 | 106 | class FileInputSource(InputSource): 107 | """ 108 | TODO: 109 | """ 110 | 111 | def __init__(self, file): 112 | base = urljoin("file:", pathname2url(os.getcwd())) 113 | system_id = URIRef(file.name, base=base) 114 | super(FileInputSource, self).__init__(system_id) 115 | self.file = file 116 | self.setByteStream(file) 117 | # TODO: self.setEncoding(encoding) 118 | 119 | def __repr__(self): 120 | return `self.file` 121 | 122 | 123 | def create_input_source(source=None, publicID=None, 124 | location=None, file=None, data=None, format=None): 125 | """ 126 | Return an appropriate InputSource instance for the given 127 | parameters. 128 | """ 129 | 130 | # TODO: test that exactly one of source, location, file, and data 131 | # is not None. 132 | 133 | input_source = None 134 | 135 | if source is not None: 136 | if isinstance(source, InputSource): 137 | input_source = source 138 | else: 139 | if isinstance(source, _StringTypes): 140 | location = source 141 | elif hasattr(source, "read") and not isinstance(source, Namespace): 142 | f = source 143 | input_source = InputSource() 144 | input_source.setByteStream(f) 145 | if hasattr(f, "name"): 146 | input_source.setSystemId(f.name) 147 | else: 148 | raise Exception("Unexpected type '%s' for source '%s'" % (type(source), source)) 149 | 150 | if location is not None: 151 | base = urljoin("file:", "%s/" % pathname2url(os.getcwd())) 152 | absolute_location = URIRef(location, base=base).defrag() 153 | if absolute_location.startswith("file:///"): 154 | filename = url2pathname(absolute_location.replace("file:///", "/")) 155 | file = __builtin__.file(filename, "rb") 156 | else: 157 | input_source = URLInputSource(absolute_location, format) 158 | publicID = publicID or absolute_location 159 | 160 | if file is not None: 161 | input_source = FileInputSource(file) 162 | 163 | if data is not None: 164 | if isinstance(data, unicode): 165 | data = data.encode('utf-8') 166 | input_source = StringInputSource(data) 167 | 168 | if input_source is None: 169 | raise Exception("could not create InputSource") 170 | else: 171 | if publicID: 172 | input_source.setPublicId(publicID) 173 | 174 | # TODO: what motivated this bit? 175 | id = input_source.getPublicId() 176 | if id is None: 177 | input_source.setPublicId("") 178 | return input_source 179 | 180 | 181 | -------------------------------------------------------------------------------- /lib/rdflib/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Plugin support for rdf. 3 | 4 | There are a number of plugin points for rdf: parser, serializer, 5 | store, query processor, and query result. Plugins can be registered 6 | either through setuptools entry_points or by calling 7 | rdf.plugin.register directly. 8 | 9 | If you have a package that uses a setuptools based setup.py you can add the following to your setup:: 10 | 11 | entry_points = { 12 | 'rdf.plugins.parser': [ 13 | 'nt = rdf.plugins.parsers.nt:NTParser', 14 | ], 15 | 'rdf.plugins.serializer': [ 16 | 'nt = rdf.plugins.serializers.NTSerializer:NTSerializer', 17 | ], 18 | } 19 | 20 | See the `setuptools dynamic discovery of services and plugins`__ for more information. 21 | 22 | .. __: http://peak.telecommunity.com/DevCenter/setuptools#dynamic-discovery-of-services-and-plugins 23 | 24 | """ 25 | 26 | from rdflib.store import Store 27 | from rdflib.parser import Parser 28 | from rdflib.serializer import Serializer 29 | from rdflib import query 30 | from rdflib.exceptions import Error 31 | 32 | 33 | entry_points = {'rdf.plugins.store': Store, 34 | 'rdf.plugins.serializer': Serializer, 35 | 'rdf.plugins.parser': Parser} 36 | 37 | _plugins = {} 38 | 39 | 40 | class PluginException(Error): 41 | pass 42 | 43 | 44 | class Plugin(object): 45 | 46 | def __init__(self, name, kind, module_path, class_name): 47 | self.name = name 48 | self.kind = kind 49 | self.module_path = module_path 50 | self.class_name = class_name 51 | self._class = None 52 | 53 | def getClass(self): 54 | if self._class is None: 55 | module = __import__(self.module_path, globals(), locals(), [""]) 56 | self._class = getattr(module, self.class_name) 57 | return self._class 58 | 59 | 60 | class PKGPlugin(Plugin): 61 | 62 | def __init__(self, name, kind, ep): 63 | self.name = name 64 | self.kind = kind 65 | self.ep = ep 66 | self._class = None 67 | 68 | def getClass(self): 69 | if self._class is None: 70 | self._class = self.ep.load() 71 | return self._class 72 | 73 | 74 | def register(name, kind, module_path, class_name): 75 | """ 76 | Register the plugin for (name, kind). The module_path and 77 | class_name should be the path to a plugin class. 78 | """ 79 | p = Plugin(name, kind, module_path, class_name) 80 | _plugins[(name, kind)] = p 81 | 82 | 83 | def get(name, kind): 84 | """ 85 | Return the class for the specified (name, kind). Raises a 86 | PluginException if unable to do so. 87 | """ 88 | try: 89 | p = _plugins[(name, kind)] 90 | except KeyError, e: 91 | raise PluginException("No plugin registered for (%s, %s)" % (name, kind)) 92 | return p.getClass() 93 | 94 | 95 | try: 96 | from pkg_resources import iter_entry_points 97 | except ImportError: 98 | pass # TODO: log a message 99 | else: 100 | # add the plugins specified via pkg_resources' EntryPoints. 101 | for entry_point, kind in entry_points.iteritems(): 102 | for ep in iter_entry_points(entry_point): 103 | _plugins[(ep.name, kind)] = PKGPlugin(ep.name, kind, ep) 104 | 105 | 106 | def plugins(name=None, kind=None): 107 | """ 108 | A generator of the plugins. 109 | 110 | Pass in name and kind to filter... else leave None to match all. 111 | """ 112 | for p in _plugins.values(): 113 | if (name is None or name==p.name) and (kind is None or kind==p.kind): 114 | yield p 115 | 116 | register('default', Store, 117 | 'rdflib.plugins.memory', 'IOMemory') 118 | register('IOMemory', Store, 119 | 'rdflib.plugins.memory', 'IOMemory') 120 | register('Sleepycat', Store, 121 | 'rdflib.plugins.sleepycat', 'Sleepycat') 122 | 123 | register('xml', Serializer, 124 | 'rdflib.plugins.serializers.rdfxml', 'XMLSerializer') 125 | register('n3', Serializer, 126 | 'rdflib.plugins.serializers.n3','N3Serializer') 127 | register('turtle', Serializer, 128 | 'rdflib.plugins.serializers.turtle', 'TurtleSerializer') 129 | register('nt', Serializer, 130 | 'rdflib.plugins.serializers.nt', 'NTSerializer') 131 | register('pretty-xml', Serializer, 132 | 'rdflib.plugins.serializers.rdfxml', 'PrettyXMLSerializer') 133 | register('trix', Serializer, 134 | 'rdflib.plugins.serializers.trix', 'TriXSerializer') 135 | 136 | register('application/rdf+xml', Parser, 137 | 'rdflib.plugins.parsers.rdfxml', 'RDFXMLParser') 138 | register('text/html', Parser, 139 | 'rdflib.plugins.parsers.rdfa', 'RDFaParser') 140 | register('application/xhtml+xml', Parser, 141 | 'rdflib.plugins.parsers.rdfa', 'RDFaParser') 142 | register('xml', Parser, 143 | 'rdflib.plugins.parsers.rdfxml', 'RDFXMLParser') 144 | register('n3', Parser, 145 | 'rdflib.plugins.parsers.notation3', 'N3Parser') 146 | register('nt', Parser, 147 | 'rdflib.plugins.parsers.nt', 'NTParser') 148 | register('trix', Parser, 149 | 'rdflib.plugins.parsers.trix', 'TriXParser') 150 | register('rdfa', Parser, 151 | 'rdflib.plugins.parsers.rdfa', 'RDFaParser') 152 | 153 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Default plugins for rdflib. 3 | 4 | This is a namespace package and contains the default plugins for 5 | rdflib. 6 | 7 | """ 8 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | """ 4 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/nt.py: -------------------------------------------------------------------------------- 1 | from rdflib.parser import Parser 2 | from rdflib.plugins.parsers.ntriples import NTriplesParser 3 | 4 | 5 | class NTSink(object): 6 | def __init__(self, graph): 7 | self.graph = graph 8 | 9 | def triple(self, s, p, o): 10 | self.graph.add((s, p, o)) 11 | 12 | 13 | class NTParser(Parser): 14 | """parser for the ntriples format, often stored with the .nt extension 15 | 16 | See http://www.w3.org/TR/rdf-testcases/#ntriples""" 17 | 18 | def __init__(self): 19 | super(NTParser, self).__init__() 20 | 21 | def parse(self, source, sink, baseURI=None): 22 | f = source.getByteStream() # TODO getCharacterStream? 23 | parser = NTriplesParser(NTSink(sink)) 24 | parser.parse(f) 25 | f.close() 26 | 27 | 28 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/ntriples.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | __doc__=""" 3 | N-Triples Parser 4 | License: GPL 2, W3C, BSD, or MIT 5 | Author: Sean B. Palmer, inamidst.com 6 | Documentation: http://inamidst.com/proj/rdf/ntriples-doc 7 | 8 | Command line usage:: 9 | 10 | ./ntriples.py - parses URI as N-Triples 11 | ./ntriples.py --help - prints out this help message 12 | 13 | # @@ fully empty document? 14 | """ 15 | 16 | import re 17 | 18 | uriref = r'<([^:]+:[^\s"<>]+)>' 19 | literal = r'"([^"\\]*(?:\\.[^"\\]*)*)"' 20 | litinfo = r'(?:@([a-z]+(?:-[a-z0-9]+)*)|\^\^' + uriref + r')?' 21 | 22 | r_line = re.compile(r'([^\r\n]*)(?:\r\n|\r|\n)') 23 | r_wspace = re.compile(r'[ \t]*') 24 | r_wspaces = re.compile(r'[ \t]+') 25 | r_tail = re.compile(r'[ \t]*\.[ \t]*') 26 | r_uriref = re.compile(uriref) 27 | r_nodeid = re.compile(r'_:([A-Za-z][A-Za-z0-9]*)') 28 | r_literal = re.compile(literal + litinfo) 29 | 30 | bufsiz = 2048 31 | validate = False 32 | 33 | class Node(unicode): pass 34 | 35 | # class URI(Node): pass 36 | # class bNode(Node): pass 37 | # class Literal(Node): 38 | # def __new__(cls, lit, lang=None, dtype=None): 39 | # n = str(lang) + ' ' + str(dtype) + ' ' + lit 40 | # return unicode.__new__(cls, n) 41 | 42 | from rdflib.term import URIRef as URI 43 | from rdflib.term import BNode as bNode 44 | from rdflib.term import Literal 45 | 46 | class Sink(object): 47 | def __init__(self): 48 | self.length = 0 49 | 50 | def triple(self, s, p, o): 51 | self.length += 1 52 | print (s, p, o) 53 | 54 | class ParseError(Exception): pass 55 | 56 | quot = {'t': '\t', 'n': '\n', 'r': '\r', '"': '"', '\\': '\\'} 57 | r_safe = re.compile(r'([\x20\x21\x23-\x5B\x5D-\x7E]+)') 58 | r_quot = re.compile(r'\\(t|n|r|"|\\)') 59 | r_uniquot = re.compile(r'\\u([0-9A-F]{4})|\\U([0-9A-F]{8})') 60 | 61 | def unquote(s): 62 | """Unquote an N-Triples string.""" 63 | result = [] 64 | while s: 65 | m = r_safe.match(s) 66 | if m: 67 | s = s[m.end():] 68 | result.append(m.group(1)) 69 | continue 70 | 71 | m = r_quot.match(s) 72 | if m: 73 | s = s[2:] 74 | result.append(quot[m.group(1)]) 75 | continue 76 | 77 | m = r_uniquot.match(s) 78 | if m: 79 | s = s[m.end():] 80 | u, U = m.groups() 81 | codepoint = int(u or U, 16) 82 | if codepoint > 0x10FFFF: 83 | raise ParseError("Disallowed codepoint: %08X" % codepoint) 84 | result.append(unichr(codepoint)) 85 | elif s.startswith('\\'): 86 | raise ParseError("Illegal escape at: %s..." % s[:10]) 87 | else: raise ParseError("Illegal literal character: %r" % s[0]) 88 | return unicode(''.join(result)) 89 | 90 | if not validate: 91 | def unquote(s): 92 | return s.decode('unicode-escape') 93 | 94 | r_hibyte = re.compile(r'([\x80-\xFF])') 95 | 96 | def uriquote(uri): 97 | return r_hibyte.sub(lambda m: '%%%02X' % ord(m.group(1)), uri) 98 | if not validate: 99 | def uriquote(uri): 100 | return uri 101 | 102 | class NTriplesParser(object): 103 | """An N-Triples Parser. 104 | 105 | Usage:: 106 | 107 | p = NTriplesParser(sink=MySink()) 108 | sink = p.parse(f) # file; use parsestring for a string 109 | """ 110 | 111 | def __init__(self, sink=None): 112 | if sink is not None: 113 | self.sink = sink 114 | else: self.sink = Sink() 115 | 116 | def parse(self, f): 117 | """Parse f as an N-Triples file.""" 118 | if not hasattr(f, 'read'): 119 | raise ParseError("Item to parse must be a file-like object.") 120 | 121 | self.file = f 122 | self.buffer = '' 123 | while True: 124 | self.line = self.readline() 125 | if self.line is None: break 126 | try: self.parseline() 127 | except ParseError: 128 | raise ParseError("Invalid line: %r" % self.line) 129 | return self.sink 130 | 131 | def parsestring(self, s): 132 | """Parse s as an N-Triples string.""" 133 | if not isinstance(s, basestring): 134 | raise ParseError("Item to parse must be a string instance.") 135 | from cStringIO import StringIO 136 | f = StringIO() 137 | f.write(s) 138 | f.seek(0) 139 | self.parse(f) 140 | 141 | def readline(self): 142 | """Read an N-Triples line from buffered input.""" 143 | # N-Triples lines end in either CRLF, CR, or LF 144 | # Therefore, we can't just use f.readline() 145 | if not self.buffer: 146 | buffer = self.file.read(bufsiz) 147 | if not buffer: return None 148 | self.buffer = buffer 149 | 150 | while True: 151 | m = r_line.match(self.buffer) 152 | if m: # the more likely prospect 153 | self.buffer = self.buffer[m.end():] 154 | return m.group(1) 155 | else: 156 | buffer = self.file.read(bufsiz) 157 | if not buffer and not self.buffer.isspace(): 158 | raise ParseError("EOF in line") 159 | elif not buffer: 160 | return None 161 | self.buffer += buffer 162 | 163 | def parseline(self): 164 | self.eat(r_wspace) 165 | if (not self.line) or self.line.startswith('#'): 166 | return # The line is empty or a comment 167 | 168 | subject = self.subject() 169 | self.eat(r_wspaces) 170 | 171 | predicate = self.predicate() 172 | self.eat(r_wspaces) 173 | 174 | object = self.object() 175 | self.eat(r_tail) 176 | 177 | if self.line: 178 | raise ParseError("Trailing garbage") 179 | self.sink.triple(subject, predicate, object) 180 | 181 | def peek(self, token): 182 | return self.line.startswith(token) 183 | 184 | def eat(self, pattern): 185 | m = pattern.match(self.line) 186 | if not m: # @@ Why can't we get the original pattern? 187 | raise ParseError("Failed to eat %s" % pattern) 188 | self.line = self.line[m.end():] 189 | return m 190 | 191 | def subject(self): 192 | # @@ Consider using dictionary cases 193 | subj = self.uriref() or self.nodeid() 194 | if not subj: 195 | raise ParseError("Subject must be uriref or nodeID") 196 | return subj 197 | 198 | def predicate(self): 199 | pred = self.uriref() 200 | if not pred: 201 | raise ParseError("Predicate must be uriref") 202 | return pred 203 | 204 | def object(self): 205 | objt = self.uriref() or self.nodeid() or self.literal() 206 | if objt is False: 207 | raise ParseError("Unrecognised object type") 208 | return objt 209 | 210 | def uriref(self): 211 | if self.peek('<'): 212 | uri = self.eat(r_uriref).group(1) 213 | uri = unquote(uri) 214 | uri = uriquote(uri) 215 | return URI(uri) 216 | return False 217 | 218 | def nodeid(self): 219 | if self.peek('_'): 220 | return bNode(self.eat(r_nodeid).group(1)) 221 | return False 222 | 223 | def literal(self): 224 | if self.peek('"'): 225 | lit, lang, dtype = self.eat(r_literal).groups() 226 | lang = lang or None 227 | dtype = dtype or None 228 | if lang and dtype: 229 | raise ParseError("Can't have both a language and a datatype") 230 | lit = unquote(lit) 231 | return Literal(lit, lang, dtype) 232 | return False 233 | 234 | def parseURI(uri): 235 | import urllib 236 | parser = NTriplesParser() 237 | u = urllib.urlopen(uri) 238 | sink = parser.parse(u) 239 | u.close() 240 | # for triple in sink: 241 | # print triple 242 | print 'Length of input:', sink.length 243 | 244 | def main(): 245 | import sys 246 | if len(sys.argv) == 2: 247 | parseURI(sys.argv[1]) 248 | else: print __doc__ 249 | 250 | if __name__=="__main__": 251 | main() 252 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/rdfa/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | From a Python file, expecting an RDF/XML pretty printed output:: 3 | 4 | import rdflib.graph as g 5 | graph = g.Graph() 6 | graph.parse('filename.html', format='rdfa') 7 | print graph.serialize(format='pretty-xml') 8 | 9 | For details on RDFa, the reader should consult the `RDFa syntax document`__. 10 | 11 | This is an adapted version of pyRdfa (`W3C RDFa Distiller page`__) by Ivan Herman 12 | 13 | .. __: http://www.w3.org/TR/rdfa-syntax 14 | .. __: http://www.w3.org/2007/08/pyRdfa/ 15 | 16 | """ 17 | 18 | 19 | import sys 20 | import urllib 21 | import xml.dom.minidom 22 | 23 | from rdflib.graph import Graph 24 | from rdflib.namespace import Namespace 25 | from rdflib.term import BNode, URIRef 26 | from rdflib.parser import Parser 27 | from rdflib.plugins.parsers.rdfa.state import ExecutionContext 28 | from rdflib.plugins.parsers.rdfa.parse import parse_one_node 29 | from rdflib.plugins.parsers.rdfa.options import (Options, _add_to_comment_graph, 30 | DIST_NS, ERROR, GENERIC_XML, XHTML_RDFA, HTML5_RDFA) 31 | 32 | from rdflib.plugins.parsers.rdfa.transform.headabout import head_about_transform 33 | 34 | 35 | # These are part of the RDFa spec. 36 | BUILT_IN_TRANSFORMERS = [ 37 | head_about_transform 38 | ] 39 | 40 | # Exception handling. Essentially, all the different exceptions are re-packaged 41 | # into separate exception class, to allow for an easier management on the user 42 | # level 43 | class RDFaError(Exception) : 44 | """Just a wrapper around the local exceptions. It does not add any new 45 | functionality to the Exception class.""" 46 | pass 47 | 48 | # For some doctype and element name combinations an automatic switch to an 49 | # input mode is done 50 | _HOST_LANG = { 51 | ("http://www.w3.org/1999/xhtml", "html"): XHTML_RDFA, 52 | ("http://www.w3.org/2000/svg", "svg"): GENERIC_XML 53 | } 54 | 55 | 56 | class RDFaParser(Parser): 57 | 58 | def parse(self, source, sink, 59 | warnings=False, space_preserve=True, 60 | transformers=None, xhtml=True, lax=True): 61 | if transformers is None: 62 | transformers = [] 63 | options = Options(warnings, space_preserve, transformers, xhtml, lax) 64 | baseURI = source.getPublicId() 65 | stream = source.getByteStream() 66 | dom = _try_process_source(stream, options) 67 | _process_DOM(dom, baseURI, sink, options) 68 | 69 | 70 | def _process_DOM(dom, base, graph, options=None): 71 | """ 72 | Core processing. The transformers ("pre-processing") is done on the DOM 73 | tree, the state is initialized, and the "real" RDFa parsing is done. 74 | The result is put into the provided Graph. 75 | 76 | The real work is done in the parser function ``parse_one_node()``. 77 | 78 | Params: 79 | dom -- XML DOM Tree node (for the top level) 80 | base -- URI for the default "base" value (usually the URI of the file to be processed) 81 | 82 | Options: 83 | obj -- `Options` for the distiller 84 | raise RDFaError -- when called via CGI, this encapsulates the possible 85 | exceptions raised by the RDFLib serializer or the processing itself 86 | """ 87 | html = dom.documentElement 88 | # Perform the built-in and external transformations on the HTML tree. This is, 89 | # in simulated form, the hGRDDL approach of Ben Adida. 90 | for trans in options.transformers + BUILT_IN_TRANSFORMERS: 91 | trans(html, options) 92 | # Collect the initial state. This takes care of things 93 | # like base, top level namespace settings, etc. 94 | # Ensure the proper initialization. 95 | state = ExecutionContext(html, graph, base=base, options=options) 96 | # The top level subject starts with the current document; this 97 | # is used by the recursion 98 | subject = URIRef(state.base) 99 | # Parse the whole thing recursively and fill the graph. 100 | parse_one_node(html, graph, subject, state, []) 101 | if options.comment_graph.graph != None: 102 | # Add the content of the comment graph to the output. 103 | graph.bind("dist", DIST_NS) 104 | for t in options.comment_graph.graph: 105 | graph.add(t) 106 | 107 | def _try_process_source(stream, options): 108 | """ 109 | Tries to parse input as xhtml, xml (e.g. svg) or html(5), modifying options 110 | while figuring out input.. 111 | 112 | Returns a DOM tree. 113 | """ 114 | parse = xml.dom.minidom.parse 115 | try: 116 | dom = parse(stream) 117 | # Try to second-guess the input type 118 | # This is _not_ really kosher, but the minidom is not really namespace aware... 119 | # In practice the goal is to have the system recognize svg content automatically 120 | # First see if there is a default namespace defined for the document: 121 | top = dom.documentElement 122 | if top.hasAttribute("xmlns"): 123 | key = (top.getAttribute("xmlns"), top.nodeName) 124 | if key in _HOST_LANG: 125 | options.host_language = _HOST_LANG[key] 126 | except: 127 | # XML Parsing error in the input 128 | type, value, traceback = sys.exc_info() 129 | if options.host_language == GENERIC_XML or options.lax == False: 130 | raise RDFaError('Parsing error in input file: "%s"' % value) 131 | 132 | # XML Parsing error in the input 133 | msg = "XHTML Parsing error in input file: %s. Falling back on the HTML5 parser" % value 134 | if options != None and options.warnings: 135 | options.comment_graph.add_warning(msg) 136 | 137 | # in Ivan's original code he reopened the stream if it was from urllib 138 | if isinstance(stream, urllib.addinfourl): 139 | stream = urllib.urlopen(stream.url) 140 | 141 | # Now try to see if and HTML5 parser is an alternative... 142 | try: 143 | from html5lib import HTMLParser, treebuilders 144 | except ImportError: 145 | # no alternative to the XHTML error, because HTML5 parser not available... 146 | msg2 = 'XHTML Parsing error in input file: %s. Though parsing is lax, HTML5 parser not available. Try installing html5lib ' % value 147 | raise RDFaError(msg2) 148 | 149 | parser = HTMLParser(tree=treebuilders.getTreeBuilder("dom")) 150 | parse = parser.parse 151 | try: 152 | dom = parse(stream) 153 | # The host language has changed 154 | options.host_language = HTML5_RDFA 155 | except: 156 | # Well, even the HTML5 parser could not do anything with this... 157 | (type, value, traceback) = sys.exc_info() 158 | msg2 = 'Parsing error in input file as HTML5: "%s"' % value 159 | msg3 = msg + '\n' + msg2 160 | raise RDFaError, msg3 161 | 162 | return dom 163 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/rdfa/embeddedrdf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Extracting possible embedded RDF/XML content from the file and parse it separately into the Graph. This is used, for example 4 | by U{SVG 1.2 Tiny}. 5 | 6 | @author: U{Ivan Herman} 7 | @license: This software is available for use under the 8 | U{W3C® SOFTWARE NOTICE AND LICENSE} 9 | @contact: Ivan Herman, ivan@w3.org 10 | """ 11 | 12 | from StringIO import StringIO 13 | 14 | def handle_embeddedRDF(node, graph, state): 15 | """ 16 | Check if the node is the top level rdf element for RDF/XML. If so, the content is parsed and added to the target graph. Note that if an separate 17 | base is defined in the state, the C{xml:base} attribute will be added to the C{rdf} node before parsing. 18 | @param node: a DOM node for the top level xml element 19 | @param graph: target rdf graph 20 | @type graph: RDFLib's Graph object instance 21 | @param state: the inherited state (namespaces, lang, etc) 22 | @type state: L{State.ExecutionContext} 23 | @return: whether an RDF/XML content has been detected or not. If TRUE, the RDFa processing should not occur on the node and its descendents. 24 | @rtype: Boolean 25 | 26 | """ 27 | if node.localName == "RDF" and node.namespaceURI == "http://www.w3.org/1999/02/22-rdf-syntax-ns#": 28 | node.setAttribute("xml:base",state.base) 29 | rdf = StringIO(node.toxml()) 30 | graph.parse(rdf) 31 | return True 32 | else: 33 | return False 34 | 35 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/rdfa/options.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | 4 | Options class: collect the possible options that govern the parsing possibilities. It also includes a reference and 5 | handling of the extra Graph for warnings, informations, errors. 6 | 7 | 8 | @summary: RDFa parser (distiller) 9 | @requires: U{RDFLib} 10 | @requires: U{html5lib} for the HTML5 parsing; note possible dependecies on Python's version on the project's web site 11 | @organization: U{World Wide Web Consortium} 12 | @author: U{Ivan Herman} 13 | @license: This software is available for use under the 14 | U{W3C® SOFTWARE NOTICE AND LICENSE} 15 | 16 | """ 17 | 18 | import sys 19 | from rdflib.graph import Graph 20 | from rdflib.term import BNode, Literal, URIRef 21 | from rdflib.namespace import Namespace 22 | 23 | DIST_URI = "http://www.w3.org/2007/08/pyRdfa/distiller" 24 | DIST_NS = DIST_URI + '#' 25 | 26 | ns_errors = Namespace(DIST_NS) 27 | distillerURI = URIRef(DIST_URI) 28 | 29 | WARNING = 'warning' 30 | ERROR = 'error' 31 | INFO = 'info' 32 | DEBUG = 'debug' 33 | 34 | _message_properties = { 35 | WARNING: ns_errors["warning"], 36 | ERROR: ns_errors["error"], 37 | INFO: ns_errors["information"], 38 | DEBUG: ns_errors["debug"] 39 | } 40 | 41 | def _add_to_comment_graph(graph, msg, prop, uri): 42 | """ 43 | Add a distiller message to the graph. 44 | 45 | @param graph: RDFLib Graph 46 | @param msg: message of an exception 47 | @type msg: RDFLIb Literal 48 | @param prop: the property to be used 49 | @type prop: string, must be one of 'warning', 'error', 'info', 'debug' 50 | @param uri: the top URI used to invoke the distiller 51 | @type uri: URIRef 52 | """ 53 | bnode = BNode() 54 | graph.add((distillerURI, _message_properties[prop], bnode)) 55 | graph.add((bnode, ns_errors["onURI"], uri)) 56 | graph.add((bnode, ns_errors["message"], msg)) 57 | 58 | 59 | class CommentGraph(object): 60 | """Class to handle the 'comment graph', ie, the (RDF) Graph containing the warnings, 61 | error messages, and informational messages. 62 | """ 63 | def __init__(self, warnings = False): 64 | """ 65 | @param warnings: whether a graph should effectively be set up, or whether this 66 | should just be an empty shell for the various calls to work (without effect) 67 | """ 68 | if warnings: 69 | self.graph = Graph() 70 | else: 71 | self.graph = None 72 | self.accumulated_literals = [] 73 | self.baseURI = None 74 | 75 | def _add_triple(self, msg, prop): 76 | obj = Literal(msg) 77 | if self.baseURI == None: 78 | self.accumulated_literals.append((obj,prop)) 79 | elif self.graph != None: 80 | _add_to_comment_graph(self.graph, obj, prop, self.baseURI) 81 | 82 | def set_base_URI(self, URI): 83 | """Set the base URI for the comment triples. 84 | 85 | Note that this method I{must} be called at some point to complete the triples. Without it the triples 86 | added via L{add_warning}, L{add_info}, etc, will not be added to the final graph. 87 | 88 | @param URI: URIRef for the subject of the comments 89 | """ 90 | self.baseURI = URI 91 | if self.graph != None: 92 | for obj, prop in self.accumulated_literals: 93 | _add_to_comment_graph(self.graph, obj, prop, self.baseURI) 94 | self.accumulated_literals = [] 95 | 96 | def add_warning(self, txt): 97 | """Add a warning. A comment triplet is added to the separate "warning" graph. 98 | @param txt: the warning text. It will be preceded by the string "==== pyRdfa Warning ==== " 99 | """ 100 | self._add_triple(txt, WARNING) 101 | 102 | def add_info(self, txt): 103 | """Add an informational comment. A comment triplet is added to the separate "warning" graph. 104 | @param txt: the information text. It will be preceded by the string "==== pyRdfa information ==== " 105 | """ 106 | self._add_triple(txt, INFO) 107 | 108 | def add_error(self, txt): 109 | """Add an error comment. A comment triplet is added to the separate "warning" graph. 110 | @param txt: the information text. It will be preceded by the string "==== pyRdfa information ==== " 111 | """ 112 | self._add_triple(txt, ERROR) 113 | 114 | def _add_debug(self, txt): 115 | self._add_triple(txt, DEBUG) 116 | 117 | 118 | GENERIC_XML = 0 119 | XHTML_RDFA = 1 120 | HTML5_RDFA = 2 121 | 122 | class Options(object): 123 | """Settable options. An instance of this class is stored in 124 | the L{execution context} of the parser. 125 | 126 | @ivar space_preserve: whether plain literals should preserve spaces at output or not 127 | @type space_preserve: Boolean 128 | @ivar comment_graph: Graph for the storage of warnings 129 | @type comment_graph: L{CommentGraph} 130 | @ivar warnings: whether warnings should be generated or not 131 | @type warnings: Boolean 132 | @ivar transformers: extra transformers 133 | @type transformers: list 134 | @type host_language: the host language for the RDFa attributes. Default is XHTML_RDFA, but it can be GENERIC_XML and HTML5_RDFA 135 | @ivar host_language: integer (logically: an enumeration) 136 | @ivar lax: whether a 'lax' parsing of XHTML (ie, HTML5) is allowed. This means that the value of the host language might change run time 137 | @type lax: Boolean 138 | """ 139 | def __init__(self, warnings=False, space_preserve=True, transformers=[], xhtml=True, lax=False): 140 | """ 141 | @param space_preserve: whether plain literals should preserve spaces at output or not 142 | @type space_preserve: Boolean 143 | @param warnings: whether warnings should be generated or not 144 | @type warnings: Boolean 145 | @param transformers: extra transformers 146 | @type transformers: list 147 | @param xhtml: initial value for the host language. If True, the value is set to XHTML_RDFA. Note that run-time the class variable might be set ot HTML5_RDFA, depending on the value of the lax flag and the result of parsing. 148 | @type xhtml: Booelan 149 | @param lax: whether a 'lax' parsing of XHTML (ie, HTML5) is allowed. This means that the value of the host language might change run time 150 | @type lax: Boolean 151 | """ 152 | self.space_preserve = space_preserve 153 | self.transformers = transformers 154 | self.comment_graph = CommentGraph(warnings) 155 | self.warnings = warnings 156 | self.lax = lax 157 | if xhtml: 158 | self.host_language = XHTML_RDFA 159 | else: 160 | self.host_language = GENERIC_XML 161 | 162 | def __str__(self): 163 | retval = """Current options: 164 | space_preserve : %s 165 | warnings : %s 166 | lax parsing : %s 167 | host language : %s 168 | """ 169 | return retval % (self.space_preserve, self.warnings, self.lax, self.host_language) 170 | 171 | 172 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/rdfa/transform/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mhausenblas/web.instata/ccebeb2d51c42e849152363ce75b22def579f4a3/lib/rdflib/plugins/parsers/rdfa/transform/__init__.py -------------------------------------------------------------------------------- /lib/rdflib/plugins/parsers/rdfa/transform/headabout.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Simple transfomer: the C{@about=""} is added to the C{} and C{} elements (unless something is already there). 4 | Note that this transformer is always invoked by the parser because this behaviour is mandated by the RDFa syntax. 5 | 6 | @summary: Add a top "about" to and 7 | @requires: U{RDFLib package} 8 | @organization: U{World Wide Web Consortium} 9 | @author: U{Ivan Herman} 10 | @license: This software is available for use under the 11 | U{W3C® SOFTWARE NOTICE AND LICENSE} 12 | @contact: Ivan Herman, ivan@w3.org 13 | """ 14 | 15 | def head_about_transform(html, options): 16 | """ 17 | @param html: a DOM node for the top level html element 18 | @param options: invocation options 19 | @type options: L{Options} 20 | """ 21 | for top in html.getElementsByTagName("head"): 22 | if not top.hasAttribute("about"): 23 | top.setAttribute("about", "") 24 | for top in html.getElementsByTagName("body"): 25 | if not top.hasAttribute("about"): 26 | top.setAttribute("about", "") 27 | 28 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/serializers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mhausenblas/web.instata/ccebeb2d51c42e849152363ce75b22def579f4a3/lib/rdflib/plugins/serializers/__init__.py -------------------------------------------------------------------------------- /lib/rdflib/plugins/serializers/n3.py: -------------------------------------------------------------------------------- 1 | """ 2 | Notation 3 (N3) RDF graph serializer for RDFLib. 3 | """ 4 | from rdflib.graph import Graph 5 | from rdflib.namespace import Namespace, OWL 6 | from rdflib.plugins.serializers.turtle import (TurtleSerializer, 7 | SUBJECT, VERB, OBJECT) 8 | 9 | 10 | SWAP_LOG = Namespace("http://www.w3.org/2000/10/swap/log#") 11 | 12 | 13 | class N3Serializer(TurtleSerializer): 14 | 15 | short_name = "n3" 16 | 17 | def __init__(self, store, parent=None): 18 | super(N3Serializer, self).__init__(store) 19 | self.keywords.update({ 20 | OWL.sameAs: '=', 21 | SWAP_LOG.implies: '=>' 22 | }) 23 | self.parent = parent 24 | 25 | def reset(self): 26 | super(N3Serializer, self).reset() 27 | self._stores = {} 28 | 29 | def subjectDone(self, subject): 30 | super(N3Serializer, self).subjectDone(subject) 31 | if self.parent: 32 | self.parent.subjectDone(subject) 33 | 34 | def isDone(self, subject): 35 | return (super(N3Serializer, self).isDone(subject) 36 | and (not self.parent or self.parent.isDone(subject))) 37 | 38 | def startDocument(self): 39 | super(N3Serializer, self).startDocument() 40 | #if not isinstance(self.store, N3Store): 41 | # return 42 | # 43 | #all_list = [self.label(var) for var in 44 | # self.store.get_universals(recurse=False)] 45 | #all_list.sort() 46 | #some_list = [self.label(var) for var in 47 | # self.store.get_existentials(recurse=False)] 48 | #some_list.sort() 49 | # 50 | #for var in all_list: 51 | # self.write('\n'+self.indent()+'@forAll %s. '%var) 52 | #for var in some_list: 53 | # self.write('\n'+self.indent()+'@forSome %s. '%var) 54 | # 55 | #if (len(all_list) + len(some_list)) > 0: 56 | # self.write('\n') 57 | 58 | def endDocument(self): 59 | if not self.parent: 60 | super(N3Serializer, self).endDocument() 61 | 62 | def indent(self, modifier=0): 63 | indent = super(N3Serializer, self).indent(modifier) 64 | if self.parent is not None: 65 | indent += self.parent.indent()#modifier) 66 | return indent 67 | 68 | def preprocessTriple(self, triple): 69 | super(N3Serializer, self).preprocessTriple(triple) 70 | if isinstance(triple[0], Graph): 71 | for t in triple[0]: 72 | self.preprocessTriple(t) 73 | if isinstance(triple[2], Graph): 74 | for t in triple[2]: 75 | self.preprocessTriple(t) 76 | 77 | def getQName(self, uri, gen_prefix=True): 78 | qname = None 79 | if self.parent is not None: 80 | qname = self.parent.getQName(uri, gen_prefix) 81 | if qname is None: 82 | qname = super(N3Serializer, self).getQName(uri, gen_prefix) 83 | return qname 84 | 85 | def statement(self, subject): 86 | self.subjectDone(subject) 87 | properties = self.buildPredicateHash(subject) 88 | if len(properties) == 0: 89 | return False 90 | return (self.s_clause(subject) 91 | or super(N3Serializer, self).statement(subject)) 92 | 93 | def path(self, node, position, newline=False): 94 | if not self.p_clause(node, position): 95 | super(N3Serializer, self).path(node, position, newline) 96 | 97 | def s_clause(self, subject): 98 | if isinstance(subject, Graph): 99 | self.write('\n'+self.indent()) 100 | self.p_clause(subject, SUBJECT) 101 | self.predicateList(subject) 102 | self.write(' .') 103 | return True 104 | else: 105 | return False 106 | 107 | def p_clause(self, node, position): 108 | if isinstance(node, Graph): 109 | self.subjectDone(node) 110 | if position is OBJECT: 111 | self.write(' ') 112 | self.write('{') 113 | self.depth += 1 114 | serializer = N3Serializer(node, parent=self) 115 | serializer.serialize(self.stream) 116 | self.depth -= 1 117 | self.write(self.indent()+'}') 118 | return True 119 | else: 120 | return False 121 | 122 | 123 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/serializers/nt.py: -------------------------------------------------------------------------------- 1 | """ 2 | N-Triples RDF graph serializer for RDFLib. 3 | See for details about the 4 | format. 5 | """ 6 | from rdflib.serializer import Serializer 7 | import warnings 8 | 9 | 10 | class NTSerializer(Serializer): 11 | """ 12 | Serializes RDF graphs to NTriples format. 13 | """ 14 | 15 | def serialize(self, stream, base=None, encoding=None, **args): 16 | if base is not None: 17 | warnings.warn("NTSerializer does not support base.") 18 | if encoding is not None: 19 | warnings.warn("NTSerializer does not use custom encoding.") 20 | encoding = self.encoding 21 | for triple in self.store: 22 | stream.write(_nt_row(triple).encode(encoding, "replace")) 23 | stream.write("\n") 24 | 25 | 26 | def _nt_row(triple): 27 | return u"%s %s %s .\n" % (triple[0].n3(), 28 | triple[1].n3(), 29 | _xmlcharref_encode(triple[2].n3())) 30 | 31 | # from 32 | def _xmlcharref_encode(unicode_data, encoding="ascii"): 33 | """Emulate Python 2.3's 'xmlcharrefreplace' encoding error handler.""" 34 | chars = [] 35 | 36 | # nothing to do about xmlchars, but replace newlines with escapes: 37 | unicode_data=unicode_data.replace("\n","\\n") 38 | if unicode_data.startswith('"""'): 39 | unicode_data = unicode_data.replace('"""', '"') 40 | 41 | # Step through the unicode_data string one character at a time in 42 | # order to catch unencodable characters: 43 | for char in unicode_data: 44 | try: 45 | chars.append(char.encode(encoding, 'strict')) 46 | except UnicodeError: 47 | chars.append('\u%04X' % ord(char) if ord(char) <= 0xFFFF else '\U%08X' % ord(char)) 48 | 49 | return ''.join(chars) 50 | 51 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/serializers/trix.py: -------------------------------------------------------------------------------- 1 | from rdflib.serializer import Serializer 2 | from rdflib.plugins.serializers.xmlwriter import XMLWriter 3 | 4 | from rdflib.term import URIRef, Literal, BNode 5 | from rdflib.namespace import Namespace 6 | 7 | from rdflib.graph import Graph, ConjunctiveGraph 8 | 9 | 10 | ## TODO: MOve this somewhere central 11 | TRIXNS=Namespace("http://www.w3.org/2004/03/trix/trix-1/") 12 | XMLNS=Namespace("http://www.w3.org/XML/1998/namespace") 13 | 14 | class TriXSerializer(Serializer): 15 | def __init__(self, store): 16 | super(TriXSerializer, self).__init__(store) 17 | 18 | def serialize(self, stream, base=None, encoding=None, **args): 19 | 20 | nm=self.store.namespace_manager 21 | 22 | self.writer=XMLWriter(stream, nm, encoding, extra_ns={"": TRIXNS}) 23 | 24 | self.writer.push(TRIXNS[u"TriX"]) 25 | self.writer.namespaces() 26 | 27 | if isinstance(self.store, ConjunctiveGraph): 28 | for subgraph in self.store.contexts(): 29 | self._writeGraph(subgraph) 30 | elif isinstance(self.store, Graph): 31 | self._writeGraph(self.store) 32 | else: 33 | raise Exception("Unknown graph type: "+type(self.store)) 34 | 35 | self.writer.pop() 36 | stream.write("\n") 37 | 38 | 39 | def _writeGraph(self, graph): 40 | self.writer.push(TRIXNS[u"graph"]) 41 | if isinstance(graph.identifier, URIRef): 42 | self.writer.element(TRIXNS[u"uri"], content=unicode(graph.identifier)) 43 | 44 | for triple in graph.triples((None,None,None)): 45 | self._writeTriple(triple) 46 | self.writer.pop() 47 | 48 | def _writeTriple(self, triple): 49 | self.writer.push(TRIXNS[u"triple"]) 50 | for component in triple: 51 | if isinstance(component, URIRef): 52 | self.writer.element(TRIXNS[u"uri"], 53 | content=unicode(component)) 54 | elif isinstance(component, BNode): 55 | self.writer.element(TRIXNS[u"id"], 56 | content=unicode(component)) 57 | elif isinstance(component, Literal): 58 | if component.datatype: 59 | self.writer.element(TRIXNS[u"typedLiteral"], 60 | content=unicode(component), 61 | attributes={ TRIXNS[u"datatype"]: unicode(component.datatype) }) 62 | elif component.language: 63 | self.writer.element(TRIXNS[u"plainLiteral"], 64 | content=unicode(component), 65 | attributes={ XMLNS[u"lang"]: unicode(component.language) }) 66 | else: 67 | self.writer.element(TRIXNS[u"plainLiteral"], 68 | content=unicode(component)) 69 | self.writer.pop() 70 | -------------------------------------------------------------------------------- /lib/rdflib/plugins/serializers/xmlwriter.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | from xml.sax.saxutils import quoteattr, escape 3 | 4 | 5 | class XMLWriter(object): 6 | def __init__(self, stream, namespace_manager, encoding=None, decl=1, extra_ns={}): 7 | encoding = encoding or 'utf-8' 8 | encoder, decoder, stream_reader, stream_writer = codecs.lookup(encoding) 9 | self.stream = stream = stream_writer(stream) 10 | if decl: 11 | stream.write('' % encoding) 12 | self.element_stack = [] 13 | self.nm = namespace_manager 14 | self.extra_ns=extra_ns 15 | self.closed = True 16 | 17 | def __get_indent(self): 18 | return " " * len(self.element_stack) 19 | indent = property(__get_indent) 20 | 21 | def __close_start_tag(self): 22 | if not self.closed: # TODO: 23 | self.closed = True 24 | self.stream.write(">") 25 | 26 | def push(self, uri): 27 | self.__close_start_tag() 28 | write = self.stream.write 29 | write("\n") 30 | write(self.indent) 31 | write("<%s" % self.qname(uri)) 32 | self.element_stack.append(uri) 33 | self.closed = False 34 | self.parent = False 35 | 36 | def pop(self, uri=None): 37 | top = self.element_stack.pop() 38 | if uri: 39 | assert uri == top 40 | write = self.stream.write 41 | if not self.closed: 42 | self.closed = True 43 | write("/>") 44 | else: 45 | if self.parent: 46 | write("\n") 47 | write(self.indent) 48 | write("" % self.qname(top)) 49 | self.parent = True 50 | 51 | def element(self, uri, content, attributes={}): 52 | """Utility method for adding a complete simple element""" 53 | self.push(uri) 54 | for k, v in attributes.iteritems(): 55 | self.attribute(k,v) 56 | self.text(content) 57 | self.pop() 58 | 59 | def namespaces(self, namespaces=None): 60 | if not namespaces: 61 | namespaces=self.nm.namespaces() 62 | 63 | write = self.stream.write 64 | write("\n") 65 | for prefix, namespace in namespaces: 66 | if prefix: 67 | write(' xmlns:%s="%s"\n' % (prefix, namespace)) 68 | else: 69 | write(' xmlns="%s"\n' % namespace) 70 | 71 | for prefix, namespace in self.extra_ns.items(): 72 | if prefix: 73 | write(' xmlns:%s="%s"\n' % (prefix, namespace)) 74 | else: 75 | write(' xmlns="%s"\n' % namespace) 76 | 77 | 78 | def attribute(self, uri, value): 79 | write = self.stream.write 80 | write(" %s=%s" % (self.qname(uri), quoteattr(value))) 81 | 82 | def text(self, text): 83 | self.__close_start_tag() 84 | if "<" in text and ">" in text and not "]]>" in text: 85 | self.stream.write("") 88 | else: 89 | self.stream.write(escape(text)) 90 | 91 | def qname(self,uri): 92 | """Compute qname for a uri using our extra namespaces, 93 | or the given namespace manager""" 94 | 95 | for pre,ns in self.extra_ns.items(): 96 | if uri.startswith(ns): 97 | if pre!="": 98 | return ":".join(pre,uri[len(ns):]) 99 | else: 100 | return uri[len(ns):] 101 | 102 | return self.nm.qname(uri) 103 | -------------------------------------------------------------------------------- /lib/rdflib/query.py: -------------------------------------------------------------------------------- 1 | """ 2 | Query plugin interface. 3 | 4 | This module is useful for those wanting to write a query processor 5 | that can plugin to rdf. If you are wanting to execute a query you 6 | likely want to do so through the Graph class query method. 7 | 8 | """ 9 | 10 | 11 | class Processor(object): 12 | 13 | def __init__(self, graph): 14 | pass 15 | 16 | def query(self, strOrQuery, initBindings={}, initNs={}, DEBUG=False): 17 | pass 18 | 19 | 20 | class Result(object): 21 | """ 22 | A common class for representing query result in a variety of formats, namely: 23 | 24 | xml : as an XML string using the XML result format of the query language 25 | python: as Python objects 26 | json : as JSON 27 | """ 28 | def __init__(self, pythonResult): 29 | self.rt = pythonResult 30 | 31 | def serialize(self, format='xml'): 32 | pass 33 | -------------------------------------------------------------------------------- /lib/rdflib/serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializer plugin interface. 3 | 4 | This module is useful for those wanting to write a serializer that can 5 | plugin to rdflib. If you are wanting to invoke a serializer you likely 6 | want to do so through the Graph class serialize method. 7 | 8 | TODO: info for how to write a serializer that can plugin to rdflib. See also rdflib.plugin 9 | 10 | """ 11 | 12 | from rdflib.term import URIRef 13 | 14 | 15 | class Serializer(object): 16 | 17 | def __init__(self, store): 18 | self.store = store 19 | self.encoding = "UTF-8" 20 | self.base = None 21 | 22 | def serialize(self, stream, base=None, encoding=None, **args): 23 | """Abstract method""" 24 | 25 | def relativize(self, uri): 26 | base = self.base 27 | if base is not None and uri.startswith(base): 28 | uri = URIRef(uri.replace(base, "", 1)) 29 | return uri 30 | 31 | -------------------------------------------------------------------------------- /lib/rdflib/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some utility functions. 3 | 4 | TODO... 5 | 6 | """ 7 | 8 | from string import rsplit 9 | 10 | from rdflib.term import URIRef 11 | from rdflib.term import BNode 12 | from rdflib.term import Literal 13 | from rdflib.graph import Graph, QuotedGraph 14 | 15 | from rdflib.exceptions import SubjectTypeError, PredicateTypeError, ObjectTypeError, ContextTypeError 16 | 17 | 18 | def list2set(seq): 19 | seen = set() 20 | return [ x for x in seq if x not in seen and not seen.add(x)] 21 | 22 | def first(seq): 23 | for result in seq: 24 | return result 25 | return None 26 | 27 | def uniq(sequence, strip=0): 28 | """removes duplicate strings from the sequence.""" 29 | set = {} 30 | if strip: 31 | map(lambda val, default: set.__setitem__(val.strip(), default), 32 | sequence, []) 33 | else: 34 | map(set.__setitem__, sequence, []) 35 | return set.keys() 36 | 37 | def more_than(sequence, number): 38 | "Returns 1 if sequence has more items than number and 0 if not." 39 | i = 0 40 | for item in sequence: 41 | i += 1 42 | if i > number: 43 | return 1 44 | return 0 45 | 46 | def term(str, default=None): 47 | """See also from_n3""" 48 | if not str: 49 | return default 50 | elif str.startswith("<") and str.endswith(">"): 51 | return URIRef(str[1:-1]) 52 | elif str.startswith('"') and str.endswith('"'): 53 | return Literal(str[1:-1]) 54 | elif str.startswith("_"): 55 | return BNode(str) 56 | else: 57 | msg = "Unknown Term Syntax: '%s'" % str 58 | raise Exception(msg) 59 | 60 | 61 | 62 | from time import mktime, time, gmtime, localtime, timezone, altzone, daylight 63 | 64 | def date_time(t=None, local_time_zone=False): 65 | """http://www.w3.org/TR/NOTE-datetime ex: 1997-07-16T19:20:30Z 66 | 67 | >>> date_time(1126482850) 68 | '2005-09-11T23:54:10Z' 69 | 70 | @@ this will change depending on where it is run 71 | #>>> date_time(1126482850, local_time_zone=True) 72 | #'2005-09-11T19:54:10-04:00' 73 | 74 | >>> date_time(1) 75 | '1970-01-01T00:00:01Z' 76 | 77 | >>> date_time(0) 78 | '1970-01-01T00:00:00Z' 79 | """ 80 | if t is None: 81 | t = time() 82 | 83 | if local_time_zone: 84 | time_tuple = localtime(t) 85 | if time_tuple[8]: 86 | tz_mins = altzone // 60 87 | else: 88 | tz_mins = timezone // 60 89 | tzd = "-%02d:%02d" % (tz_mins // 60, tz_mins % 60) 90 | else: 91 | time_tuple = gmtime(t) 92 | tzd = "Z" 93 | 94 | year, month, day, hh, mm, ss, wd, y, z = time_tuple 95 | s = "%0004d-%02d-%02dT%02d:%02d:%02d%s" % ( year, month, day, hh, mm, ss, tzd) 96 | return s 97 | 98 | def parse_date_time(val): 99 | """always returns seconds in UTC 100 | 101 | # tests are written like this to make any errors easier to understand 102 | >>> parse_date_time('2005-09-11T23:54:10Z') - 1126482850.0 103 | 0.0 104 | 105 | >>> parse_date_time('2005-09-11T16:54:10-07:00') - 1126482850.0 106 | 0.0 107 | 108 | >>> parse_date_time('1970-01-01T00:00:01Z') - 1.0 109 | 0.0 110 | 111 | >>> parse_date_time('1970-01-01T00:00:00Z') - 0.0 112 | 0.0 113 | >>> parse_date_time("2005-09-05T10:42:00") - 1125916920.0 114 | 0.0 115 | """ 116 | 117 | if "T" not in val: 118 | val += "T00:00:00Z" 119 | 120 | ymd, time = val.split("T") 121 | hms, tz_str = time[0:8], time[8:] 122 | 123 | if not tz_str or tz_str=="Z": 124 | time = time[:-1] 125 | tz_offset = 0 126 | else: 127 | signed_hrs = int(tz_str[:3]) 128 | mins = int(tz_str[4:6]) 129 | secs = (cmp(signed_hrs, 0) * mins + signed_hrs * 60) * 60 130 | tz_offset = -secs 131 | 132 | year, month, day = ymd.split("-") 133 | hour, minute, second = hms.split(":") 134 | 135 | t = mktime((int(year), int(month), int(day), int(hour), 136 | int(minute), int(second), 0, 0, 0)) 137 | t = t - timezone + tz_offset 138 | return t 139 | 140 | def from_n3(s, default=None, backend=None): 141 | """ Creates the Identifier corresponding to the given n3 string. WARNING: untested, may contain bugs. TODO: add test cases.""" 142 | if not s: 143 | return default 144 | if s.startswith('<'): 145 | return URIRef(s[1:-1]) 146 | elif s.startswith('"'): 147 | # TODO: would a regex be faster? 148 | value, rest = rsplit(s, '"', 1) 149 | value = value[1:] # strip leading quote 150 | if rest.startswith("@"): 151 | if "^^" in rest: 152 | language, rest = rsplit(rest, '^^', 1) 153 | language = language[1:] # strip leading at sign 154 | else: 155 | language = rest[1:] # strip leading at sign 156 | rest = '' 157 | else: 158 | language = None 159 | if rest.startswith("^^"): 160 | datatype = rest[3:-1] 161 | else: 162 | datatype = None 163 | value = value.replace('\\"', '"').replace('\\\\', '\\').decode("unicode-escape") 164 | return Literal(value, language, datatype) 165 | elif s.startswith('{'): 166 | identifier = from_n3(s[1:-1]) 167 | return QuotedGraph(backend, identifier) 168 | elif s.startswith('['): 169 | identifier = from_n3(s[1:-1]) 170 | return Graph(backend, identifier) 171 | else: 172 | if s.startswith("_:"): 173 | return BNode(s[2:]) 174 | else: 175 | return BNode(s) 176 | 177 | def check_context(c): 178 | if not (isinstance(c, URIRef) or \ 179 | isinstance(c, BNode)): 180 | raise ContextTypeError("%s:%s" % (c, type(c))) 181 | 182 | def check_subject(s): 183 | """ Test that s is a valid subject identifier.""" 184 | if not (isinstance(s, URIRef) or isinstance(s, BNode)): 185 | raise SubjectTypeError(s) 186 | 187 | def check_predicate(p): 188 | """ Test that p is a valid predicate identifier.""" 189 | if not isinstance(p, URIRef): 190 | raise PredicateTypeError(p) 191 | 192 | def check_object(o): 193 | """ Test that o is a valid object identifier.""" 194 | if not (isinstance(o, URIRef) or \ 195 | isinstance(o, Literal) or \ 196 | isinstance(o, BNode)): 197 | raise ObjectTypeError(o) 198 | 199 | def check_statement((s, p, o)): 200 | if not (isinstance(s, URIRef) or isinstance(s, BNode)): 201 | raise SubjectTypeError(s) 202 | 203 | if not isinstance(p, URIRef): 204 | raise PredicateTypeError(p) 205 | 206 | if not (isinstance(o, URIRef) or \ 207 | isinstance(o, Literal) or \ 208 | isinstance(o, BNode)): 209 | raise ObjectTypeError(o) 210 | 211 | def check_pattern((s, p, o)): 212 | if s and not (isinstance(s, URIRef) or isinstance(s, BNode)): 213 | raise SubjectTypeError(s) 214 | 215 | if p and not isinstance(p, URIRef): 216 | raise PredicateTypeError(p) 217 | 218 | if o and not (isinstance(o, URIRef) or \ 219 | isinstance(o, Literal) or \ 220 | isinstance(o, BNode)): 221 | raise ObjectTypeError(o) 222 | 223 | def graph_to_dot(graph, dot): 224 | """ Turns graph into dot (graphviz graph drawing format) using pydot. """ 225 | import pydot 226 | nodes = {} 227 | for s, o in graph.subject_objects(): 228 | for i in s,o: 229 | if i not in nodes.keys(): 230 | nodes[i] = i 231 | for s, p, o in graph.triples((None,None,None)): 232 | dot.add_edge(pydot.Edge(nodes[s], nodes[o], label=p)) 233 | 234 | 235 | if __name__ == "__main__": 236 | # try to make the tests work outside of the time zone they were written in 237 | #import os, time 238 | #os.environ['TZ'] = 'US/Pacific' 239 | #try: 240 | # time.tzset() 241 | #except AttributeError, e: 242 | # print e 243 | #pass 244 | # tzset missing! see 245 | # http://mail.python.org/pipermail/python-dev/2003-April/034480.html 246 | import doctest 247 | doctest.testmod() 248 | -------------------------------------------------------------------------------- /lib/rdflib_schemaorg_csv.py: -------------------------------------------------------------------------------- 1 | """ 2 | This rdflib plugin (based on Ed's https://github.com/edsu/rdflib-microdata/) lets you parse CSV 3 | files that use Schema.org column headers into an RDF graph. You shouldn't have to use this module 4 | directly, since it's a plugin. You'll just want to: 5 | 6 | >>> import rdflib 7 | >>> import rdflib_schemaorg_csv 8 | >>> g = rdflib.Graph() 9 | >>> g.parse("test.csv", format="schemaorg_csv") 10 | >>> print g.serialize() 11 | """ 12 | 13 | import csv 14 | import StringIO 15 | import sys 16 | from rdflib import URIRef, Literal, BNode, Namespace, RDF 17 | from rdflib.plugin import register 18 | from rdflib.parser import Parser 19 | 20 | register("schemaorg_csv", Parser, "rdflib_schemaorg_csv", "SchemaOrgCSVParser") 21 | 22 | class SchemaOrgCSVParser(Parser): 23 | NAMESPACES = { 24 | 'schema' : Namespace('http://schema.org/'), 25 | 'scsv' : Namespace('http://purl.org/NET/schema-org-csv#'), 26 | 'dcterms' : Namespace('http://purl.org/dc/terms/') 27 | } 28 | 29 | def parse(self, source, sink, **kwargs): 30 | """ 31 | Pass in a file or file-like object containing CSV with Schema.org 32 | column headers and populate the sink graph with triples. 33 | """ 34 | row_num = 1 35 | fURI = kwargs.get("csv_file_URI", "") 36 | self._add_table(fURI, sink) 37 | try: 38 | f = source.getByteStream() 39 | rows = csv.reader(f, delimiter=',', quoting=csv.QUOTE_ALL) 40 | for row in rows: 41 | if row_num == 1: 42 | columns = self._add_header(fURI, row, row_num, sink) 43 | else: 44 | self._add_row(fURI, columns, row, row_num, sink) 45 | row_num = row_num + 1 46 | f.close() 47 | except csv.Error, e: 48 | sys.exit('%s' %e) 49 | 50 | def _add_table(self, fURI, sink): 51 | t = URIRef(fURI + '#table') 52 | sink.add((t, RDF.type, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['Table']))) 53 | sink.add((t, SchemaOrgCSVParser.NAMESPACES['dcterms']['source'], URIRef(fURI))) 54 | sink.add((t, SchemaOrgCSVParser.NAMESPACES['dcterms']['title'], Literal(fURI.split('/')[-1]))) 55 | 56 | def _add_header(self, fURI, row, row_num, sink): 57 | t = URIRef(fURI + '#table') 58 | r = URIRef(fURI + '#row:' + str(row_num)) 59 | # row-level: 60 | sink.add((t, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['row']), r)) 61 | sink.add((r, RDF.type, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['HeaderRow']))) 62 | sink.add((r, SchemaOrgCSVParser.NAMESPACES['dcterms']['title'], Literal('header'))) 63 | # cell-level: 64 | col_num = 1 65 | columns = [] 66 | for cell in row: 67 | c = URIRef(fURI + '#row:' + str(row_num) + ',' + 'col:' + str(col_num)) 68 | sink.add((r, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['cell']), c)) 69 | sink.add((c, SchemaOrgCSVParser.NAMESPACES['dcterms']['title'], Literal(cell))) 70 | columns.append(self._lookup_schemaorg_term(cell)) 71 | col_num = col_num + 1 72 | return columns 73 | 74 | def _add_row(self, fURI, columns, row, row_num, sink): 75 | t = URIRef(fURI + '#table') 76 | r = URIRef(fURI + '#row:' + str(row_num)) 77 | # row-level: 78 | sink.add((t, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['row']), r)) 79 | sink.add((r, RDF.type, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['Row']))) 80 | sink.add((r, SchemaOrgCSVParser.NAMESPACES['dcterms']['title'], Literal('row ' + str(row_num)))) 81 | # cell-level: 82 | col_num = 1 83 | for cell in row: 84 | c = URIRef(fURI + '#row:' + str(row_num) + ',' + 'col:' + str(col_num)) 85 | sink.add((r, URIRef(SchemaOrgCSVParser.NAMESPACES['scsv']['cell']), c)) 86 | sink.add((c, RDF.type, URIRef(columns[col_num - 1]))) 87 | if cell.startswith('http://'): 88 | sink.add((c, RDF.value, URIRef(cell))) 89 | else: 90 | sink.add((c, RDF.value, Literal(cell))) 91 | col_num = col_num + 1 92 | return r 93 | 94 | def _lookup_schemaorg_term(self, cell): 95 | return 'http://schema.org/' + cell # TODO: look up cell value in http://schema.rdfs.org/all-classes.csv -------------------------------------------------------------------------------- /lib/simplejson/scanner.py: -------------------------------------------------------------------------------- 1 | """JSON token scanner 2 | """ 3 | import re 4 | try: 5 | from simplejson._speedups import make_scanner as c_make_scanner 6 | except ImportError: 7 | c_make_scanner = None 8 | 9 | __all__ = ['make_scanner'] 10 | 11 | NUMBER_RE = re.compile( 12 | r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', 13 | (re.VERBOSE | re.MULTILINE | re.DOTALL)) 14 | 15 | def py_make_scanner(context): 16 | parse_object = context.parse_object 17 | parse_array = context.parse_array 18 | parse_string = context.parse_string 19 | match_number = NUMBER_RE.match 20 | encoding = context.encoding 21 | strict = context.strict 22 | parse_float = context.parse_float 23 | parse_int = context.parse_int 24 | parse_constant = context.parse_constant 25 | object_hook = context.object_hook 26 | 27 | def _scan_once(string, idx): 28 | try: 29 | nextchar = string[idx] 30 | except IndexError: 31 | raise StopIteration 32 | 33 | if nextchar == '"': 34 | return parse_string(string, idx + 1, encoding, strict) 35 | elif nextchar == '{': 36 | return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook) 37 | elif nextchar == '[': 38 | return parse_array((string, idx + 1), _scan_once) 39 | elif nextchar == 'n' and string[idx:idx + 4] == 'null': 40 | return None, idx + 4 41 | elif nextchar == 't' and string[idx:idx + 4] == 'true': 42 | return True, idx + 4 43 | elif nextchar == 'f' and string[idx:idx + 5] == 'false': 44 | return False, idx + 5 45 | 46 | m = match_number(string, idx) 47 | if m is not None: 48 | integer, frac, exp = m.groups() 49 | if frac or exp: 50 | res = parse_float(integer + (frac or '') + (exp or '')) 51 | else: 52 | res = parse_int(integer) 53 | return res, m.end() 54 | elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': 55 | return parse_constant('NaN'), idx + 3 56 | elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': 57 | return parse_constant('Infinity'), idx + 8 58 | elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': 59 | return parse_constant('-Infinity'), idx + 9 60 | else: 61 | raise StopIteration 62 | 63 | return _scan_once 64 | 65 | make_scanner = c_make_scanner or py_make_scanner 66 | -------------------------------------------------------------------------------- /lib/simplejson/tool.py: -------------------------------------------------------------------------------- 1 | r"""Command-line tool to validate and pretty-print JSON 2 | 3 | Usage:: 4 | 5 | $ echo '{"json":"obj"}' | python -m simplejson.tool 6 | { 7 | "json": "obj" 8 | } 9 | $ echo '{ 1.2:3.4}' | python -m simplejson.tool 10 | Expecting property name: line 1 column 2 (char 2) 11 | 12 | """ 13 | import sys 14 | import simplejson 15 | 16 | def main(): 17 | if len(sys.argv) == 1: 18 | infile = sys.stdin 19 | outfile = sys.stdout 20 | elif len(sys.argv) == 2: 21 | infile = open(sys.argv[1], 'rb') 22 | outfile = sys.stdout 23 | elif len(sys.argv) == 3: 24 | infile = open(sys.argv[1], 'rb') 25 | outfile = open(sys.argv[2], 'wb') 26 | else: 27 | raise SystemExit(sys.argv[0] + " [infile [outfile]]") 28 | try: 29 | obj = simplejson.load(infile) 30 | except ValueError, e: 31 | raise SystemExit(e) 32 | simplejson.dump(obj, outfile, sort_keys=True, indent=4) 33 | outfile.write('\n') 34 | 35 | 36 | if __name__ == '__main__': 37 | main() 38 | -------------------------------------------------------------------------------- /mappings/sioc-2011-08-01.rdf: -------------------------------------------------------------------------------- 1 | 2 | 12 | 13 | 14 | 15 | Mappings between SIOC and schema.rdfs.org terms 16 | This is a set of mappings betweens the terms defined in the SIOC Ontology (http://sioc-project.org) and the ones defined at schema.rdfs.org. 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 2011-08-01T09:42:47+01:00 74 | 22 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /templates/base.tpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ds_name}} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | {{ds_name}} 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | %for h in theader: 22 | {{str(h[1])}} 23 | %end 24 | 25 | 26 | 27 | 28 | %for rec in trows: 29 | %if 'col:1' in rec[1]: 30 | 31 | {{rec[3]}} 32 | %else: 33 | {{rec[3]}} 34 | %end 35 | %if int(rec[1].split('#')[1].split(',')[1].split(':')[1]) == len(theader): 36 | 37 | %end 38 | %end 39 | 40 | 41 | 42 | 43 | Related terms from other vocabularies: 44 | 45 | %for schemaTerm, match in aliases.items(): 46 | DBpedia: {{schemaTerm.split('/')[-1]}} 47 | %end 48 | 49 | 50 | 51 | 52 | 55 | 56 | 57 | 58 | 65 | -------------------------------------------------------------------------------- /templates/web.instata-style.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 4em 6em 0 4em; 3 | padding: 0; 4 | background: #fff; 5 | font-family: 'Lucida Sans Unicode', 'Lucida Grande', sans-serif; 6 | } 7 | 8 | img { 9 | border: 0px solid white; 10 | } 11 | 12 | p { 13 | text-align: justify; 14 | } 15 | 16 | a { 17 | color: #44A; 18 | text-decoration: none; 19 | } 20 | 21 | a:hover { 22 | text-decoration: underline; 23 | } 24 | 25 | h1 { 26 | font-size: 28pt; 27 | font-weight: bold; 28 | } 29 | 30 | h2 { 31 | font-size: 16pt; 32 | } 33 | 34 | h3 { 35 | font-size: 14pt; 36 | } 37 | 38 | #header { 39 | text-shadow: 2px 2px 4px #333; 40 | } 41 | 42 | #main { 43 | 44 | } 45 | 46 | #instatable { 47 | width: 90%; 48 | margin: 1em 2em 1em 2em; 49 | } 50 | 51 | #instatable th { 52 | cursor: pointer; 53 | } 54 | 55 | #related { 56 | font-size: 80%; 57 | color: #d0d0d0; 58 | } 59 | 60 | #footer { 61 | text-align: center; 62 | margin: 0 auto; 63 | margin-top: 4em; 64 | padding: 0 0 .5em 0; 65 | font-size: 8pt; 66 | color: #666; 67 | } -------------------------------------------------------------------------------- /test/potd_0.csv: -------------------------------------------------------------------------------- 1 | Recipe,name,author,datePublished 2 | bb,"Mom's World Famous Banana Bread","John Smith","May 8, 2009" 3 | mf,"Michael's favourite salad","Michael Hausenblas","July 31, 2011" 4 | t1,"Test 1","Michael Hausenblas","July 31, 2011" 5 | t2,"Test 2","Michael Hausenblas","July 31, 2011" 6 | -------------------------------------------------------------------------------- /test/potd_1.csv: -------------------------------------------------------------------------------- 1 | Park,name 2 | p_sh,"Salthill Park" 3 | p_ss,"South Park" 4 | -------------------------------------------------------------------------------- /web.instata.config: -------------------------------------------------------------------------------- 1 | @prefix dc: . 2 | @prefix xsd: . 3 | @prefix c: <#> . 4 | 5 | c:default-config 6 | # publishing options 7 | c:csv_input "test/potd_0.csv" ; 8 | c:output_base_uri ; 9 | c:schema_matching "dbpedia-2011-07-31.rdf" ; 10 | 11 | # directory and file options 12 | c:templates_dir "templates/" ; 13 | c:mappings_dir "mappings/" ; 14 | c:output_dir "output/" ; 15 | c:base_template "base.tpl" ; 16 | c:base_style_file "web.instata-style.css" ; 17 | 18 | # metadata about the config file 19 | dc:title "The default configuration for web.instata" ; 20 | dc:modified "2011-08-01"^^xsd:date ; 21 | dc:creator ; 22 | . 23 | --------------------------------------------------------------------------------
The table element is not allowed to appear 67 | inside a p element
table
p
It seems the p element has not been closed