├── .gitignore ├── test ├── data │ ├── clustering_kmeans.R │ ├── sklearn_clustering_kmeans.py │ ├── datasets │ │ └── iris.csv │ └── clustering_kmeans.raw.graphml ├── ontology │ ├── Ontology.jl │ ├── rdf │ │ ├── OntologyRDF.jl │ │ ├── WiringRDF.jl │ │ ├── ConceptRDF.jl │ │ └── AnnotationRDF.jl │ ├── data │ │ ├── foobar.json │ │ └── employee.json │ ├── OntologyDBs.jl │ └── OntologyJSON.jl ├── runtests.jl ├── CLI.jl ├── Doctrine.jl └── SemanticEnrichment.jl ├── src ├── extras │ ├── CLI-R.jl │ └── CLI-Python.jl ├── SemanticFlowGraphs.jl ├── Visualization.jl ├── ontology │ ├── Ontology.jl │ ├── rdf │ │ ├── schema │ │ │ ├── wiring.ttl │ │ │ ├── list.ttl │ │ │ ├── annotation.ttl │ │ │ └── concept.ttl │ │ ├── OntologyRDF.jl │ │ ├── ConceptPROV.jl │ │ ├── RDFUtils.jl │ │ ├── ConceptRDF.jl │ │ ├── AnnotationRDF.jl │ │ └── WiringRDF.jl │ ├── OntologyJSON.jl │ └── OntologyDBs.jl ├── RawFlowGraphs.jl ├── Serialization.jl ├── Doctrine.jl ├── SemanticEnrichment.jl └── CLI.jl ├── .github └── workflows │ └── tests.yml ├── Project.toml ├── bin └── flowgraphs.jl ├── README.md └── LICENSE.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vscode 3 | 4 | Manifest.toml 5 | -------------------------------------------------------------------------------- /test/data/clustering_kmeans.R: -------------------------------------------------------------------------------- 1 | iris = read.csv("datasets/iris.csv", stringsAsFactors=FALSE) 2 | iris = iris[, names(iris) != "Species"] 3 | 4 | km = kmeans(iris, 3) 5 | centroids = km$centers 6 | clusters = km$cluster -------------------------------------------------------------------------------- /src/extras/CLI-R.jl: -------------------------------------------------------------------------------- 1 | using .RCall 2 | 3 | @rimport flowgraph as RFlowGraphs 4 | 5 | function record_file(inpath::String, outpath::String, args::Dict, ::Val{:r}) 6 | RFlowGraphs.record_file( 7 | inpath, out=outpath, cwd=dirname(inpath), annotate=true, 8 | annotations=args["annotations"]) 9 | end 10 | -------------------------------------------------------------------------------- /test/data/sklearn_clustering_kmeans.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.cluster import KMeans 3 | 4 | iris = pd.read_csv('datasets/iris.csv') 5 | iris = iris.drop('Species', 1) 6 | 7 | kmeans = KMeans(n_clusters=3).fit(iris.values) 8 | centroids = kmeans.cluster_centers_ 9 | clusters = kmeans.labels_ 10 | -------------------------------------------------------------------------------- /src/extras/CLI-Python.jl: -------------------------------------------------------------------------------- 1 | using Compat 2 | using .PyCall 3 | 4 | const PyFlowGraphs = pyimport("flowgraph.core.record") 5 | const PyAnnotationDBs = pyimport("flowgraph.core.annotation_db") 6 | 7 | function record_file(inpath::String, outpath::String, args::Dict, ::Val{:python}) 8 | if isnothing(args["annotations"]) 9 | db = nothing 10 | else 11 | db = PyAnnotationDBs.AnnotationDB() 12 | db.load_file(abspath(args["annotations"])) 13 | end 14 | PyFlowGraphs.record_script( 15 | inpath, out=outpath, cwd=dirname(inpath), db=db, 16 | graph_outputs=args["graph-outputs"], store_slots=false) 17 | end 18 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | runs-on: ${{ matrix.os }} 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | julia_version: ['1.6'] 10 | os: [ubuntu-latest] 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: "Set up Julia" 14 | uses: julia-actions/setup-julia@latest 15 | with: 16 | version: ${{ matrix.julia_version }} 17 | - name: "Build package" 18 | uses: julia-actions/julia-buildpkg@v1 19 | - name: "Run tests" 20 | uses: julia-actions/julia-runtest@v1 21 | -------------------------------------------------------------------------------- /test/ontology/Ontology.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestOntology 16 | using Test 17 | 18 | @testset "JSON" begin 19 | include("OntologyJSON.jl") 20 | end 21 | 22 | @testset "Database" begin 23 | include("OntologyDBs.jl") 24 | end 25 | 26 | @testset "RDF" begin 27 | include("rdf/OntologyRDF.jl") 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /test/ontology/rdf/OntologyRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestOntologyRDF 16 | using Test 17 | 18 | @testset "Wiring" begin 19 | include("WiringRDF.jl") 20 | end 21 | 22 | @testset "Concept" begin 23 | include("ConceptRDF.jl") 24 | end 25 | 26 | @testset "Annotation" begin 27 | include("AnnotationRDF.jl") 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /test/runtests.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | using Test 16 | 17 | @testset "Doctrine" begin 18 | include("Doctrine.jl") 19 | end 20 | 21 | @testset "Ontology" begin 22 | include("ontology/Ontology.jl") 23 | end 24 | 25 | @testset "SemanticEnrichment" begin 26 | include("SemanticEnrichment.jl") 27 | end 28 | 29 | @testset "CLI" begin 30 | include("CLI.jl") 31 | end 32 | -------------------------------------------------------------------------------- /Project.toml: -------------------------------------------------------------------------------- 1 | name = "SemanticFlowGraphs" 2 | uuid = "ce4a9274-ad72-11e8-1140-c9a2f4ff8ac8" 3 | license = "Apache2" 4 | authors = ["Evan Patterson "] 5 | version = "0.1.0" 6 | 7 | [deps] 8 | ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" 9 | AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f" 10 | Catlab = "134e5e36-593f-5add-ad60-77f754baafbe" 11 | Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" 12 | DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" 13 | DefaultApplication = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85" 14 | HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" 15 | JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" 16 | LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d" 17 | MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5" 18 | Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" 19 | Reexport = "189a3867-3050-52da-a836-e630ba90ab69" 20 | Requires = "ae029012-a4dd-5104-9daa-d747884805df" 21 | Serd = "42699a24-3520-5f41-a549-6a478ed667c2" 22 | Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" 23 | 24 | [compat] 25 | Catlab = "0.12" 26 | Serd = "0.3" 27 | julia = "1.6" 28 | -------------------------------------------------------------------------------- /src/SemanticFlowGraphs.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module SemanticFlowGraphs 16 | using Reexport 17 | 18 | include("Doctrine.jl") 19 | include("ontology/Ontology.jl") 20 | include("RawFlowGraphs.jl") 21 | include("Serialization.jl") 22 | include("Visualization.jl") 23 | include("SemanticEnrichment.jl") 24 | include("CLI.jl") 25 | 26 | @reexport using .Doctrine 27 | @reexport using .Ontology 28 | @reexport using .RawFlowGraphs 29 | @reexport using .Serialization 30 | @reexport using .Visualization 31 | @reexport using .SemanticEnrichment 32 | 33 | end 34 | -------------------------------------------------------------------------------- /src/Visualization.jl: -------------------------------------------------------------------------------- 1 | """ Visualize raw and semantic flow graphs. 2 | """ 3 | module Visualization 4 | 5 | using Compat 6 | 7 | using Catlab.WiringDiagrams, Catlab.Graphics 8 | import Catlab.Graphics.WiringDiagramLayouts: box_label, wire_label 9 | using ..Doctrine 10 | using ..RawFlowGraphs 11 | import ..Serialization: text_label 12 | 13 | # Raw flow graphs 14 | ################# 15 | 16 | # FIXME: These methods use language-specific attributes. Perhaps there should 17 | # be some standardization across languages. 18 | 19 | function box_label(::MIME"text/plain", node::RawNode) 20 | lang = node.language 21 | get_first(lang, ["slot", "qual_name", "function", "kind"], "") 22 | end 23 | 24 | function wire_label(::MIME"text/plain", port::RawPort) 25 | lang = port.language 26 | get_first(lang, ["qual_name", "class"], "") 27 | end 28 | 29 | function get_first(collection, keys, default) 30 | if isempty(keys); return default end 31 | get(collection, splice!(keys, 1)) do 32 | get_first(collection, keys, default) 33 | end 34 | end 35 | 36 | # Semantic flow graphs 37 | ###################### 38 | 39 | box_label(::MIME"text/plain", f::Monocl.Hom) = text_label(f) 40 | wire_label(::MIME"text/plain", A::Monocl.Ob) = text_label(A) 41 | 42 | end 43 | -------------------------------------------------------------------------------- /bin/flowgraphs.jl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env julia 2 | 3 | # Copyright 2018 IBM Corp. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import SemanticFlowGraphs: CLI 18 | 19 | function expand_paths(path::String)::Vector{String} 20 | if isfile(path); [ path ] 21 | elseif isdir(path); readdir(path) 22 | else String[] end 23 | end 24 | 25 | cmds, cmd_args = CLI.parse(ARGS) 26 | 27 | # XXX: Reduce load time by only importing extra packages as needed. 28 | # Julia really needs a better solution to this problem... 29 | if first(cmds) == "record" 30 | paths = expand_paths(cmd_args["path"]) 31 | if any(endswith(path, ".py") for path in paths) 32 | import PyCall 33 | end 34 | if any(endswith(path, ".R") for path in paths) 35 | import RCall 36 | end 37 | end 38 | 39 | CLI.invoke(cmds, cmd_args) 40 | -------------------------------------------------------------------------------- /src/ontology/Ontology.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module Ontology 16 | export Annotation, AnnotationID, ObAnnotation, HomAnnotation 17 | 18 | using AutoHashEquals 19 | using Reexport 20 | 21 | using ..Doctrine 22 | 23 | # Data types 24 | ############ 25 | 26 | """ Semantic annotation of computer program. 27 | 28 | This type is agnostic to the programming language of the computer program. All 29 | language-specific information is stored in the `language` dictionary. 30 | """ 31 | abstract type Annotation end 32 | 33 | """ Unique identifer of annotation. 34 | """ 35 | @auto_hash_equals struct AnnotationID 36 | language::String 37 | package::String 38 | id::String 39 | end 40 | 41 | struct ObAnnotation <: Annotation 42 | name::AnnotationID 43 | language::Dict{Symbol,Any} 44 | definition::Monocl.Ob 45 | slots::Vector{Monocl.Hom} 46 | end 47 | 48 | struct HomAnnotation <: Annotation 49 | name::AnnotationID 50 | language::Dict{Symbol,Any} 51 | definition::Monocl.Hom 52 | end 53 | 54 | # Modules 55 | ######### 56 | 57 | include("OntologyJSON.jl") 58 | include("OntologyDBs.jl") 59 | include("rdf/OntologyRDF.jl") 60 | 61 | @reexport using .OntologyJSON 62 | @reexport using .OntologyDBs 63 | @reexport using .OntologyRDF 64 | 65 | end 66 | -------------------------------------------------------------------------------- /src/ontology/rdf/schema/wiring.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix monocl: . 5 | 6 | monocl:WiringDiagramElement rdf:type owl:Class ; 7 | rdfs:label "Diagram element" ; 8 | rdfs:comment "Any element of a wiring diagram" . 9 | 10 | # Boxes 11 | 12 | monocl:AbstractBox rdf:type owl:Class ; 13 | rdfs:subClassOf monocl:Function, monocl:WiringDiagramElement ; 14 | rdfs:label "Abstract box" ; 15 | rdfs:comment "Any box with inputs and outputs" . 16 | 17 | monocl:Box rdf:type owl:Class ; 18 | rdfs:subClassOf monocl:AbstractBox ; 19 | rdfs:comment "Black box with no internal structure" . 20 | 21 | monocl:WiringDiagram rdf:type owl:Class ; 22 | rdfs:subClassOf monocl:AbstractBox ; 23 | rdfs:label "Wiring diagram" ; 24 | rdfs:comment "Diagram of boxes connected by wires" . 25 | 26 | monocl:hasBox rdf:type owl:ObjectProperty ; 27 | rdfs:domain monocl:WiringDiagram ; 28 | rdfs:range monocl:AbstractBox ; 29 | rdfs:label "Has box" ; 30 | rdfs:comment "Box (object) belongs to wiring diagram (subject)" . 31 | 32 | monocl:hasWire rdf:type owl:ObjectProperty ; 33 | rdfs:domain monocl:WiringDiagram ; 34 | rdfs:range monocl:Wire ; 35 | rdfs:label "Has wire" ; 36 | rdfs:comment "Wire (object) belongs to wiring diagram (subject)" . 37 | 38 | # Ports 39 | 40 | monocl:Port rdf:type owl:Class ; 41 | rdfs:subClassOf monocl:WiringDiagramElement ; 42 | rdfs:comment "Input or output port on a box" . 43 | 44 | # Wires 45 | 46 | monocl:Wire rdf:type owl:Class ; 47 | rdfs:subClassOf monocl:WiringDiagramElement ; 48 | rdfs:comment "Wire in a wiring diagram" . 49 | 50 | monocl:source rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 51 | rdfs:domain monocl:Wire ; 52 | rdfs:range monocl:Port ; 53 | rdfs:comment "Source port of wire" . 54 | 55 | monocl:target rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 56 | rdfs:domain monocl:Wire ; 57 | rdfs:range monocl:Port ; 58 | rdfs:comment "Target port of wire" . 59 | 60 | monocl:wire rdf:type owl:ObjectProperty ; 61 | rdfs:comment "Wire between two ports" . 62 | -------------------------------------------------------------------------------- /src/ontology/rdf/schema/list.ttl: -------------------------------------------------------------------------------- 1 | # List Ontology 2 | # 3 | # We define a singly linked list (chain of cons cells), following the List 4 | # Ontology in the CO-ODE project (defunct as of 2009): 5 | # 6 | # 7 | # 8 | # A list item may also specifiy an integer index, as in the Ordered List 9 | # Ontology: 10 | # 11 | # 12 | # Indices are optionally. They make lists easier to query but harder to update. 13 | # 14 | # Further reading: 15 | # 16 | # - Drummond, Rector, Stevens et al, 2006: Putting OWL in order: Patterns for 17 | # sequences in OWL 18 | # - W3C, 2006: Defining n-ary relations on the Semantic Web 19 | # 20 | 21 | @prefix rdf: . 22 | @prefix rdfs: . 23 | @prefix owl: . 24 | @prefix list: . 25 | 26 | list:OWLList rdf:type owl:Class ; 27 | rdfs:label "List" ; 28 | rdfs:comment "Singly linked list" . 29 | 30 | list:hasListProperty 31 | rdf:type owl:ObjectProperty ; 32 | rdfs:domain list:OWLList . 33 | 34 | list:hasContents 35 | rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 36 | rdfs:subPropertyOf list:hasListProperty ; 37 | rdfs:label "Has contents" . 38 | 39 | list:hasNext 40 | rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 41 | rdfs:subPropertyOf list:isFollowedBy ; 42 | rdfs:label "Has next" . 43 | 44 | list:index 45 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 46 | rdfs:subPropertyOf list:hasListProperty ; 47 | rdfs:label "Index in list" . 48 | 49 | list:isFollowedBy 50 | rdf:type owl:ObjectProperty, owl:TransitiveProperty ; 51 | rdfs:range list:OWLList ; 52 | rdfs:subPropertyOf list:hasListProperty ; 53 | rdfs:label "Is followed by" . 54 | 55 | list:EmptyList rdf:type owl:Class ; 56 | owl:equivalentClass [ 57 | owl:intersectionOf ( 58 | list:OWLList 59 | [ 60 | owl:complementOf [ 61 | rdf:type owl:Restriction ; 62 | owl:onProperty list:isFollowedBy ; 63 | owl:someValuesFrom owl:Thing 64 | ] 65 | ] 66 | ) 67 | ] ; 68 | rdfs:label "Empty list" . 69 | -------------------------------------------------------------------------------- /test/ontology/rdf/WiringRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestWiringRDF 16 | using Test 17 | 18 | using Serd, Serd.RDF 19 | using Catlab, Catlab.Theories, Catlab.WiringDiagrams 20 | 21 | using SemanticFlowGraphs 22 | 23 | const R = Resource 24 | 25 | A, B, C, D = Ob(FreeSymmetricMonoidalCategory, :A, :B, :C, :D) 26 | f = Hom(:f, A, B) 27 | g = Hom(:g, B, C) 28 | h = Hom(:h, D, D) 29 | diagram = to_wiring_diagram(otimes(compose(f,g), h)) 30 | _, stmts = wiring_diagram_to_rdf(diagram) 31 | #write_rdf(stdout, stmts) 32 | 33 | vs = box_ids(diagram) 34 | vin, vout = input_id(diagram), output_id(diagram) 35 | find_vertex = value -> vs[findfirst(v -> box(diagram, v).value == value, vs)] 36 | fv, gv, hv = find_vertex(:f), find_vertex(:g), find_vertex(:h) 37 | 38 | # Check RDF type of boxes and ports. 39 | @test Triple(Blank("n$fv"), R("rdf","type"), R("monocl","Box")) in stmts 40 | @test Triple(Blank("n$fv:in1"), R("rdf","type"), R("monocl","Port")) in stmts 41 | @test Triple(Blank("n$fv:out1"), R("rdf","type"), R("monocl","Port")) in stmts 42 | 43 | # Check values of boxes and ports. 44 | @test Triple(Blank("n$fv"), R("monocl","value"), Literal("f")) in stmts 45 | @test Triple(Blank("n$fv:in1"), R("monocl","value"), Literal("A")) in stmts 46 | @test Triple(Blank("n$fv:out1"), R("monocl","value"), Literal("B")) in stmts 47 | 48 | # Check wires, in both reified and non-reified form. 49 | i = first([ i for (i, wire) in enumerate(wires(diagram)) 50 | if wire.source.box == fv && wire.target.box == gv ]) 51 | @test Triple(Blank("e$i"), R("rdf","type"), R("monocl","Wire")) in stmts 52 | @test Triple(Blank("e$i"), R("monocl","source"), Blank("n$fv:out1")) in stmts 53 | @test Triple(Blank("e$i"), R("monocl","target"), Blank("n$gv:in1")) in stmts 54 | @test Triple(Blank("n$fv:out1"), R("monocl","wire"), Blank("n$gv:in1")) in stmts 55 | 56 | end 57 | -------------------------------------------------------------------------------- /test/ontology/data/foobar.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "concept/number", 4 | "schema": "concept", 5 | "kind": "type", 6 | "id": "number", 7 | "name": "number", 8 | "description": "a number, integer or decimal" 9 | }, 10 | { 11 | "_id": "concept/string", 12 | "schema": "concept", 13 | "kind": "type", 14 | "id": "string", 15 | "name": "string", 16 | "description": "a string of characters" 17 | }, 18 | { 19 | "_id": "concept/foo", 20 | "schema": "concept", 21 | "kind": "type", 22 | "id": "foo", 23 | "name": "Foo" 24 | }, 25 | { 26 | "_id": "concept/bar", 27 | "schema": "concept", 28 | "kind": "type", 29 | "id": "bar", 30 | "name": "Bar", 31 | "is-a": "foo" 32 | }, 33 | { 34 | "_id": "concept/baz", 35 | "schema": "concept", 36 | "kind": "type", 37 | "id": "baz", 38 | "name": "Baz", 39 | "is-a": "bar" 40 | }, 41 | { 42 | "_id": "concept/bar-from-foo", 43 | "schema": "concept", 44 | "kind": "function", 45 | "id": "bar-from-foo", 46 | "name": "transform foo to bar", 47 | "inputs": [ 48 | { 49 | "type": "foo" 50 | } 51 | ], 52 | "outputs": [ 53 | { 54 | "type": "bar" 55 | } 56 | ] 57 | }, 58 | { 59 | "_id": "concept/foo-x", 60 | "schema": "concept", 61 | "kind": "function", 62 | "id": "foo-x", 63 | "name": "slot x of foo", 64 | "inputs": [ 65 | { 66 | "type": "foo" 67 | } 68 | ], 69 | "outputs": [ 70 | { 71 | "type": "number", 72 | "name": "x" 73 | } 74 | ] 75 | }, 76 | { 77 | "_id": "concept/foo-y", 78 | "schema": "concept", 79 | "kind": "function", 80 | "id": "foo-y", 81 | "name": "slot y of foo", 82 | "inputs": [ 83 | { 84 | "type": "foo" 85 | } 86 | ], 87 | "outputs": [ 88 | { 89 | "type": "number", 90 | "name": "y" 91 | } 92 | ] 93 | }, 94 | { 95 | "_id": "concept/foo-sum", 96 | "schema": "concept", 97 | "kind": "function", 98 | "id": "foo-sum", 99 | "name": "sum slot of foo", 100 | "inputs": [ 101 | { 102 | "type": "foo" 103 | } 104 | ], 105 | "outputs": [ 106 | { 107 | "type": "number", 108 | "name": "sum" 109 | } 110 | ] 111 | } 112 | ] 113 | -------------------------------------------------------------------------------- /src/ontology/rdf/OntologyRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module OntologyRDF 16 | export ontology_to_rdf, presentation_to_rdf, annotation_to_rdf, 17 | wiring_diagram_to_rdf 18 | 19 | using Serd 20 | using Catlab 21 | 22 | using ..OntologyDBs 23 | 24 | # Submodules 25 | ############ 26 | 27 | include("RDFUtils.jl") 28 | include("WiringRDF.jl") 29 | include("ConceptRDF.jl") 30 | include("ConceptPROV.jl") 31 | include("AnnotationRDF.jl") 32 | 33 | using .RDFUtils 34 | using .WiringRDF 35 | using .ConceptRDF 36 | using .ConceptPROV 37 | using .AnnotationRDF 38 | 39 | # Ontology RDF 40 | ############## 41 | 42 | """ Convert ontology's concepts and annotations into RDF/OWL ontology. 43 | """ 44 | function ontology_to_rdf(db::OntologyDB, prefix::RDF.Prefix; 45 | include_provenance::Bool=true, 46 | include_wiring_diagrams::Bool=true)::Vector{<:RDF.Statement} 47 | # Create RDF triples for concepts. 48 | function concept_labels(expr, node::RDF.Node)::Vector{<:RDF.Statement} 49 | # Add RDFS labels for concept. 50 | doc = concept_document(db, first(expr)) 51 | rdfs_labels(doc, node) 52 | end 53 | stmts = presentation_to_rdf(concepts(db), prefix; extra_rdf=concept_labels) 54 | 55 | # Create RDF triples for concept hierarchy based on PROV-O. 56 | if include_provenance 57 | append!(stmts, presentation_to_prov(concepts(db), prefix)) 58 | end 59 | 60 | # Create RDF triples for annotations. 61 | for note in annotations(db) 62 | append!(stmts, annotation_to_rdf(note, prefix; 63 | include_provenance=include_provenance, 64 | include_wiring_diagrams=include_wiring_diagrams)) 65 | 66 | # Add RDFS labels for annotation. 67 | doc = annotation_document(db, note.name) 68 | node = annotation_rdf_node(note, prefix) 69 | append!(stmts, rdfs_labels(doc, node)) 70 | end 71 | 72 | return stmts 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /src/ontology/rdf/schema/annotation.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix monocl: . 5 | 6 | 7 | monocl:Annotation rdf:type owl:Class ; 8 | owl:disjointWith monocl:Concept ; 9 | rdfs:label "Annotation" ; 10 | rdfs:comment "Semantic annotation of computer code" . 11 | 12 | monocl:TypeAnnotation rdf:type owl:Class ; 13 | rdfs:subClassOf monocl:Annotation, monocl:Type ; 14 | rdfs:label "Type annotation" ; 15 | rdfs:comment "Annotation of concrete class or type" . 16 | 17 | monocl:FunctionAnnotation rdf:type owl:Class ; 18 | rdfs:subClassOf monocl:Annotation, monocl:Function ; 19 | rdfs:label "Function annotation" ; 20 | rdfs:comment "Annotation of concrete function or method" . 21 | 22 | monocl:SlotAnnotation rdf:type owl:Class ; 23 | rdfs:subClassOf monocl:Annotation ; 24 | rdfs:label "Slot annotation" ; 25 | rdfs:comment "Annotation of slot (property) of concrete class" . 26 | 27 | 28 | monocl:codeDefinition rdf:type owl:ObjectProperty ; 29 | rdfs:label "Definition" ; 30 | rdfs:comment "Definition of annotated code in terms of universal concepts" . 31 | 32 | monocl:annotatedLanguage 33 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 34 | rdfs:label "Programming language" ; 35 | rdfs:comment "Programming language of annotated code" . 36 | 37 | monocl:annotatedPackage 38 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 39 | rdfs:label "Package" ; 40 | rdfs:comment "Library or package of annotated code" . 41 | 42 | monocl:annotatedClass 43 | rdf:type owl:DatatypeProperty ; 44 | rdfs:label "Class" ; 45 | rdfs:comment "Class or classes (intersection) to which annotation applies" . 46 | 47 | monocl:annotatedFunction 48 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 49 | rdfs:label "Function" ; 50 | rdfs:comment "Function to which annotation applies" . 51 | 52 | monocl:annotatedMethod 53 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 54 | rdfs:label "Method" ; 55 | rdfs:comment "Method of class to which annotation applies" . 56 | 57 | monocl:annotatedSlot rdf:type owl:ObjectProperty ; 58 | rdfs:label "Annotated slot" ; 59 | rdfs:comment "Slot annotated by an type annotation" . 60 | 61 | monocl:codeSlot 62 | rdf:type owl:DatatypeProperty, owl:FunctionalProperty ; 63 | rdfs:label "Slot" ; 64 | rdfs:comment "Slot of class or function input/outputs" . 65 | -------------------------------------------------------------------------------- /src/ontology/rdf/schema/concept.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix monocl: . 5 | 6 | 7 | monocl:Type rdf:type owl:Class ; 8 | rdfs:label "Type" ; 9 | rdfs:comment "A type (aka, object)" . 10 | 11 | monocl:Function rdf:type owl:Class ; 12 | owl:disjointWith monocl:Type ; 13 | rdfs:label "Function" ; 14 | rdfs:comment "A function (aka, morphism)" . 15 | 16 | 17 | monocl:inputs rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 18 | rdfs:domain monocl:Function ; 19 | rdfs:label "Inputs" ; 20 | rdfs:comment "All inputs to the function" . 21 | 22 | monocl:outputs rdf:type owl:ObjectProperty, owl:FunctionalProperty ; 23 | rdfs:domain monocl:Function ; 24 | rdfs:label "Outputs" ; 25 | rdfs:comment "All outputs of the function" . 26 | 27 | monocl:hasInput rdf:type owl:ObjectProperty ; 28 | rdfs:domain monocl:Function ; 29 | rdfs:label "Has input" ; 30 | rdfs:comment "An input to the function" . 31 | 32 | monocl:hasOutput rdf:type owl:ObjectProperty ; 33 | rdfs:domain monocl:Function ; 34 | rdfs:label "Has output" ; 35 | rdfs:comment "An output of the function" . 36 | 37 | 38 | monocl:subtypeOf rdf:type owl:TransitiveProperty ; 39 | rdfs:domain monocl:Type ; 40 | rdfs:range monocl:Type ; 41 | rdfs:label "Subtype of" ; 42 | rdfs:comment "A subtype (is-a) relation between two types" . 43 | 44 | monocl:subfunctionOf rdf:type owl:TransitiveProperty ; 45 | rdfs:domain monocl:Function ; 46 | rdfs:range monocl:Function ; 47 | rdfs:label "Subfunction of" ; 48 | rdfs:comment "A subfunction (is-a) relation between two functions" . 49 | 50 | 51 | monocl:Concept rdf:type owl:Class ; 52 | rdfs:label "Concept" ; 53 | rdfs:comment "A concept in a Monocl ontology" . 54 | 55 | monocl:TypeConcept rdf:type owl:Class ; 56 | rdfs:subClassOf monocl:Concept, monocl:Type ; 57 | rdfs:label "Type concept" ; 58 | rdfs:comment "A type concept in a Monocl ontology" . 59 | 60 | monocl:FunctionConcept rdf:type owl:Class ; 61 | rdfs:subClassOf monocl:Concept, monocl:Function ; 62 | rdfs:label "Function concept" ; 63 | rdfs:comment "A function concept in a Monocl ontology" . 64 | 65 | 66 | monocl:id rdf:type owl:DatatypeProperty ; 67 | rdfs:label "ID" ; 68 | rdfs:comment "Identifer or name of Monocl entity, unique in some context" . 69 | 70 | monocl:isConcept rdf:type owl:ObjectProperty ; 71 | rdfs:label "Is concept" ; 72 | rdfs:comment "Asserts that the subject is, or instantiates, a concept" . 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Semantic flow graphs 2 | 3 | [![Build Status](https://github.com/IBM/semanticflowgraph/workflows/Tests/badge.svg)](https://github.com/IBM/semanticflowgraph/actions?query=workflow%3ATests) 4 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1401686.svg)](https://doi.org/10.5281/zenodo.1401686) 5 | 6 | **Create semantic dataflow graphs of data science code.** 7 | 8 | Using this package, you can convert data science code to dataflow graphs with 9 | semantic content. The package works in tandem with the [Data Science 10 | Ontology](https://github.com/ibm/datascienceontology) and our language-specific 11 | program analysis tools. Currently [Python](https://github.com/ibm/pyflowgraph) 12 | and [R](https://github.com/ibm/rflowgraph) are supported. 13 | 14 | For more information, please see our [research 15 | paper](https://www.epatters.org/papers/#2018-semantic-enrichment) on "Teaching 16 | machines to understand data science code by semantic enrichment of dataflow 17 | graphs". 18 | 19 | ## Command-line interface 20 | 21 | We provide a CLI that supports the recording, semantic enrichment, and 22 | visualization of flow graphs. To set up the CLI, install this package and add 23 | the `bin` directory to your `PATH`. Invoke the CLI by running `flowgraphs.jl` 24 | in your terminal. 25 | 26 | The CLI includes the following commands: 27 | 28 | - `record`: Record a raw flow graph by running a script. 29 | **Requirements**: To record a Python script, you must install the Julia 30 | package [PyCall.jl](https://github.com/JuliaPy/PyCall.jl) and the Python 31 | package [flowgraph](https://github.com/ibm/pyflowgraph). Likewise, to record 32 | an R script, you must install the Julia package 33 | [RCall.jl](https://github.com/JuliaInterop/RCall.jl) and the R package 34 | [flowgraph](https://github.com/ibm/rflowgraph). 35 | - `enrich`: Convert a raw flow graph to a semantic flow graph. 36 | - `visualize`: Visualize a flow graph using 37 | [Graphviz](https://gitlab.com/graphviz/graphviz). 38 | **Requirements**: To output an image, using the `--to` switch, you must 39 | install Graphviz. 40 | 41 | 42 | All the commands take as primary argument either a directory, which is filtered 43 | by file extension, or a single file, arbitrarily named. 44 | 45 | ### CLI examples 46 | 47 | Record all Python/R scripts in the current directory, yielding raw flow graphs: 48 | 49 | ``` 50 | flowgraphs.jl record . 51 | ``` 52 | 53 | Convert a raw flow graph to a semantic flow graph: 54 | 55 | ``` 56 | flowgraphs.jl enrich my_script.py.graphml --out my_script.graphml 57 | ``` 58 | 59 | Visualize a semantic flow graph, creating and opening an SVG file: 60 | 61 | ``` 62 | flowgraphs.jl visualize myscript.graphml --to svg --open 63 | ``` 64 | -------------------------------------------------------------------------------- /test/CLI.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestCLI 16 | using Test 17 | import JSON 18 | 19 | import SemanticFlowGraphs: CLI 20 | 21 | const data_dir = joinpath(@__DIR__, "data") 22 | const test_py = false 23 | const test_r = false 24 | 25 | mktempdir() do dir 26 | # Visualize raw flow graph. 27 | inpath = joinpath(data_dir, "clustering_kmeans.raw.graphml") 28 | outpath = joinpath(dir, "clustering_kmeans.raw.dot") 29 | CLI.main(["visualize", inpath, "--out", outpath]) 30 | @test isfile(outpath) 31 | 32 | # Convert raw flow graph to semantic flow graph. 33 | outpath = joinpath(dir, "clustering_kmeans.semantic.graphml") 34 | CLI.main(["enrich", inpath, "--out", outpath]) 35 | @test isfile(outpath) 36 | 37 | # Visualize semantic flow graph. 38 | inpath = outpath 39 | outpath = joinpath(dir, "clustering_kmeans.semantic.dot") 40 | CLI.main(["visualize", inpath, "--out", outpath]) 41 | @test isfile(outpath) 42 | end 43 | 44 | mktempdir() do dir 45 | # Export concepts as JSON. 46 | outpath = joinpath(dir, "concepts.json") 47 | CLI.main(["ontology", "json", "--no-annotations", "--out", outpath]) 48 | @test isfile(outpath) 49 | @test JSON.parsefile(outpath) isa AbstractVector 50 | end 51 | 52 | mktempdir() do dir 53 | # Export concepts as RDF/OWL. 54 | outpath = joinpath(dir, "concepts.ttl") 55 | CLI.main(["ontology", "rdf", "--no-annotations", "--out", outpath]) 56 | @test isfile(outpath) 57 | end 58 | 59 | if test_py 60 | import PyCall 61 | 62 | mktempdir() do dir 63 | # Record Python raw flow graph. 64 | inpath = joinpath(data_dir, "sklearn_clustering_kmeans.py") 65 | outpath = joinpath(dir, "sklearn_clustering_kmeans.raw.graphml") 66 | CLI.main(["record", inpath, "--out", outpath]) 67 | @test isfile(outpath) 68 | end 69 | end 70 | 71 | if test_r 72 | import RCall 73 | 74 | mktempdir() do dir 75 | # Record R raw flow graph. 76 | inpath = joinpath(data_dir, "clustering_kmeans.R") 77 | outpath = joinpath(dir, "clustering_kmeans.raw.graphml") 78 | CLI.main(["record", inpath, "--out", outpath]) 79 | @test isfile(outpath) 80 | end 81 | end 82 | 83 | end 84 | -------------------------------------------------------------------------------- /test/ontology/rdf/ConceptRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestConceptRDF 16 | using Test 17 | 18 | using Serd 19 | using Serd.RDF: Triple, Literal, Resource 20 | using Catlab 21 | 22 | using SemanticFlowGraphs 23 | 24 | const R = Resource 25 | 26 | @present TestPres(Monocl) begin 27 | A::Ob 28 | B::Ob 29 | C::Ob 30 | 31 | A0::Ob 32 | B0::Ob 33 | ::SubOb(A0,A) 34 | ::SubOb(B0,B) 35 | 36 | f::Hom(A,B) 37 | g::Hom(A,otimes(B,C)) 38 | 39 | f0::Hom(A0,B0) 40 | ::SubHom(f0,f,SubOb(A0,A),SubOb(B0,B)) 41 | end 42 | 43 | prefix = RDF.Prefix("ex", "http://www.example.org/#") 44 | stmts = presentation_to_rdf(TestPres, prefix) 45 | #write_rdf(stdout, stmts) 46 | 47 | @test Triple(R("ex","A"), R("rdf","type"), R("monocl","TypeConcept")) in stmts 48 | @test Triple(R("ex","A"), R("rdf","type"), R("monocl","TypeConcept")) in stmts 49 | @test Triple(R("ex","A0"), R("rdf","type"), R("monocl","TypeConcept")) in stmts 50 | @test Triple(R("ex","f"), R("rdf","type"), R("monocl","FunctionConcept")) in stmts 51 | 52 | @test Triple(R("ex","A"), R("monocl","id"), RDF.Literal("A")) in stmts 53 | @test Triple(R("ex","f"), R("monocl","id"), RDF.Literal("f")) in stmts 54 | 55 | @test Triple(R("ex","A0"), R("monocl","subtypeOf"), R("ex","A")) in stmts 56 | @test Triple(R("ex","f0"), R("monocl","subfunctionOf"), R("ex","f")) in stmts 57 | 58 | @test Triple(R("ex","g"), R("monocl","inputs"), R("ex","g:in1")) in stmts 59 | @test Triple(R("ex","g"), R("monocl","hasInput"), R("ex","g:in1")) in stmts 60 | @test Triple(R("ex","g:in1"), R("list","hasContents"), R("ex","A")) in stmts 61 | @test Triple(R("ex","g:in1"), R("list","index"), Literal(1)) in stmts 62 | 63 | @test Triple(R("ex","g"), R("monocl","outputs"), R("ex","g:out1")) in stmts 64 | @test Triple(R("ex","g"), R("monocl","hasOutput"), R("ex","g:out1")) in stmts 65 | @test Triple(R("ex","g"), R("monocl","hasOutput"), R("ex","g:out2")) in stmts 66 | @test Triple(R("ex","g:out1"), R("list","hasContents"), R("ex","B")) in stmts 67 | @test Triple(R("ex","g:out1"), R("list","hasNext"), R("ex","g:out2")) in stmts 68 | @test Triple(R("ex","g:out1"), R("list","index"), Literal(1)) in stmts 69 | @test Triple(R("ex","g:out2"), R("list","hasContents"), R("ex","C")) in stmts 70 | @test Triple(R("ex","g:out2"), R("list","index"), Literal(2)) in stmts 71 | 72 | end 73 | -------------------------------------------------------------------------------- /test/ontology/OntologyDBs.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestOntologyDBs 16 | using Test 17 | 18 | using Catlab 19 | using SemanticFlowGraphs 20 | 21 | # Local file 22 | ############ 23 | 24 | # Load concepts. 25 | db = OntologyDB() 26 | @test !has_concept(db, "foo") 27 | @test_throws OntologyError concept(db, "foo") 28 | load_ontology_file(db, joinpath(@__DIR__, "data", "foobar.json")) 29 | @test has_concept(db, "foo") 30 | @test concept(db, "foo") isa Monocl.Ob 31 | @test concept(db, "bar-from-foo") isa Monocl.Hom 32 | 33 | # Concept accessors. 34 | @test concepts(db) isa Presentation 35 | @test concepts(db, ["foo", "bar-from-foo"]) == 36 | [ concept(db, "foo"), concept(db, "bar-from-foo") ] 37 | 38 | # Remote database 39 | ################# 40 | 41 | # Load all concepts. 42 | db = OntologyDB() 43 | @test !has_concept(db, "model") 44 | load_concepts(db) 45 | @test has_concept(db, "model") 46 | @test concept(db, "model") isa Monocl.Ob 47 | @test concept(db, "fit") isa Monocl.Hom 48 | 49 | # Load single annotation. 50 | df_id = AnnotationID("python", "pandas", "data-frame") 51 | @test !has_annotation(db, df_id) 52 | @test_throws OntologyError annotation(db, df_id) 53 | @test isa(load_annotation(db, df_id), ObAnnotation) 54 | @test isa(annotation(db, df_id), ObAnnotation) 55 | @test isa(annotation_document(db, df_id), AbstractDict) 56 | @test has_annotation(db, df_id) 57 | @test annotation(db, df_id) == annotation(db, "python/pandas/data-frame") 58 | @test annotation(db, df_id) == annotation(db, "annotation/python/pandas/data-frame") 59 | 60 | bad_id = AnnotationID("python", "pandas", "xxx") 61 | @test_throws OntologyError load_annotation(db, bad_id) 62 | 63 | # Load all annotations in a package. 64 | series_id = AnnotationID("python", "pandas", "series") 65 | ndarray_id = AnnotationID("python", "numpy", "ndarray") 66 | @test_throws OntologyError annotation(db, series_id) 67 | @test_throws OntologyError annotation(db, ndarray_id) 68 | load_annotations(db, language="python", package="pandas") 69 | @test_throws OntologyError annotation(db, ndarray_id) 70 | 71 | note = annotation(db, series_id) 72 | @test isa(note, Annotation) 73 | @test isa(note.definition, Monocl.Ob) 74 | 75 | note = annotation(db, AnnotationID("python", "pandas", "read-table")) 76 | @test isa(note, Annotation) 77 | @test isa(note.definition, Monocl.Hom) 78 | 79 | end 80 | -------------------------------------------------------------------------------- /test/ontology/rdf/AnnotationRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestAnnotationRDF 16 | using Test 17 | 18 | using Serd, Serd.RDF 19 | using Catlab 20 | 21 | using SemanticFlowGraphs 22 | 23 | const R = Resource 24 | 25 | A, B, C, D = Ob(Monocl, "A", "B", "C", "D") 26 | f = Hom("f", A, B) 27 | g = Hom("g", B, C) 28 | h = Hom("h", D, D) 29 | 30 | # Object annotations 31 | 32 | prefix = RDF.Prefix("ex", "http://www.example.org/#") 33 | annotation = ObAnnotation( 34 | AnnotationID("python", "mypkg", "a"), 35 | Dict( 36 | :class => "ClassA", 37 | :slots => [Dict("slot" => "attrB")], 38 | ), 39 | A, [f] 40 | ) 41 | stmts = annotation_to_rdf(annotation, prefix) 42 | node = R("ex", "python:mypkg:a") 43 | @test Triple(node, R("monocl", "annotatedLanguage"), Literal("python")) in stmts 44 | @test Triple(node, R("monocl", "annotatedPackage"), Literal("mypkg")) in stmts 45 | @test Triple(node, R("monocl", "annotatedClass"), Literal("ClassA")) in stmts 46 | @test Triple(node, R("monocl", "codeDefinition"), R("ex","A")) in stmts 47 | 48 | slot_node = R("ex", "python:mypkg:a:slot1") 49 | @test Triple(node, R("monocl", "annotatedSlot"), slot_node) in stmts 50 | @test Triple(slot_node, R("monocl", "codeDefinition"), R("ex","f")) in stmts 51 | @test Triple(slot_node, R("monocl", "codeSlot"), Literal("attrB")) in stmts 52 | 53 | # Morphism annotations 54 | 55 | annotation = HomAnnotation( 56 | AnnotationID("python", "mypkg", "a-do-composition"), 57 | Dict( 58 | :class => ["ClassA", "MixinB"], 59 | :method => "do_composition", 60 | :inputs => [Dict("slot" => 1)], 61 | :outputs => [Dict("slot" => "return")], 62 | ), 63 | compose(f,g) 64 | ) 65 | node = R("ex", "python:mypkg:a-do-composition") 66 | root_node = R("ex", "python:mypkg:a-do-composition:diagram:root") 67 | stmts = annotation_to_rdf(annotation, prefix) 68 | @test Triple(node, R("monocl", "annotatedLanguage"), Literal("python")) in stmts 69 | @test Triple(node, R("monocl", "annotatedPackage"), Literal("mypkg")) in stmts 70 | @test Triple(node, R("monocl", "annotatedClass"), Literal("ClassA")) in stmts 71 | @test Triple(node, R("monocl", "annotatedClass"), Literal("MixinB")) in stmts 72 | @test Triple(node, R("monocl", "annotatedMethod"), Literal("do_composition")) in stmts 73 | @test Triple(node, R("monocl", "codeDefinition"), root_node) in stmts 74 | 75 | end 76 | -------------------------------------------------------------------------------- /src/ontology/rdf/ConceptPROV.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module ConceptPROV 16 | export presentation_to_prov 17 | 18 | using Serd 19 | using Catlab 20 | 21 | using ...Doctrine, ...Ontology 22 | using ..ConceptRDF 23 | 24 | const R = RDF.Resource 25 | 26 | 27 | """ Convert concepts into hierarchy of PROV-O classes. 28 | 29 | Type concepts are not really OWL classes and function concepts are not even 30 | close to OWL classes (nor are they particularly close to OWL properties). 31 | However, for the sake of Semantic Web interoperability, we encode type concepts 32 | as a hierarchy of classes below PROV's Entity and function concepts as a 33 | hierarchy of classes below PROV's Activity. 34 | 35 | In the primary export function (`presentation_to_rdf`), concepts are represented 36 | as individuals, not classes, which is necessary because they have properties. 37 | Thus, when used in conjunction with this function, concepts are simultaneously 38 | classes and individuals. This is prohibited in OWL 1, but permitted in OWL 2 DL, 39 | where is it called "punning". 40 | 41 | References: 42 | - https://www.w3.org/TR/owl2-new-features/#F12:_Punning 43 | - https://www.w3.org/TR/prov-o/ 44 | """ 45 | function presentation_to_prov(pres::Presentation, prefix::RDF.Prefix) 46 | stmts = RDF.Statement[ 47 | RDF.Prefix("rdf"), RDF.Prefix("rdfs"), RDF.Prefix("owl"), 48 | RDF.Prefix("prov"), 49 | RDF.Prefix("monocl", "https://www.datascienceontology.org/ns/monocl/"), 50 | prefix 51 | ] 52 | 53 | # Create mapping from concept node to all its super concept nodes. 54 | super_map = Dict{RDF.Node,Vector{RDF.Node}}() 55 | for sub in [generators(pres, Monocl.SubOb); generators(pres, Monocl.SubHom)] 56 | push!(get!(super_map, generator_rdf_node(dom(sub), prefix), RDF.Node[]), 57 | generator_rdf_node(codom(sub), prefix)) 58 | end 59 | 60 | # Generate RDF triples. 61 | for gen in [generators(pres, Monocl.Ob); generators(pres, Monocl.Hom)] 62 | node = generator_rdf_node(gen, prefix) 63 | super_nodes = get(super_map, node) do 64 | [ gen isa Monocl.Ob ? R("prov","Entity") : R("prov","Activity") ] 65 | end 66 | push!(stmts, RDF.Triple(node, R("rdf","type"), R("owl","Class"))) 67 | for super_node in super_nodes 68 | push!(stmts, RDF.Triple(node, R("rdfs","subClassOf"), super_node)) 69 | end 70 | end 71 | stmts 72 | end 73 | 74 | end 75 | -------------------------------------------------------------------------------- /src/ontology/rdf/RDFUtils.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module RDFUtils 16 | export owl_list, owl_inputs_outputs, rdfs_labels 17 | 18 | using Serd 19 | 20 | const R = RDF.Resource 21 | 22 | 23 | """ Convert sequence of RDF nodes into an OWL list. 24 | 25 | OWL doesn't officially support lists, but it's straightforward to implement a 26 | singly linked list (chain of cons cells). See the `list` schema for more info. 27 | 28 | Note: We don't use the builtin RDF List because OWL doesn't support RDF Lists. 29 | """ 30 | function owl_list(cell_content::Function, cell_node::Function, n::Int; 31 | index::Bool=false) 32 | stmts = RDF.Statement[] 33 | for i in 1:n 34 | cell = cell_node(i) 35 | rest = cell_node(i+1) 36 | append!(stmts, [ 37 | [ RDF.Triple(cell, R("rdf","type"), R("list","OWLList")) ]; 38 | cell_content(cell, i); 39 | [ RDF.Triple(cell, R("list","hasNext"), rest) ]; 40 | index ? 41 | [ RDF.Triple(cell, R("list","index"), RDF.Literal(i)) ] : []; 42 | ]) 43 | end 44 | nil = cell_node(n+1) 45 | push!(stmts, RDF.Triple(nil, R("rdf","type"), R("list","EmptyList"))) 46 | stmts 47 | end 48 | 49 | """ Create RDF/OWL lists for the inputs and outputs of a function-like node. 50 | """ 51 | function owl_inputs_outputs(cell_content::Function, node::RDF.Node, 52 | cell_node::Function, nin::Int, nout::Int; index::Bool=false) 53 | input_cell = i -> cell_node("in$i") 54 | output_cell = i -> cell_node("out$i") 55 | RDF.Statement[ 56 | [ 57 | RDF.Triple(node, R("monocl","inputs"), input_cell(1)), 58 | RDF.Triple(node, R("monocl","outputs"), output_cell(1)), 59 | ]; 60 | [ RDF.Triple(node, R("monocl","hasInput"), input_cell(i)) for i in 1:nin ]; 61 | [ RDF.Triple(node, R("monocl","hasOutput"), output_cell(i)) for i in 1:nout ]; 62 | owl_list(input_cell, nin, index=index) do cell, i 63 | cell_content(cell, true, i) 64 | end; 65 | owl_list(output_cell, nout, index=index) do cell, i 66 | cell_content(cell, false, i) 67 | end; 68 | ] 69 | end 70 | 71 | """ Create RDFS label/comment from document name/description. 72 | """ 73 | function rdfs_labels(doc, node::RDF.Node)::Vector{<:RDF.Statement} 74 | stmts = RDF.Statement[] 75 | if haskey(doc, "name") 76 | push!(stmts, RDF.Triple( 77 | node, 78 | RDF.Resource("rdfs", "label"), 79 | RDF.Literal(doc["name"]) 80 | )) 81 | end 82 | if haskey(doc, "description") 83 | push!(stmts, RDF.Triple( 84 | node, 85 | RDF.Resource("rdfs", "comment"), 86 | RDF.Literal(doc["description"]) 87 | )) 88 | end 89 | stmts 90 | end 91 | 92 | end 93 | -------------------------------------------------------------------------------- /test/Doctrine.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestDoctrine 16 | using Test 17 | 18 | using Catlab, Catlab.WiringDiagrams, Catlab.Graphics 19 | import Catlab.Graphics: Graphviz 20 | using SemanticFlowGraphs.Doctrine 21 | 22 | # Monocl category 23 | ################# 24 | 25 | A, B, C = Ob(Monocl, :A, :B, :C) 26 | A0, B0, C0, A1, B1, C1 = Ob(Monocl, :A0, :B0, :C0, :A1, :B1, :C1) 27 | I = munit(Monocl.Ob) 28 | f = Hom(:f, A, B) 29 | f0, f1 = Hom(:f0, A0, B0), Hom(:f1, A1, B1) 30 | 31 | # Subobjects 32 | subA = SubOb(A0, A) 33 | subB = SubOb(B0, B) 34 | @test dom(subA) == A0 35 | @test codom(subA) == A 36 | 37 | sub = compose(SubOb(A0, A), SubOb(A, A1)) 38 | @test dom(sub) == A0 39 | @test codom(sub) == A1 40 | @test dom(subob_id(A)) == A 41 | @test codom(subob_id(A)) == A 42 | @test_throws SyntaxDomainError compose(subA, subB) 43 | 44 | sub = otimes(subA, subB) 45 | @test dom(sub) == otimes(A0,B0) 46 | @test codom(sub) == otimes(A,B) 47 | 48 | # Submorphisms 49 | subf = SubHom(f0, f, subA, subB) 50 | @test dom(subf) == f0 51 | @test codom(subf) == f 52 | @test subob_dom(subf) == subA 53 | @test subob_codom(subf) == subB 54 | 55 | subf1 = SubHom(f, f1, SubOb(A, A1), SubOb(B, B1)) 56 | sub = compose(subf, subf1) 57 | @test dom(sub) == f0 58 | @test codom(sub) == f1 59 | @test subob_dom(sub) == compose(SubOb(A0,A), SubOb(A,A1)) 60 | @test subob_codom(sub) == compose(SubOb(B0,B), SubOb(B,B1)) 61 | @test dom(subhom_id(f)) == f 62 | @test codom(subhom_id(f)) == f 63 | @test subob_dom(subhom_id(f)) == subob_id(A) 64 | @test subob_codom(subhom_id(f)) == subob_id(B) 65 | 66 | g, g0 = Hom(:g, B, C), Hom(:g0, B0, C0) 67 | subg = SubHom(g0, g, SubOb(B0, B), SubOb(C0, C)) 68 | sub = compose2(subf, subg) 69 | @test dom(sub) == compose(f0,g0) 70 | @test codom(sub) == compose(f,g) 71 | @test subob_dom(sub) == SubOb(A0, A) 72 | @test subob_codom(sub) == SubOb(C0, C) 73 | 74 | # Explicit coercions 75 | @test dom(coerce(subA)) == A0 76 | @test codom(coerce(subA)) == A 77 | @test coerce(subob_id(A)) == id(A) 78 | @test compose(coerce(subob_id(A)), f) == f 79 | @test compose(f, coerce(subob_id(B))) == f 80 | 81 | # Constructors 82 | @test dom(construct(A)) == I 83 | @test codom(construct(A)) == A 84 | @test dom(construct(f)) == B 85 | @test codom(construct(f)) == A 86 | 87 | # Pairs 88 | f, g = Hom(:f, A, B), Hom(:g, A, C) 89 | @test dom(pair(f,g)) == A 90 | @test codom(pair(f,g)) == otimes(B,C) 91 | @test pair(f,g) == compose(mcopy(A), otimes(f,g)) 92 | @test dom(pair(A0,f,g)) == A0 93 | @test codom(pair(A0,f,g)) == otimes(B,C) 94 | 95 | # Monocl wiring diagram 96 | ####################### 97 | 98 | A0, A, B0, B = Ob(Monocl, :A0, :A, :B0, :B) 99 | f = Hom(:f, A, B) 100 | g = Hom(:g, B, A) 101 | 102 | diagram = to_wiring_diagram(f) 103 | @test boxes(diagram) == [ Box(f) ] 104 | @test input_ports(diagram) == [ A ] 105 | @test output_ports(diagram) == [ B ] 106 | 107 | # Coercion 108 | sub = SubOb(A0, A) 109 | diagram = to_wiring_diagram(compose(coerce(sub), f)) 110 | @test boxes(diagram) == [ Box(f) ] 111 | @test input_ports(diagram) == [ A0 ] 112 | @test output_ports(diagram) == [ B ] 113 | 114 | # Graphviz support. 115 | diagram = to_wiring_diagram(compose(coerce(SubOb(A0,A)), construct(g))) 116 | @test isa(to_graphviz(diagram), Graphviz.Graph) 117 | 118 | end 119 | -------------------------------------------------------------------------------- /src/ontology/rdf/ConceptRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module ConceptRDF 16 | export presentation_to_rdf, generator_rdf_node 17 | 18 | using Serd 19 | using Catlab 20 | 21 | using ...Doctrine, ...Ontology 22 | using ..RDFUtils 23 | 24 | const R = RDF.Resource 25 | 26 | 27 | """ Convert concepts into RDF/OWL ontology. 28 | 29 | Concepts are repesented as individuals of the OWL class `Concept`. 30 | """ 31 | function presentation_to_rdf(pres::Presentation, prefix::RDF.Prefix; 32 | extra_rdf::Union{Function,Nothing}=nothing) 33 | stmts = RDF.Statement[ 34 | RDF.Prefix("rdf"), RDF.Prefix("rdfs"), RDF.Prefix("owl"), 35 | RDF.Prefix("list", "http://www.co-ode.org/ontologies/list.owl#"), 36 | RDF.Prefix("monocl", "https://www.datascienceontology.org/ns/monocl/"), 37 | prefix 38 | ] 39 | for expr in generators(pres) 40 | append!(stmts, expr_to_rdf(expr, prefix)) 41 | if extra_rdf != nothing && first(expr) != nothing 42 | append!(stmts, extra_rdf(expr, generator_rdf_node(expr, prefix))) 43 | end 44 | end 45 | stmts 46 | end 47 | 48 | """ Generate RDF for object generator. 49 | """ 50 | function expr_to_rdf(ob::Monocl.Ob{:generator}, prefix::RDF.Prefix) 51 | node = generator_rdf_node(ob, prefix) 52 | [ RDF.Triple(node, R("rdf","type"), R("monocl","TypeConcept")), 53 | RDF.Triple(node, R("monocl","id"), RDF.Literal(string(first(ob)))) ] 54 | end 55 | 56 | """ Generate RDF for subobject relation. 57 | """ 58 | function expr_to_rdf(sub::Monocl.SubOb, prefix::RDF.Prefix) 59 | dom_node = generator_rdf_node(dom(sub), prefix) 60 | codom_node = generator_rdf_node(codom(sub), prefix) 61 | [ RDF.Triple(dom_node, R("monocl","subtypeOf"), codom_node) ] 62 | end 63 | 64 | """ Generate RDF for morphism generator. 65 | 66 | The domain and codomain objects are represented as OWL lists. 67 | """ 68 | function expr_to_rdf(hom::Monocl.Hom{:generator}, prefix::RDF.Prefix) 69 | node = generator_rdf_node(hom, prefix) 70 | cell_node = name -> R(prefix.name, "$(node.name):$name") 71 | input_nodes = [ generator_rdf_node(ob, prefix) for ob in collect(dom(hom)) ] 72 | output_nodes = [ generator_rdf_node(ob, prefix) for ob in collect(codom(hom)) ] 73 | nin, nout = length(input_nodes), length(output_nodes) 74 | RDF.Statement[ 75 | [ RDF.Triple(node, R("rdf","type"), R("monocl","FunctionConcept")), 76 | RDF.Triple(node, R("monocl","id"), RDF.Literal(string(first(hom)))) ]; 77 | owl_inputs_outputs(node, cell_node, nin, nout, index=true) do cell, is_input, i 78 | gen_node = (is_input ? input_nodes : output_nodes)[i] 79 | [ RDF.Triple(cell, R("list","hasContents"), gen_node), 80 | RDF.Triple(cell, R("monocl","isConcept"), gen_node) ] 81 | end; 82 | ] 83 | end 84 | 85 | """ Generate RDF for submorphism relation. 86 | """ 87 | function expr_to_rdf(sub::Monocl.SubHom, prefix::RDF.Prefix) 88 | dom_node = generator_rdf_node(dom(sub), prefix) 89 | codom_node = generator_rdf_node(codom(sub), prefix) 90 | [ RDF.Triple(dom_node, R("monocl","subfunctionOf"), codom_node) ] 91 | end 92 | 93 | """ Create RDF node for generator expression. 94 | """ 95 | function generator_rdf_node(expr::GATExpr{:generator}, prefix::RDF.Prefix) 96 | @assert first(expr) != nothing 97 | R(prefix.name, string(first(expr))) 98 | end 99 | 100 | end 101 | -------------------------------------------------------------------------------- /src/RawFlowGraphs.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Datatypes and post-processing for raw flow graphs. 16 | """ 17 | module RawFlowGraphs 18 | export RawNode, RawPort, RawNodeAnnotationKind, 19 | FunctionAnnotation, ConstructAnnotation, SlotAnnotation, 20 | rem_literals, rem_unused_ports 21 | 22 | using AutoHashEquals, Parameters 23 | 24 | using Catlab.WiringDiagrams 25 | 26 | @enum(RawNodeAnnotationKind, 27 | FunctionAnnotation = 0, 28 | ConstructAnnotation = 1, 29 | SlotAnnotation = 2) 30 | 31 | function Base.convert(::Type{RawNodeAnnotationKind}, s::String) 32 | if (s == "function") FunctionAnnotation 33 | elseif (s == "construct") ConstructAnnotation 34 | elseif (s == "slot") SlotAnnotation 35 | else error("Unknown annotation kind \"$s\"") end 36 | end 37 | 38 | @with_kw struct RawNode 39 | language::Dict{String,Any} = Dict{String,Any}() 40 | annotation::Union{String,Nothing} = nothing 41 | annotation_index::Union{Int,Nothing} = nothing 42 | annotation_kind::RawNodeAnnotationKind = FunctionAnnotation 43 | end 44 | 45 | function Base.:(==)(n1::RawNode, n2::RawNode) 46 | n1.language == n2.language && 47 | isequal(n1.annotation, n2.annotation) && 48 | isequal(n1.annotation_index, n2.annotation_index) && 49 | n1.annotation_kind == n2.annotation_kind 50 | end 51 | 52 | @with_kw struct RawPort 53 | language::Dict{String,Any} = Dict{String,Any}() 54 | annotation::Union{String,Nothing} = nothing 55 | annotation_index::Union{Int,Nothing} = nothing 56 | value::Any = nothing 57 | end 58 | 59 | function Base.:(==)(p1::RawPort, p2::RawPort) 60 | p1.language == p2.language && 61 | isequal(p1.annotation, p2.annotation) && 62 | isequal(p1.annotation_index, p2.annotation_index) && 63 | isequal(p1.value, p2.value) 64 | end 65 | 66 | # Graph post-processing 67 | ####################### 68 | 69 | """ Remove literals from raw flow graph. 70 | 71 | Removes all nodes that are literal value constructors. (Currently, such nodes 72 | occur in raw flow graphs for R, but not Python.) 73 | """ 74 | function rem_literals(d::WiringDiagram) 75 | nonliterals = filter(box_ids(d)) do v 76 | kind = get(box(d,v).value.language, "kind", "function") 77 | kind != "literal" 78 | end 79 | induced_subdiagram(d, nonliterals) 80 | end 81 | 82 | """ Remove input and output ports with no connecting wires. 83 | 84 | This simplification is practically necessary to visualize raw flow graphs 85 | because scientific computing functions often have dozens of keyword arguments 86 | (which manifest as input ports). 87 | """ 88 | function rem_unused_ports(diagram::WiringDiagram) 89 | result = WiringDiagram(input_ports(diagram), output_ports(diagram)) 90 | for v in box_ids(diagram) 91 | # Note: To ensure that port numbers on wires remain valid, we only remove 92 | # unused ports beyond the last used port. 93 | b = box(diagram, v) 94 | last_used_input = maximum([0; [wire.target.port for wire in in_wires(diagram, v)]]) 95 | last_used_output = maximum([0; [wire.source.port for wire in out_wires(diagram, v)]]) 96 | unused_inputs = input_ports(b)[1:last_used_input] 97 | unused_outputs = output_ports(b)[1:last_used_output] 98 | @assert add_box!(result, Box(b.value, unused_inputs, unused_outputs)) == v 99 | end 100 | add_wires!(result, wires(diagram)) 101 | result 102 | end 103 | 104 | end 105 | -------------------------------------------------------------------------------- /test/ontology/OntologyJSON.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestOntologyJSON 16 | using Test 17 | 18 | using Catlab 19 | using SemanticFlowGraphs 20 | 21 | # Concepts 22 | ########## 23 | 24 | TestPres = Presentation{String}(Monocl) 25 | A, B, C, A0, B0 = Ob(Monocl, "A", "B", "C", "A0", "B0") 26 | I = munit(Monocl.Ob) 27 | f, f0 = Hom("f", A, B), Hom("f0", A0, B0) 28 | add_generators!(TestPres, [A, B, C, A0, B0]) 29 | add_generators!(TestPres, [SubOb(A0, A), SubOb(B0, B)]) 30 | add_generators!(TestPres, [f, f0]) 31 | add_generator!(TestPres, Hom("g", I, otimes(A,B))) 32 | add_generator!(TestPres, SubHom(f0, f, SubOb(A0, A), SubOb(B0, B))) 33 | 34 | concept(pairs...) = Dict("schema" => "concept", pairs...) 35 | 36 | const docs = [ 37 | concept( 38 | "kind" => "type", 39 | "id" => "A", 40 | ), 41 | concept( 42 | "kind" => "type", 43 | "id" => "B", 44 | ), 45 | concept( 46 | "kind" => "type", 47 | "id" => "C", 48 | ), 49 | concept( 50 | "kind" => "type", 51 | "id" => "A0", 52 | "is-a" => "A", 53 | ), 54 | concept( 55 | "kind" => "type", 56 | "id" => "B0", 57 | "is-a" => "B", 58 | ), 59 | concept( 60 | "kind" => "function", 61 | "id" => "f", 62 | "inputs" => [ 63 | Dict("type" => "A"), 64 | ], 65 | "outputs" => [ 66 | Dict("type" => "B"), 67 | ] 68 | ), 69 | concept( 70 | "kind" => "function", 71 | "id" => "f0", 72 | "is-a" => "f", 73 | "inputs" => [ 74 | Dict("type" => "A0"), 75 | ], 76 | "outputs" => [ 77 | Dict("type" => "B0"), 78 | ] 79 | ), 80 | concept( 81 | "kind" => "function", 82 | "id" => "g", 83 | "inputs" => [], 84 | "outputs" => [ 85 | Dict("type" => "A"), 86 | Dict("type" => "B"), 87 | ] 88 | ) 89 | ] 90 | 91 | pres = presentation_from_json(docs) 92 | @test generators(pres, Monocl.Ob) == generators(TestPres, Monocl.Ob) 93 | @test generators(pres, Monocl.Hom) == generators(TestPres, Monocl.Hom) 94 | @test generators(pres, Monocl.SubOb) == generators(TestPres, Monocl.SubOb) 95 | @test generators(pres, Monocl.SubHom) == generators(TestPres, Monocl.SubHom) 96 | 97 | # Annotations 98 | ############# 99 | 100 | pres = Presentation{String}(Monocl) 101 | A, B, C, D = Ob(Monocl, "A", "B", "C", "D") 102 | f = Hom("f", A, B) 103 | g = Hom("g", B, C) 104 | h = Hom("h", D, D) 105 | add_generators!(pres, [A, B, C, D, f, g]) 106 | 107 | note = annotation_from_json(Dict( 108 | "schema" => "annotation", 109 | "kind" => "type", 110 | "language" => "python", 111 | "package" => "mypkg", 112 | "id" => "a", 113 | "class" => "ClassA", 114 | "definition" => "A" 115 | ), pres) 116 | @test note.name == AnnotationID("python", "mypkg", "a") 117 | @test note.language == Dict(:class => "ClassA") 118 | @test note.definition == A 119 | 120 | note = annotation_from_json(Dict( 121 | "schema" => "annotation", 122 | "kind" => "function", 123 | "language" => "python", 124 | "package" => "mypkg", 125 | "id" => "a-do-composition", 126 | "class" => "ClassA", 127 | "method" => "do_composition", 128 | "definition" => [ 129 | "otimes", 130 | ["compose", "f", "g" ], 131 | ["Hom", "h", "D", "D" ], 132 | ] 133 | ), pres) 134 | @test note.name == AnnotationID("python", "mypkg", "a-do-composition") 135 | @test note.language == Dict(:class => "ClassA", :method => "do_composition") 136 | @test note.definition == otimes(compose(f, g), h) 137 | 138 | end 139 | -------------------------------------------------------------------------------- /src/Serialization.jl: -------------------------------------------------------------------------------- 1 | """ Serialize raw and semantic flow graphs. 2 | """ 3 | module Serialization 4 | export parse_raw_graphml, parse_raw_graph_json, 5 | parse_semantic_graphml, parse_semantic_graph_json, 6 | read_raw_graphml, read_raw_graph_json, 7 | read_semantic_graphml, read_semantic_graph_json 8 | 9 | using Compat 10 | 11 | using Catlab, Catlab.WiringDiagrams 12 | import Catlab.WiringDiagrams: convert_from_graph_data, convert_to_graph_data 13 | using ..Doctrine 14 | using ..RawFlowGraphs 15 | 16 | # Raw flow graphs 17 | ################# 18 | 19 | parse_raw_graphml(xml) = parse_graphml(RawNode, RawPort, Nothing, xml) 20 | parse_raw_graph_json(json) = parse_json_graph(RawNode, RawPort, Nothing, json) 21 | read_raw_graphml(filename) = read_graphml(RawNode, RawPort, Nothing, filename) 22 | read_raw_graph_json(filename) = read_json_graph(RawNode, RawPort, Nothing, filename) 23 | 24 | function convert_from_graph_data(::Type{RawNode}, data::AbstractDict) 25 | annotation = pop!(data, "annotation", nothing) 26 | annotation_index = pop!(data, "annotation_index", nothing) 27 | annotation_kind_str = pop!(data, "annotation_kind", nothing) 28 | annotation_kind = isnothing(annotation_kind_str) ? FunctionAnnotation : 29 | convert(RawNodeAnnotationKind, annotation_kind_str) 30 | RawNode(data, annotation, annotation_index, annotation_kind) 31 | end 32 | 33 | function convert_from_graph_data(::Type{RawPort}, data::AbstractDict) 34 | annotation = pop!(data, "annotation", nothing) 35 | annotation_index = pop!(data, "annotation_index", nothing) 36 | value = pop!(data, "value", nothing) 37 | RawPort(data, annotation, annotation_index, value) 38 | end 39 | 40 | # Semantic flow graphs 41 | ###################### 42 | 43 | parse_semantic_graphml(xml) = parse_graphml( 44 | Union{Monocl.Hom,Nothing}, Union{Monocl.Ob,Nothing}, Nothing, xml) 45 | parse_semantic_graph_json(json) = parse_json_graph( 46 | Union{Monocl.Hom,Nothing}, Union{Monocl.Ob,Nothing}, Nothing, json) 47 | read_semantic_graphml(filename) = read_graphml( 48 | Union{Monocl.Hom,Nothing}, Union{Monocl.Ob,Nothing}, Nothing, filename) 49 | read_semantic_graph_json(filename) = read_json_graph( 50 | Union{Monocl.Hom,Nothing}, Union{Monocl.Ob,Nothing}, Nothing, filename) 51 | 52 | function convert_from_graph_data(::Type{Monocl.Ob}, data::AbstractDict) 53 | parse_json_sexpr(Monocl, data["ob"]; symbols=false) 54 | end 55 | function convert_from_graph_data(::Type{Monocl.Hom}, data::AbstractDict) 56 | parse_json_sexpr(Monocl, data["hom"]; symbols=false) 57 | end 58 | function convert_from_graph_data(::Type{Union{Monocl.Ob,Nothing}}, data::AbstractDict) 59 | isempty(data) ? nothing : convert_from_graph_data(Monocl.Ob, data) 60 | end 61 | function convert_from_graph_data(::Type{Union{Monocl.Hom,Nothing}}, data::AbstractDict) 62 | isempty(data) ? nothing : convert_from_graph_data(Monocl.Hom, data) 63 | end 64 | 65 | function convert_to_graph_data(expr::Monocl.Ob) 66 | Dict( 67 | "ob" => to_json_sexpr(expr), 68 | "label" => Dict( 69 | "text" => text_label(expr) 70 | ) 71 | ) 72 | end 73 | function convert_to_graph_data(expr::Monocl.Hom) 74 | Dict( 75 | "hom" => to_json_sexpr(expr), 76 | "label" => Dict( 77 | "text" => text_label(expr) 78 | ) 79 | ) 80 | end 81 | 82 | function convert_from_graph_data(::Type{MonoclElem}, data::AbstractDict) 83 | ob = haskey(data, "ob") ? 84 | parse_json_sexpr(Monocl, data["ob"]; symbols=false) : nothing 85 | value = get(data, "value", nothing) 86 | MonoclElem(ob, value) 87 | end 88 | 89 | function convert_to_graph_data(elem::MonoclElem) 90 | data = Dict{String,Any}() 91 | if !isnothing(elem.ob) 92 | data["ob"] = to_json_sexpr(elem.ob) 93 | end 94 | if !isnothing(elem.value) 95 | data["value"] = elem.value 96 | end 97 | data 98 | end 99 | 100 | """ Short, human-readable, plain text label for Moncl expression. 101 | """ 102 | text_label(expr::Monocl.Ob) = string(expr) 103 | text_label(expr::Monocl.Hom) = string(expr) 104 | text_label(expr::Monocl.Hom{:coerce}) = "coerce" 105 | text_label(expr::Monocl.Hom{:construct}) = string(codom(expr)) 106 | 107 | end 108 | -------------------------------------------------------------------------------- /test/data/datasets/iris.csv: -------------------------------------------------------------------------------- 1 | SepalLength,SepalWidth,PetalLength,PetalWidth,Species 2 | 5.1,3.5,1.4,0.2,setosa 3 | 4.9,3.0,1.4,0.2,setosa 4 | 4.7,3.2,1.3,0.2,setosa 5 | 4.6,3.1,1.5,0.2,setosa 6 | 5.0,3.6,1.4,0.2,setosa 7 | 5.4,3.9,1.7,0.4,setosa 8 | 4.6,3.4,1.4,0.3,setosa 9 | 5.0,3.4,1.5,0.2,setosa 10 | 4.4,2.9,1.4,0.2,setosa 11 | 4.9,3.1,1.5,0.1,setosa 12 | 5.4,3.7,1.5,0.2,setosa 13 | 4.8,3.4,1.6,0.2,setosa 14 | 4.8,3.0,1.4,0.1,setosa 15 | 4.3,3.0,1.1,0.1,setosa 16 | 5.8,4.0,1.2,0.2,setosa 17 | 5.7,4.4,1.5,0.4,setosa 18 | 5.4,3.9,1.3,0.4,setosa 19 | 5.1,3.5,1.4,0.3,setosa 20 | 5.7,3.8,1.7,0.3,setosa 21 | 5.1,3.8,1.5,0.3,setosa 22 | 5.4,3.4,1.7,0.2,setosa 23 | 5.1,3.7,1.5,0.4,setosa 24 | 4.6,3.6,1.0,0.2,setosa 25 | 5.1,3.3,1.7,0.5,setosa 26 | 4.8,3.4,1.9,0.2,setosa 27 | 5.0,3.0,1.6,0.2,setosa 28 | 5.0,3.4,1.6,0.4,setosa 29 | 5.2,3.5,1.5,0.2,setosa 30 | 5.2,3.4,1.4,0.2,setosa 31 | 4.7,3.2,1.6,0.2,setosa 32 | 4.8,3.1,1.6,0.2,setosa 33 | 5.4,3.4,1.5,0.4,setosa 34 | 5.2,4.1,1.5,0.1,setosa 35 | 5.5,4.2,1.4,0.2,setosa 36 | 4.9,3.1,1.5,0.1,setosa 37 | 5.0,3.2,1.2,0.2,setosa 38 | 5.5,3.5,1.3,0.2,setosa 39 | 4.9,3.1,1.5,0.1,setosa 40 | 4.4,3.0,1.3,0.2,setosa 41 | 5.1,3.4,1.5,0.2,setosa 42 | 5.0,3.5,1.3,0.3,setosa 43 | 4.5,2.3,1.3,0.3,setosa 44 | 4.4,3.2,1.3,0.2,setosa 45 | 5.0,3.5,1.6,0.6,setosa 46 | 5.1,3.8,1.9,0.4,setosa 47 | 4.8,3.0,1.4,0.3,setosa 48 | 5.1,3.8,1.6,0.2,setosa 49 | 4.6,3.2,1.4,0.2,setosa 50 | 5.3,3.7,1.5,0.2,setosa 51 | 5.0,3.3,1.4,0.2,setosa 52 | 7.0,3.2,4.7,1.4,versicolor 53 | 6.4,3.2,4.5,1.5,versicolor 54 | 6.9,3.1,4.9,1.5,versicolor 55 | 5.5,2.3,4.0,1.3,versicolor 56 | 6.5,2.8,4.6,1.5,versicolor 57 | 5.7,2.8,4.5,1.3,versicolor 58 | 6.3,3.3,4.7,1.6,versicolor 59 | 4.9,2.4,3.3,1.0,versicolor 60 | 6.6,2.9,4.6,1.3,versicolor 61 | 5.2,2.7,3.9,1.4,versicolor 62 | 5.0,2.0,3.5,1.0,versicolor 63 | 5.9,3.0,4.2,1.5,versicolor 64 | 6.0,2.2,4.0,1.0,versicolor 65 | 6.1,2.9,4.7,1.4,versicolor 66 | 5.6,2.9,3.6,1.3,versicolor 67 | 6.7,3.1,4.4,1.4,versicolor 68 | 5.6,3.0,4.5,1.5,versicolor 69 | 5.8,2.7,4.1,1.0,versicolor 70 | 6.2,2.2,4.5,1.5,versicolor 71 | 5.6,2.5,3.9,1.1,versicolor 72 | 5.9,3.2,4.8,1.8,versicolor 73 | 6.1,2.8,4.0,1.3,versicolor 74 | 6.3,2.5,4.9,1.5,versicolor 75 | 6.1,2.8,4.7,1.2,versicolor 76 | 6.4,2.9,4.3,1.3,versicolor 77 | 6.6,3.0,4.4,1.4,versicolor 78 | 6.8,2.8,4.8,1.4,versicolor 79 | 6.7,3.0,5.0,1.7,versicolor 80 | 6.0,2.9,4.5,1.5,versicolor 81 | 5.7,2.6,3.5,1.0,versicolor 82 | 5.5,2.4,3.8,1.1,versicolor 83 | 5.5,2.4,3.7,1.0,versicolor 84 | 5.8,2.7,3.9,1.2,versicolor 85 | 6.0,2.7,5.1,1.6,versicolor 86 | 5.4,3.0,4.5,1.5,versicolor 87 | 6.0,3.4,4.5,1.6,versicolor 88 | 6.7,3.1,4.7,1.5,versicolor 89 | 6.3,2.3,4.4,1.3,versicolor 90 | 5.6,3.0,4.1,1.3,versicolor 91 | 5.5,2.5,4.0,1.3,versicolor 92 | 5.5,2.6,4.4,1.2,versicolor 93 | 6.1,3.0,4.6,1.4,versicolor 94 | 5.8,2.6,4.0,1.2,versicolor 95 | 5.0,2.3,3.3,1.0,versicolor 96 | 5.6,2.7,4.2,1.3,versicolor 97 | 5.7,3.0,4.2,1.2,versicolor 98 | 5.7,2.9,4.2,1.3,versicolor 99 | 6.2,2.9,4.3,1.3,versicolor 100 | 5.1,2.5,3.0,1.1,versicolor 101 | 5.7,2.8,4.1,1.3,versicolor 102 | 6.3,3.3,6.0,2.5,virginica 103 | 5.8,2.7,5.1,1.9,virginica 104 | 7.1,3.0,5.9,2.1,virginica 105 | 6.3,2.9,5.6,1.8,virginica 106 | 6.5,3.0,5.8,2.2,virginica 107 | 7.6,3.0,6.6,2.1,virginica 108 | 4.9,2.5,4.5,1.7,virginica 109 | 7.3,2.9,6.3,1.8,virginica 110 | 6.7,2.5,5.8,1.8,virginica 111 | 7.2,3.6,6.1,2.5,virginica 112 | 6.5,3.2,5.1,2.0,virginica 113 | 6.4,2.7,5.3,1.9,virginica 114 | 6.8,3.0,5.5,2.1,virginica 115 | 5.7,2.5,5.0,2.0,virginica 116 | 5.8,2.8,5.1,2.4,virginica 117 | 6.4,3.2,5.3,2.3,virginica 118 | 6.5,3.0,5.5,1.8,virginica 119 | 7.7,3.8,6.7,2.2,virginica 120 | 7.7,2.6,6.9,2.3,virginica 121 | 6.0,2.2,5.0,1.5,virginica 122 | 6.9,3.2,5.7,2.3,virginica 123 | 5.6,2.8,4.9,2.0,virginica 124 | 7.7,2.8,6.7,2.0,virginica 125 | 6.3,2.7,4.9,1.8,virginica 126 | 6.7,3.3,5.7,2.1,virginica 127 | 7.2,3.2,6.0,1.8,virginica 128 | 6.2,2.8,4.8,1.8,virginica 129 | 6.1,3.0,4.9,1.8,virginica 130 | 6.4,2.8,5.6,2.1,virginica 131 | 7.2,3.0,5.8,1.6,virginica 132 | 7.4,2.8,6.1,1.9,virginica 133 | 7.9,3.8,6.4,2.0,virginica 134 | 6.4,2.8,5.6,2.2,virginica 135 | 6.3,2.8,5.1,1.5,virginica 136 | 6.1,2.6,5.6,1.4,virginica 137 | 7.7,3.0,6.1,2.3,virginica 138 | 6.3,3.4,5.6,2.4,virginica 139 | 6.4,3.1,5.5,1.8,virginica 140 | 6.0,3.0,4.8,1.8,virginica 141 | 6.9,3.1,5.4,2.1,virginica 142 | 6.7,3.1,5.6,2.4,virginica 143 | 6.9,3.1,5.1,2.3,virginica 144 | 5.8,2.7,5.1,1.9,virginica 145 | 6.8,3.2,5.9,2.3,virginica 146 | 6.7,3.3,5.7,2.5,virginica 147 | 6.7,3.0,5.2,2.3,virginica 148 | 6.3,2.5,5.0,1.9,virginica 149 | 6.5,3.0,5.2,2.0,virginica 150 | 6.2,3.4,5.4,2.3,virginica 151 | 5.9,3.0,5.1,1.8,virginica 152 | -------------------------------------------------------------------------------- /src/ontology/OntologyJSON.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module OntologyJSON 16 | export presentation_from_json, annotation_from_json 17 | 18 | using Catlab 19 | using ...Doctrine, ..Ontology 20 | 21 | # Concepts 22 | ########## 23 | 24 | """ Load Monocl concepts (as a presentation) from JSON documents. 25 | """ 26 | function presentation_from_json(docs)::Presentation 27 | # Be careful about load order because: 28 | # - To add a morphism, the domain/codomain objects must be added 29 | # - To add a subobject, the domain/codomain objects must already be added 30 | # - To add a submorphism, the domain/codomain morphisms must already be added 31 | presentation = Presentation{String}(Monocl) 32 | ob_docs = filter(doc -> doc["kind"] == "type", docs) 33 | hom_docs = filter(doc -> doc["kind"] == "function", docs) 34 | for doc in ob_docs 35 | add_ob_generator_from_json!(presentation, doc) 36 | end 37 | for doc in ob_docs 38 | add_subob_generators_from_json!(presentation, doc) 39 | end 40 | for doc in hom_docs 41 | add_hom_generator_from_json!(presentation, doc) 42 | end 43 | for doc in hom_docs 44 | add_subhom_generators_from_json!(presentation, doc) 45 | end 46 | presentation 47 | end 48 | 49 | """ Add object from JSON document to presentation. 50 | """ 51 | function add_ob_generator_from_json!(pres::Presentation, doc::AbstractDict) 52 | ob = Ob(Monocl, doc["id"]) 53 | add_generator!(pres, ob) 54 | end 55 | 56 | """ Add subobjects from JSON document to presentation. 57 | """ 58 | function add_subob_generators_from_json!(pres::Presentation, doc::AbstractDict) 59 | ob = generator(pres, doc["id"])::Monocl.Ob 60 | names = get(doc, "is-a", []) 61 | names = isa(names, AbstractString) ? [ names ] : names 62 | for super_name in names 63 | super_ob = generator(pres, super_name)::Monocl.Ob 64 | add_generator!(pres, SubOb(ob, super_ob)) 65 | end 66 | end 67 | 68 | """ Add morphism from JSON document to presentation. 69 | """ 70 | function add_hom_generator_from_json!(pres::Presentation, doc::AbstractDict) 71 | dom_ob = domain_ob_from_json(pres, doc["inputs"]) 72 | codom_ob = domain_ob_from_json(pres, doc["outputs"]) 73 | hom = Hom(doc["id"], dom_ob, codom_ob) 74 | add_generator!(pres, hom) 75 | end 76 | 77 | """ Add submorphisms from JSON document to presentation. 78 | """ 79 | function add_subhom_generators_from_json!(pres::Presentation, doc::AbstractDict) 80 | hom = generator(pres, doc["id"])::Monocl.Hom 81 | names = get(doc, "is-a", []) 82 | names = isa(names, AbstractString) ? [ names ] : names 83 | for super_name in names 84 | super_hom = generator(pres, super_name)::Monocl.Hom 85 | # FIXME: Do basic type inference to check these subobject relations hold. 86 | subob_dom = SubOb(dom(hom), dom(super_hom)) 87 | subob_codom = SubOb(codom(hom), codom(super_hom)) 88 | add_generator!(pres, SubHom(hom, super_hom, subob_dom, subob_codom)) 89 | end 90 | end 91 | 92 | function domain_ob_from_json(pres::Presentation, docs)::Monocl.Ob 93 | if isempty(docs) 94 | munit(Monocl.Ob) 95 | else 96 | otimes([ generator(pres, doc["type"]) for doc in docs ]) 97 | end 98 | end 99 | 100 | # Annotations 101 | ############# 102 | 103 | const language_keys = [ 104 | "class", "function", "method", "inputs", "outputs", "slots", 105 | ] 106 | 107 | """ Load annotation from JSON document. 108 | """ 109 | function annotation_from_json(doc::AbstractDict, load_ref::Function)::Annotation 110 | parse_def = sexpr -> parse_json_sexpr(Monocl, sexpr; 111 | symbols=false, parse_head=parse_json_sexpr_term, parse_reference=load_ref) 112 | name = AnnotationID(doc["language"], doc["package"], doc["id"]) 113 | lang = Dict{Symbol,Any}( 114 | Symbol(key) => doc[key] for key in language_keys if haskey(doc, key) 115 | ) 116 | definition = parse_def(doc["definition"]) 117 | if doc["kind"] == "type" 118 | slots = [ parse_def(slot["definition"]) for slot in get(doc, "slots", []) ] 119 | ObAnnotation(name, lang, definition, slots) 120 | elseif doc["kind"] == "function" 121 | HomAnnotation(name, lang, definition) 122 | else 123 | error("Invalid kind of annotation: $(doc["kind"])") 124 | end 125 | end 126 | function annotation_from_json(doc::AbstractDict, pres::Presentation) 127 | annotation_from_json(doc, name -> generator(pres, name)) 128 | end 129 | 130 | """ Replace term names (S-exp head) in JSON S-expression. 131 | 132 | Translates PLT terminology into category theory terminology. 133 | """ 134 | function parse_json_sexpr_term(x::String) 135 | get(json_sexpr_term_table, x, x) 136 | end 137 | 138 | const json_sexpr_term_table = Dict( 139 | "Type" => "Ob", 140 | "Function" => "Hom", 141 | "Subtype" => "SubOb", 142 | "Subfunction" => "SubHom", 143 | "product" => "otimes", 144 | "unit" => "munit", 145 | "swap" => "braid", 146 | "copy" => "mcopy", 147 | "apply" => "pair", 148 | ) 149 | 150 | end 151 | -------------------------------------------------------------------------------- /src/ontology/rdf/AnnotationRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module AnnotationRDF 16 | export annotation_to_rdf, annotation_rdf_node 17 | 18 | using Serd 19 | using Catlab, Catlab.WiringDiagrams 20 | 21 | using ...Doctrine, ...Ontology 22 | using ..RDFUtils 23 | using ..ConceptRDF: generator_rdf_node 24 | using ..WiringRDF 25 | 26 | const R = RDF.Resource 27 | 28 | # Constants 29 | ########### 30 | 31 | const language_properties = Dict( 32 | :class => "annotatedClass", 33 | :function => "annotatedFunction", 34 | :method => "annotatedMethod", 35 | ) 36 | 37 | # RDF 38 | ##### 39 | 40 | """ Convert annotation into triples for RDF/OWL ontology. 41 | """ 42 | function annotation_to_rdf(annotation::ObAnnotation, prefix::RDF.Prefix; kw...) 43 | node = annotation_rdf_node(annotation, prefix) 44 | stmts = RDF.Statement[ 45 | [ RDF.Triple(node, R("rdf","type"), R("monocl","TypeAnnotation")) ]; 46 | annotation_name_to_rdf(annotation, prefix); 47 | annotation_language_to_rdf(annotation, prefix); 48 | ] 49 | 50 | # Definition as expression, assuming it's a basic object. 51 | gen_node = generator_rdf_node(annotation.definition, prefix) 52 | push!(stmts, RDF.Triple(node, R("monocl","codeDefinition"), gen_node)) 53 | 54 | # Slot annotations. 55 | for (i, hom) in enumerate(annotation.slots) 56 | slot = annotation.language[:slots][i]["slot"] 57 | slot_node = R(prefix.name, "$(node.name):slot$i") 58 | append!(stmts, [ 59 | RDF.Triple(node, R("monocl","annotatedSlot"), slot_node), 60 | RDF.Triple(slot_node, R("rdf","type"), R("monocl","SlotAnnotation")), 61 | RDF.Triple(slot_node, R("monocl","codeSlot"), RDF.Literal(slot)), 62 | ]) 63 | if head(hom) == :generator 64 | gen_node = generator_rdf_node(hom, prefix) 65 | push!(stmts, RDF.Triple(slot_node, R("monocl","codeDefinition"), gen_node)) 66 | end 67 | end 68 | 69 | stmts 70 | end 71 | 72 | function annotation_to_rdf(annotation::HomAnnotation, prefix::RDF.Prefix; 73 | include_provenance::Bool=true, 74 | include_wiring_diagrams::Bool=true) 75 | node = annotation_rdf_node(annotation, prefix) 76 | stmts = RDF.Statement[ 77 | [ RDF.Triple(node, R("rdf","type"), R("monocl","FunctionAnnotation")) ]; 78 | annotation_name_to_rdf(annotation, prefix); 79 | annotation_language_to_rdf(annotation, prefix); 80 | annotation_domain_to_rdf(annotation, prefix); 81 | ] 82 | 83 | # Definition as expression, if it's a basic morphism. 84 | if head(annotation.definition) == :generator 85 | gen_node = generator_rdf_node(annotation.definition, prefix) 86 | push!(stmts, RDF.Triple(node, R("monocl","codeDefinition"), gen_node)) 87 | end 88 | 89 | # Definition as wiring diagram. 90 | if include_wiring_diagrams 91 | diagram = to_wiring_diagram(annotation.definition) 92 | diagram_name = "$(node.name):diagram" 93 | root_node, diagram_stmts = semantic_graph_to_rdf(diagram, 94 | expr -> generator_rdf_node(expr, prefix); 95 | box_rdf_node = box -> R(prefix.name, "$diagram_name:$box"), 96 | port_rdf_node = (box, port) -> R(prefix.name, "$diagram_name:$box:$port"), 97 | wire_rdf_node = wire -> R(prefix.name, "$diagram_name:$wire"), 98 | include_provenance = include_provenance) 99 | append!(stmts, [ 100 | [ RDF.Triple(node, R("monocl","codeDefinition"), root_node) ]; 101 | diagram_stmts; 102 | ]) 103 | end 104 | 105 | stmts 106 | end 107 | 108 | """ Convert annotation's name to RDF. 109 | """ 110 | function annotation_name_to_rdf(annotation::Annotation, prefix::RDF.Prefix) 111 | node = annotation_rdf_node(annotation, prefix) 112 | name = annotation.name 113 | RDF.Statement[ 114 | RDF.Triple(node, R("monocl","annotatedLanguage"), RDF.Literal(name.language)), 115 | RDF.Triple(node, R("monocl","annotatedPackage"), RDF.Literal(name.package)), 116 | RDF.Triple(node, R("monocl","id"), RDF.Literal(name.id)), 117 | ] 118 | end 119 | 120 | """ Convert annotation's language-specific data to RDF. 121 | """ 122 | function annotation_language_to_rdf(annotation::Annotation, prefix::RDF.Prefix) 123 | node = annotation_rdf_node(annotation, prefix) 124 | stmts = RDF.Statement[] 125 | for key in intersect(keys(language_properties), keys(annotation.language)) 126 | value = annotation.language[key] 127 | values = value isa AbstractArray ? value : [ value ] 128 | append!(stmts, [ 129 | RDF.Triple(node, R("monocl", language_properties[key]), RDF.Literal(v)) 130 | for v in values 131 | ]) 132 | end 133 | stmts 134 | end 135 | 136 | """ Convert annotation's language-specific domain and codomain data to RDF. 137 | """ 138 | function annotation_domain_to_rdf(annotation::HomAnnotation, prefix::RDF.Prefix) 139 | node = annotation_rdf_node(annotation, prefix) 140 | cell_node = name -> R(prefix.name, "$(node.name):$name") 141 | inputs, outputs = annotation.language[:inputs], annotation.language[:outputs] 142 | nin, nout = length(inputs), length(outputs) 143 | owl_inputs_outputs(node, cell_node, nin, nout, index=true) do cell, is_input, i 144 | data = (is_input ? inputs : outputs)[i] 145 | slot = data["slot"] 146 | [ RDF.Triple(cell, R("monocl","codeSlot"), RDF.Literal(slot)) ] 147 | end 148 | end 149 | 150 | """ Create RDF node for annotation. 151 | """ 152 | function annotation_rdf_node(annotation::Annotation, prefix::RDF.Prefix)::RDF.Node 153 | name = annotation.name 154 | node_name = join([name.language, name.package, name.id], ":") 155 | R(prefix.name, node_name) 156 | end 157 | 158 | end 159 | -------------------------------------------------------------------------------- /src/ontology/OntologyDBs.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module OntologyDBs 16 | export OntologyDB, OntologyError, 17 | concept, concept_document, concepts, has_concept, 18 | annotation, annotation_document, annotations, has_annotation, 19 | load_concepts, load_annotation, load_annotations, load_ontology_file 20 | 21 | using Compat 22 | using DataStructures: OrderedDict 23 | import JSON, HTTP 24 | 25 | using Catlab 26 | using ...Doctrine, ..Ontology 27 | 28 | const api_url_default = "https://api.datascienceontology.org" 29 | 30 | # Data types 31 | ############ 32 | 33 | """ Ontology database, containing concepts and annotations. 34 | """ 35 | mutable struct OntologyDB 36 | api_url::String 37 | concepts::Presentation 38 | concept_docs::OrderedDict{String,AbstractDict} 39 | annotations::OrderedDict{String,Annotation} 40 | annotation_docs::OrderedDict{String,AbstractDict} 41 | 42 | function OntologyDB(api_url) 43 | new(api_url, Presentation{String}(Monocl), 44 | OrderedDict(), OrderedDict(), OrderedDict()) 45 | end 46 | end 47 | OntologyDB() = OntologyDB(api_url_default) 48 | 49 | struct OntologyError <: Exception 50 | message::String 51 | end 52 | 53 | # Ontology accessors 54 | #################### 55 | 56 | function concept(db::OntologyDB, id::String) 57 | if !has_generator(db.concepts, id) 58 | throw(OntologyError("No concept named '$id'")) 59 | end 60 | generator(db.concepts, id) 61 | end 62 | 63 | concept_document(db::OntologyDB, id::String) = db.concept_docs[id] 64 | has_concept(db::OntologyDB, id::String) = has_generator(db.concepts, id) 65 | 66 | concepts(db::OntologyDB) = db.concepts 67 | concepts(db::OntologyDB, ids) = [ concept(db, id) for id in ids ] 68 | 69 | function annotation(db::OntologyDB, id) 70 | doc_id = annotation_document_id(id) 71 | if !haskey(db.annotations, doc_id) 72 | throw(OntologyError("No annotation named '$id'")) 73 | end 74 | db.annotations[doc_id] 75 | end 76 | 77 | function annotation_document(db::OntologyDB, id) 78 | db.annotation_docs[annotation_document_id(id)] 79 | end 80 | 81 | function has_annotation(db::OntologyDB, id) 82 | haskey(db.annotations, annotation_document_id(id)) 83 | end 84 | 85 | annotations(db::OntologyDB) = values(db.annotations) 86 | annotations(db::OntologyDB, ids) = [ annotation(db, id) for id in ids ] 87 | 88 | function annotation_document_id(id::String) 89 | startswith(id, "annotation/") ? id : "annotation/$id" 90 | end 91 | function annotation_document_id(id::AnnotationID) 92 | join(["annotation", id.language, id.package, id.id], "/") 93 | end 94 | 95 | # Local file 96 | ############ 97 | 98 | """ Load concepts/annotations from a list of JSON documents. 99 | """ 100 | function load_documents(db::OntologyDB, docs) 101 | concept_docs = filter(doc -> doc["schema"] == "concept", docs) 102 | merge_presentation!(db.concepts, presentation_from_json(concept_docs)) 103 | merge!(db.concept_docs, OrderedDict(doc["id"] => doc for doc in concept_docs)) 104 | 105 | annotation_docs = filter(doc -> doc["schema"] == "annotation", docs) 106 | load_reference = id -> concept(db, id) 107 | for doc in annotation_docs 108 | db.annotations[doc["_id"]] = annotation_from_json(doc, load_reference) 109 | db.annotation_docs[doc["_id"]] = doc 110 | end 111 | end 112 | 113 | """ Load concepts/annotations from a local JSON file. 114 | """ 115 | function load_ontology_file(db::OntologyDB, filename::String) 116 | open(filename) do file 117 | load_ontology_file(db, file) 118 | end 119 | end 120 | function load_ontology_file(db::OntologyDB, io::IO) 121 | load_documents(db, JSON.parse(io)::Vector) 122 | end 123 | 124 | """ Merge the second presentation into the first. 125 | 126 | The first presentation is mutated and returned; the second is not. 127 | """ 128 | function merge_presentation!(pres::Presentation, other::Presentation) 129 | for expr in generators(other) 130 | name = first(expr) 131 | if isnothing(name) || !has_generator(pres, name) 132 | add_generator!(pres, expr) 133 | end 134 | end 135 | union!(pres.equations, other.equations) 136 | return pres 137 | end 138 | 139 | # Remote database 140 | ################# 141 | 142 | """ Load all concepts in ontology from remote database. 143 | """ 144 | function load_concepts(db::OntologyDB; ids=nothing) 145 | load_documents(db, api_get(db, "/concepts")) 146 | end 147 | 148 | """ Load annotations in ontology from remote database. 149 | """ 150 | function load_annotations(db::OntologyDB; language=nothing, package=nothing) 151 | endpoint = if isnothing(language) 152 | "/annotations" 153 | elseif isnothing(package) 154 | "/annotations/$language" 155 | else 156 | "/annotations/$language/$package" 157 | end 158 | load_documents(db, api_get(db, endpoint)) 159 | end 160 | 161 | """ Load single annotation from remote database, if it's not already loaded. 162 | """ 163 | function load_annotation(db::OntologyDB, id)::Annotation 164 | if has_annotation(db, id) 165 | return annotation(db, id) 166 | end 167 | doc = try 168 | api_get(db, "/$(annotation_document_id(id))") 169 | catch err 170 | if isa(err, HTTP.StatusError) && err.status == 404 171 | throw(OntologyError("No annotation named '$id'")) 172 | end 173 | rethrow() 174 | end 175 | load_documents(db, [doc]) 176 | annotation(db, id) 177 | end 178 | 179 | # REST API client 180 | ################# 181 | 182 | function api_get(api_url::String, endpoint::String) 183 | response = HTTP.get(string(api_url, endpoint)) 184 | JSON.parse(String(response.body), dicttype=OrderedDict) 185 | end 186 | 187 | api_get(db::OntologyDB, endpoint::String) = api_get(db.api_url, endpoint) 188 | 189 | end 190 | -------------------------------------------------------------------------------- /test/SemanticEnrichment.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module TestSemanticEnrichment 16 | using Test 17 | 18 | using Catlab.WiringDiagrams 19 | using SemanticFlowGraphs 20 | 21 | const db_filename = joinpath(@__DIR__, "ontology", "data", "employee.json") 22 | db = OntologyDB() 23 | load_ontology_file(db, db_filename) 24 | 25 | # Convenience methods to make raw boxes and ports with or without annotations. 26 | const prefix = "opendisc/employee" 27 | add_raw_box!(f::WiringDiagram, args...; kw...) = 28 | add_box!(f, raw_box(args...; kw...)) 29 | raw_box(inputs, outputs; kw...) = 30 | Box(RawNode(; kw...), raw_ports(inputs), raw_ports(outputs)) 31 | raw_box(name::String, inputs, outputs) = 32 | Box(RawNode(annotation="$prefix/$name"), raw_ports(inputs), raw_ports(outputs)) 33 | raw_ports(n::Int) = RawPort[ RawPort() for i in 1:n ] 34 | raw_ports(xs::Vector) = RawPort[ raw_port(x) for x in xs ] 35 | raw_port(::Nothing) = RawPort() 36 | raw_port(name::String) = RawPort(annotation="$prefix/$name") 37 | raw_port(name::String, index::Int) = 38 | RawPort(annotation="$prefix/$name", annotation_index=index) 39 | raw_port(args::Tuple) = raw_port(args...) 40 | 41 | # Expand single annotated box. 42 | f = WiringDiagram(raw_ports(["employee"]), raw_ports(["employee"])) 43 | v = add_raw_box!(f, "manager", [("employee",1)], [("employee",1)]) 44 | add_wires!(f, [ 45 | (input_id(f),1) => (v,1), 46 | (v,1) => (output_id(f),1) 47 | ]) 48 | actual = to_semantic_graph(db, f) 49 | target = WiringDiagram(concepts(db, ["employee"]), concepts(db, ["employee"])) 50 | reports_to = add_box!(target, Box(concept(db, "reports-to"))) 51 | add_wires!(target, [ 52 | (input_id(target), 1) => (reports_to, 1), 53 | (reports_to, 1) => (output_id(target), 1), 54 | ]) 55 | @test actual == target 56 | 57 | # Expand slots. 58 | f = WiringDiagram(raw_ports(["employee"]), raw_ports(["department","str","str"])) 59 | manager = add_raw_box!(f, "manager", [("employee",1)], [("employee",1)]) 60 | dept = add_raw_box!(f, "employee-department", [("employee",1)], [("department",1)]) 61 | first_name = add_raw_box!(f, [("employee",1)], [("str",1)], 62 | annotation="$prefix/employee", 63 | annotation_index=1, annotation_kind=SlotAnnotation) 64 | last_name = add_raw_box!(f, [("employee",1)], [("str",1)], 65 | annotation="$prefix/employee", 66 | annotation_index=2, annotation_kind=SlotAnnotation) 67 | add_wires!(f, [ 68 | (input_id(f), 1) => (manager, 1), 69 | (manager, 1) => (dept, 1), 70 | (manager, 1) => (first_name, 1), 71 | (manager, 1) => (last_name, 1), 72 | (dept, 1) => (output_id(f), 1), 73 | (first_name, 1) => (output_id(f), 2), 74 | (last_name, 1) => (output_id(f), 3), 75 | ]) 76 | actual = to_semantic_graph(db, f) 77 | target = WiringDiagram(concepts(db, ["employee"]), 78 | concepts(db, ["department", "string", "string"])) 79 | reports_to = add_box!(target, Box(concept(db, "reports-to"))) 80 | works_in = add_box!(target, Box(concept(db, "works-in"))) 81 | first_name = add_box!(target, Box(concept(db, "person-first-name"))) 82 | last_name = add_box!(target, Box(concept(db, "person-last-name"))) 83 | add_wires!(target, [ 84 | (input_id(target), 1) => (reports_to, 1), 85 | (reports_to, 1) => (works_in, 1), 86 | (reports_to, 1) => (first_name, 1), 87 | (reports_to, 1) => (last_name, 1), 88 | (works_in, 1) => (output_id(target), 1), 89 | (first_name, 1) => (output_id(target), 2), 90 | (last_name, 1) => (output_id(target), 3), 91 | ]) 92 | @test actual == target 93 | 94 | # Collapse adjacent unannotated boxes. 95 | f = WiringDiagram(raw_ports(1), raw_ports(1)) 96 | u = add_raw_box!(f, 1, 1) 97 | v = add_raw_box!(f, 1, 1) 98 | add_wires!(f, [ 99 | (input_id(f),1) => (u,1), 100 | (u,1) => (v,1), 101 | (v,1) => (output_id(f),1) 102 | ]) 103 | actual = to_semantic_graph(db, f) 104 | target = WiringDiagram([nothing], [nothing]) 105 | v = add_box!(target, Box(nothing, [nothing], [nothing])) 106 | add_wires!(target, [ 107 | (input_id(target), 1) => (v,1), 108 | (v,1) => (output_id(target), 1), 109 | ]) 110 | @test actual == target 111 | 112 | # Don't collapse adjacent unannotated boxes with an intermediate annotated box. 113 | f = WiringDiagram(raw_ports(0), raw_ports(0)) 114 | u = add_raw_box!(f, [], [nothing, "employee"]) 115 | v = add_raw_box!(f, [nothing, "employee"], []) 116 | manager = add_raw_box!(f, "manager", [("employee",1)], [("employee",1)]) 117 | add_wires!(f, [ 118 | (u,1) => (v,1), 119 | (u,2) => (manager,1), 120 | (manager,1) => (v,2), 121 | ]) 122 | actual = to_semantic_graph(db, f) 123 | target = WiringDiagram([], []) 124 | u = add_box!(target, Box(nothing, [], [nothing, concept(db,"employee")])) 125 | v = add_box!(target, Box(nothing, [nothing, concept(db,"employee")], [])) 126 | reports_to = add_box!(target, Box(concept(db,"reports-to"))) 127 | add_wires!(target, [ 128 | (u,1) => (v,1), 129 | (u,2) => (reports_to,1), 130 | (reports_to,1) => (v,2), 131 | ]) 132 | @test actual == target 133 | 134 | # Don't assume that collapsibility is a transitive relation. 135 | f = WiringDiagram(raw_ports(0), raw_ports(0)) 136 | u = add_raw_box!(f, [], [nothing, "employee"]) 137 | v = add_raw_box!(f, 1, 1) 138 | w = add_raw_box!(f, [nothing, "employee"], []) 139 | manager = add_raw_box!(f, "manager", [("employee",1)], [("employee",1)]) 140 | add_wires!(f, [ 141 | (u,1) => (v,1), 142 | (v,1) => (w,1), 143 | (u,2) => (manager,1), 144 | (manager,1) => (w,2), 145 | ]) 146 | actual = to_semantic_graph(db, f) 147 | target = WiringDiagram([], []) 148 | u = add_box!(target, Box(nothing, [], [concept(db,"employee"), nothing])) 149 | v = add_box!(target, Box(nothing, [nothing, concept(db,"employee")], [])) 150 | reports_to = add_box!(target, Box(concept(db,"reports-to"))) 151 | add_wires!(target, [ 152 | (u,2) => (v,1), 153 | (u,1) => (reports_to,1), 154 | (reports_to,1) => (v,2), 155 | ]) 156 | @test actual == target 157 | 158 | end 159 | -------------------------------------------------------------------------------- /src/ontology/rdf/WiringRDF.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module WiringRDF 16 | export wiring_diagram_to_rdf, semantic_graph_to_rdf 17 | 18 | using Compat 19 | using Parameters 20 | using Serd 21 | 22 | using Catlab 23 | using Catlab.WiringDiagrams, Catlab.WiringDiagrams.WiringDiagramSerialization 24 | using ...Doctrine 25 | using ..RDFUtils 26 | 27 | const R = RDF.Resource 28 | 29 | # Configuration 30 | ############### 31 | 32 | default_box_rdf_node(box::String) = RDF.Blank(box) 33 | default_port_rdf_node(box::String, port::String) = RDF.Blank("$box:$port") 34 | default_wire_rdf_node(wire::String) = RDF.Blank(wire) 35 | 36 | function default_value_to_rdf(node::RDF.Node, value) 37 | if isnothing(value) 38 | RDF.Statement[] 39 | else 40 | [ RDF.Triple(node, R("monocl","value"), RDF.Literal(string(value))) ] 41 | end 42 | end 43 | 44 | @with_kw struct RDFConfig 45 | box_rdf_node::Function = default_box_rdf_node 46 | port_rdf_node::Function = default_port_rdf_node 47 | wire_rdf_node::Function = default_wire_rdf_node 48 | box_value_to_rdf::Function = default_value_to_rdf 49 | port_value_to_rdf::Function = default_value_to_rdf 50 | wire_value_to_rdf::Function = default_value_to_rdf 51 | include_provenance::Bool = false 52 | end 53 | 54 | # Wiring diagrams 55 | ################# 56 | 57 | """ Convert a wiring diagram to RDF. 58 | """ 59 | function wiring_diagram_to_rdf(diagram::WiringDiagram; kw...) 60 | config = RDFConfig(; kw...) 61 | box_to_rdf(config, diagram, Int[]) 62 | end 63 | 64 | function box_to_rdf(config::RDFConfig, diagram::WiringDiagram, path::Vector{Int}) 65 | node = config.box_rdf_node(box_id(path)) 66 | stmts = RDF.Statement[ 67 | [ RDF.Triple(node, R("rdf","type"), R("monocl","WiringDiagram")) ]; 68 | ports_to_rdf(config, diagram, path); 69 | ] 70 | 71 | # Add RDF for boxes. 72 | for v in box_ids(diagram) 73 | box_node, box_stmts = box_to_rdf(config, box(diagram, v), [path; v]) 74 | push!(stmts, RDF.Triple(node, R("monocl","hasBox"), box_node)) 75 | append!(stmts, box_stmts) 76 | end 77 | 78 | # Add RDF for wires. 79 | box_rdf_node = port::Port -> config.box_rdf_node( 80 | box_id(diagram, [path; port.box])) 81 | port_rdf_node = port::Port -> config.port_rdf_node( 82 | box_id(diagram, [path; port.box]), port_name(diagram, port)) 83 | 84 | for (i, wire) in enumerate(wires(diagram)) 85 | wire_node = config.wire_rdf_node(wire_id(path, i)) 86 | src_port_node = port_rdf_node(wire.source) 87 | tgt_port_node = port_rdf_node(wire.target) 88 | append!(stmts, [ 89 | RDF.Triple(node, R("monocl","hasWire"), wire_node), 90 | RDF.Triple(wire_node, R("rdf","type"), R("monocl","Wire")), 91 | RDF.Triple(wire_node, R("monocl","source"), src_port_node), 92 | RDF.Triple(wire_node, R("monocl","target"), tgt_port_node), 93 | RDF.Triple(src_port_node, R("monocl","wire"), tgt_port_node), 94 | ]) 95 | append!(stmts, config.wire_value_to_rdf(wire_node, wire.value)) 96 | if config.include_provenance 97 | src_box_node = box_rdf_node(wire.source) 98 | tgt_box_node = box_rdf_node(wire.target) 99 | append!(stmts, [ 100 | RDF.Triple(tgt_box_node, R("prov","wasInformedBy"), src_box_node), 101 | RDF.Triple(tgt_port_node, R("prov","wasDerivedFrom"), src_port_node), 102 | ]) 103 | end 104 | end 105 | 106 | (node, stmts) 107 | end 108 | 109 | function box_to_rdf(config::RDFConfig, box::Box, path::Vector{Int}) 110 | node = config.box_rdf_node(box_id(path)) 111 | stmts = RDF.Statement[ 112 | [ RDF.Triple(node, R("rdf","type"), R("monocl","Box")) ]; 113 | config.box_value_to_rdf(node, box.value); 114 | ports_to_rdf(config, box, path); 115 | ] 116 | (node, stmts) 117 | end 118 | 119 | function ports_to_rdf(config::RDFConfig, box::AbstractBox, path::Vector{Int}) 120 | name = box_id(path) 121 | node = config.box_rdf_node(name) 122 | port_node = port -> config.port_rdf_node(name, port) 123 | inputs, outputs = input_ports(box), output_ports(box) 124 | nin, nout = length(inputs), length(outputs) 125 | owl_inputs_outputs(node, port_node, nin, nout, index=true) do cell, is_input, i 126 | port_value = (is_input ? inputs : outputs)[i] 127 | RDF.Statement[ 128 | [ RDF.Triple(cell, R("rdf","type"), R("monocl","Port")) ]; 129 | config.port_value_to_rdf(cell, port_value); 130 | if config.include_provenance 131 | if is_input 132 | [ RDF.Triple(node, R("prov","used"), cell) ] 133 | else 134 | [ RDF.Triple(cell, R("prov","wasGeneratedBy"), node) ] 135 | end 136 | else [] end; 137 | ] 138 | end 139 | end 140 | 141 | # Semantic flow graphs 142 | ###################### 143 | 144 | """ Convert a semantic flow graph to RDF. 145 | """ 146 | function semantic_graph_to_rdf(diagram::WiringDiagram, concept_rdf_node::Function; 147 | include_provenance::Bool=false, kw...) 148 | 149 | function box_expr_to_rdf(node::RDF.Node, expr::Monocl.Hom) 150 | concept = if head(expr) == :generator 151 | concept_rdf_node(expr) 152 | elseif head(expr) == :construct 153 | concept_rdf_node(codom(expr)) 154 | else 155 | error("Cannot serialize Monocl morphism of type: ", head(expr)) 156 | end 157 | RDF.Statement[ 158 | [ RDF.Triple(node, R("monocl","isConcept"), concept) ]; 159 | include_provenance && head(expr) == :generator ? 160 | [ RDF.Triple(node, R("rdf","type"), concept) ] : []; 161 | ] 162 | end 163 | 164 | function port_expr_to_rdf(node::RDF.Node, expr::Monocl.Ob) 165 | concept = concept_rdf_node(expr) 166 | RDF.Statement[ 167 | [ RDF.Triple(node, R("monocl","isConcept"), concept) ]; 168 | include_provenance ? 169 | [ RDF.Triple(node, R("rdf","type"), concept) ] : []; 170 | ] 171 | end 172 | 173 | wiring_diagram_to_rdf(diagram; 174 | box_value_to_rdf = box_expr_to_rdf, 175 | port_value_to_rdf = port_expr_to_rdf, 176 | include_provenance = include_provenance, 177 | kw...) 178 | end 179 | 180 | end 181 | -------------------------------------------------------------------------------- /test/ontology/data/employee.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "concept/person", 4 | "schema": "concept", 5 | "kind": "type", 6 | "id": "person", 7 | "name": "Person" 8 | }, 9 | { 10 | "_id": "concept/employee", 11 | "schema": "concept", 12 | "kind": "type", 13 | "id": "employee", 14 | "name": "Employee", 15 | "description": "Employee of an organization", 16 | "is-a": "person" 17 | }, 18 | { 19 | "_id": "concept/manager", 20 | "schema": "concept", 21 | "kind": "type", 22 | "id": "manager", 23 | "name": "Manager", 24 | "description": "Employee who manages other employees", 25 | "is-a": "employee" 26 | }, 27 | { 28 | "_id": "concept/department", 29 | "schema": "concept", 30 | "kind": "type", 31 | "id": "department", 32 | "name": "Department", 33 | "description": "Department in an organization" 34 | }, 35 | { 36 | "_id": "concept/string", 37 | "schema": "concept", 38 | "kind": "type", 39 | "id": "string", 40 | "name": "String", 41 | "description": "primitive string type" 42 | }, 43 | { 44 | "_id": "concept/number", 45 | "schema": "concept", 46 | "kind": "type", 47 | "id": "number", 48 | "name": "Number", 49 | "description": "primitive number type" 50 | }, 51 | { 52 | "_id": "concept/person-first-name", 53 | "schema": "concept", 54 | "kind": "function", 55 | "id": "person-first-name", 56 | "name": "first name", 57 | "description": "first name (given name) of person", 58 | "inputs": [ 59 | { 60 | "type": "person" 61 | } 62 | ], 63 | "outputs": [ 64 | { 65 | "type": "string" 66 | } 67 | ] 68 | }, 69 | { 70 | "_id": "concept/person-last-name", 71 | "schema": "concept", 72 | "kind": "function", 73 | "id": "person-last-name", 74 | "name": "last name", 75 | "description": "last name (family name) of person", 76 | "inputs": [ 77 | { 78 | "type": "person" 79 | } 80 | ], 81 | "outputs": [ 82 | { 83 | "type": "string" 84 | } 85 | ] 86 | }, 87 | { 88 | "_id": "concept/employee-salary", 89 | "schema": "concept", 90 | "kind": "function", 91 | "id": "employee-salary", 92 | "name": "salary", 93 | "description": "salary of an employee (in USD)", 94 | "inputs": [ 95 | { 96 | "type": "employee" 97 | } 98 | ], 99 | "outputs": [ 100 | { 101 | "type": "number" 102 | } 103 | ] 104 | }, 105 | { 106 | "_id": "concept/works-in", 107 | "schema": "concept", 108 | "kind": "function", 109 | "id": "works-in", 110 | "name": "works in", 111 | "description": "department that employee works in", 112 | "inputs": [ 113 | { 114 | "type": "employee" 115 | } 116 | ], 117 | "outputs": [ 118 | { 119 | "type": "department" 120 | } 121 | ] 122 | }, 123 | { 124 | "_id": "concept/reports-to", 125 | "schema": "concept", 126 | "kind": "function", 127 | "id": "reports-to", 128 | "name": "reports to", 129 | "description": "manager that employee reports to", 130 | "inputs": [ 131 | { 132 | "type": "employee" 133 | } 134 | ], 135 | "outputs": [ 136 | { 137 | "type": "manager" 138 | } 139 | ] 140 | }, 141 | { 142 | "_id": "concept/department-secretary", 143 | "schema": "concept", 144 | "kind": "function", 145 | "id": "department-secretary", 146 | "name": "secretary", 147 | "description": "secretary of a department", 148 | "inputs": [ 149 | { 150 | "type": "department" 151 | } 152 | ], 153 | "outputs": [ 154 | { 155 | "type": "employee" 156 | } 157 | ] 158 | }, 159 | { 160 | "_id": "annotation/opendisc/employee/employee", 161 | "schema": "annotation", 162 | "language": "opendisc", 163 | "package": "employee", 164 | "id": "emplyee", 165 | "kind": "type", 166 | "definition": "employee", 167 | "slots": [ 168 | { 169 | "slot": "first_name", 170 | "definition": "person-first-name" 171 | }, 172 | { 173 | "slot": "last_name", 174 | "definition": "person-last-name" 175 | }, 176 | { 177 | "slot": "salary", 178 | "definition": "employee-salary" 179 | } 180 | ] 181 | }, 182 | { 183 | "_id": "annotation/opendisc/employee/department", 184 | "schema": "annotation", 185 | "language": "opendisc", 186 | "package": "employee", 187 | "id": "department", 188 | "kind": "type", 189 | "definition": "department" 190 | }, 191 | { 192 | "_id": "annotation/opendisc/employee/str", 193 | "schema": "annotation", 194 | "language": "opendisc", 195 | "package": "employee", 196 | "id": "department", 197 | "kind": "type", 198 | "definition": "string" 199 | }, 200 | { 201 | "_id": "annotation/opendisc/employee/employee-department", 202 | "schema": "annotation", 203 | "language": "opendisc", 204 | "package": "employee", 205 | "id": "manager", 206 | "kind": "function", 207 | "definition": "works-in" 208 | }, 209 | { 210 | "_id": "annotation/opendisc/employee/manager", 211 | "schema": "annotation", 212 | "language": "opendisc", 213 | "package": "employee", 214 | "id": "manager", 215 | "kind": "function", 216 | "definition": [ 217 | "compose", 218 | "reports-to", 219 | [ 220 | "coerce", 221 | [ 222 | "SubOb", 223 | "manager", 224 | "employee" 225 | ] 226 | ] 227 | ] 228 | }, 229 | { 230 | "_id": "annotation/opendisc/employee/second-level-manager", 231 | "schema": "annotation", 232 | "language": "opendisc", 233 | "package": "employee", 234 | "id": "second-level-manager", 235 | "kind": "function", 236 | "definition": [ 237 | "compose", 238 | "reports-to", 239 | "reports-to", 240 | [ 241 | "coerce", 242 | [ 243 | "SubOb", 244 | "manager", 245 | "employee" 246 | ] 247 | ] 248 | ] 249 | }, 250 | { 251 | "_id": "annotation/opendisc/employee/third-level-manager", 252 | "schema": "annotation", 253 | "language": "opendisc", 254 | "package": "employee", 255 | "id": "third-level-manager", 256 | "kind": "function", 257 | "definition": [ 258 | "compose", 259 | "reports-to", 260 | "reports-to", 261 | "reports-to", 262 | [ 263 | "coerce", 264 | [ 265 | "SubOb", 266 | "manager", 267 | "employee" 268 | ] 269 | ] 270 | ] 271 | } 272 | ] 273 | -------------------------------------------------------------------------------- /src/Doctrine.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | module Doctrine 16 | export Monocl, MonoclCategory, MonoclError, Ob, Hom, SubOb, SubHom, 17 | dom, codom, subob_dom, subob_codom, subob_id, subhom_id, 18 | compose, compose2, id, otimes, munit, opow, braid, mcopy, delete, pair, 19 | hom, ev, curry, coerce, construct, MonoclElem, to_wiring_diagram 20 | 21 | using AutoHashEquals 22 | 23 | using Catlab 24 | import Catlab.Theories: CategoryExpr, ObExpr, HomExpr, 25 | SymmetricMonoidalCategory, Ob, Hom, dom, codom, compose, id, otimes, munit, 26 | braid, mcopy, delete, pair, hom, ev, curry 27 | 28 | using Catlab.WiringDiagrams 29 | import Catlab.WiringDiagrams: Box, WiringDiagram, to_wiring_diagram 30 | 31 | # Cartesian (closed) category 32 | ############################# 33 | 34 | """ Doctrine of *cartesian category* 35 | 36 | This signature differs from the official Catlab doctrine by allowing `mcopy` 37 | terms of size greater than 2. 38 | """ 39 | @signature CartesianCategory{Ob,Hom} <: SymmetricMonoidalCategory{Ob,Hom} begin 40 | opow(A::Ob, n::Int)::Ob 41 | 42 | mcopy(A::Ob, n::Int)::Hom(A,opow(A,n)) 43 | delete(A::Ob)::Hom(A,munit()) 44 | end 45 | 46 | """ Doctrine of *cartesian closed category* 47 | 48 | This signature is identical to the official Catlab doctrine, except for 49 | inheriting from a different doctrine. 50 | """ 51 | @signature CartesianClosedCategory{Ob,Hom} <: CartesianCategory{Ob,Hom} begin 52 | hom(A::Ob, B::Ob)::Ob 53 | ev(A::Ob, B::Ob)::Hom(otimes(hom(A,B),A),B) 54 | curry(A::Ob, B::Ob, f::Hom(otimes(A,B),C))::Hom(A,hom(B,C)) ⊣ (C::Ob) 55 | end 56 | 57 | # Monocl category 58 | ################# 59 | 60 | struct MonoclError <: Exception 61 | message::String 62 | end 63 | 64 | """ Doctrine for Monocl: MONoidal Ontology and Computer Language 65 | 66 | A doctrine of monoidal categories derived from the doctrine of cartesian closed 67 | categories with implicit conversion of types. 68 | """ 69 | @signature MonoclCategory{Ob,Hom,SubOb,SubHom} <: CartesianClosedCategory{Ob,Hom} begin 70 | """ Subobject relation. 71 | 72 | The domain object is a subobject of the codomain object. Alternatively, 73 | in PLT jargon, the domain type is a subtype of the codomain type. 74 | """ 75 | SubOb(dom::Ob, codom::Ob)::TYPE 76 | 77 | """ Submorphism relation. 78 | 79 | The domain morphism is a submorphism of the codomain morphism. This means 80 | there is a natural transformation, whose components are subobject morphisms, 81 | having as (co)domain the functor on the interval category determined by the 82 | (co)domain morphism. 83 | 84 | Alternatively, in PLT jargon, the domain function is a specialization of the 85 | codomain function, a form of ad-hoc polymorphism. 86 | """ 87 | SubHom(dom::Hom(A0,B0), codom::Hom(A,B), 88 | subob_dom::SubOb(A0,A), subob_codom::SubOb(B0,B))::TYPE ⊣ 89 | (A0::Ob, B0::Ob, A::Ob, B::Ob) 90 | 91 | """ Coercion morphism of type A to type B. 92 | """ 93 | coerce(sub::SubOb(A,B))::Hom(A,B) ⊣ (A::Ob, B::Ob) 94 | 95 | """ Constructor for instances of type A with data f: A -> B. 96 | 97 | The semantics of this term are: 98 | compose(construct(f), f) = id(B) 99 | """ 100 | construct(f::Hom(A,B))::Hom(B,A) ⊣ (A::Ob, B::Ob) 101 | 102 | # Category of subobject morphisms. 103 | # TODO: Support internal homs. 104 | # XXX: Cannot reuse `id` for subobjects because cannot dispatch on return type. 105 | subob_id(A::Ob)::SubOb(A,A) 106 | compose(f::SubOb(A,B), g::SubOb(B,C))::SubOb(A,C) ⊣ (A::Ob, B::Ob, C::Ob) 107 | otimes(f::SubOb(A,B), g::SubOb(C,D))::SubOb(otimes(A,C),otimes(B,D)) ⊣ 108 | (A::Ob, B::Ob, C::Ob, D::Ob) 109 | 110 | # 2-category of submorphism natural transformations. 111 | subhom_id(f::Hom(A,B))::SubHom(f,f,subob_id(dom(f)),subob_id(codom(f))) ⊣ 112 | (A::Ob, B::Ob) 113 | compose(α::SubHom(f,g,α0,α1), β::SubHom(g,h,β0,β1))::SubHom(f,h,compose(α0,β0),compose(α1,β1)) ⊣ 114 | (A0::Ob, B0::Ob, A::Ob, B::Ob, A1::Ob, B1::Ob, 115 | f::Hom(A0,B0), g::Hom(A,B), h::Hom(A1,B1), 116 | α0::SubOb(A0,A), α1::SubOb(B0,B), β0::SubOb(A,A1), β1::SubOb(B,B1)) 117 | compose2(α::SubHom(f,g,α0,α1), β::SubHom(h,k,β0,β1))::SubHom(compose(f,h),compose(g,k),α0,β1) ⊣ 118 | (A0::Ob, B0::Ob, C0::Ob, A::Ob, B::Ob, C::Ob, 119 | f::Hom(A0,B0), g::Hom(A,B), h::Hom(B0,C0), k::Hom(B,C), 120 | α0::SubOb(A0,A), α1::SubOb(B0,B), β0::SubOb(B0,B), β1::SubOb(C0,C)) 121 | end 122 | 123 | """ Syntax system for Monocl: MONoidal Ontology and Computer Language 124 | """ 125 | @syntax Monocl{ObExpr,HomExpr,CategoryExpr,CategoryExpr} MonoclCategory begin 126 | # TODO: Implicit conversion is not yet implemented, so we have disabled 127 | # domain checks when composing morphisms! 128 | # TODO: Domain checks when composing submorphisms need only check domain 129 | # and codomain of subobjects because subobjects form a pre-order. 130 | compose(f::Hom, g::Hom) = associate_unit(new(f,g; strict=false), id) 131 | compose(f::SubOb, g::SubOb) = associate_unit(new(f,g; strict=true), subob_id) 132 | compose(f::SubHom, g::SubHom) = associate_unit(new(f,g; strict=false), subhom_id) 133 | compose2(f::SubHom, g::SubHom) = associate_unit(new(f,g; strict=false), subhom_id) 134 | 135 | otimes(A::Ob, B::Ob) = associate_unit(new(A,B), munit) 136 | otimes(f::Hom, g::Hom) = associate(new(f,g)) 137 | otimes(f::SubOb, g::SubOb) = associate(new(f,g)) 138 | opow(A::Ob, n::Int) = otimes([A for i=1:n]) 139 | 140 | # TODO: Enforce pre-order, not just reflexivity. 141 | coerce(sub::SubOb) = dom(sub) == codom(sub) ? id(dom(sub)) : new(sub) 142 | end 143 | 144 | SubOb(dom::Monocl.Ob, codom::Monocl.Ob) = SubOb(nothing, dom, codom) 145 | SubHom(dom::Monocl.Hom, codom::Monocl.Hom, 146 | subob_dom::Monocl.SubOb, subob_codom::Monocl.SubOb) = 147 | SubHom(nothing, dom, codom, subob_dom, subob_codom) 148 | 149 | """ Pairing of two (or more) morphisms. 150 | 151 | Pairing is possible in any cartesian category. This method differs from the 152 | standard Catlab definition by allowing coercion of the common domain object. 153 | """ 154 | function pair(A::Monocl.Ob, fs::Vector{Monocl.Hom}) 155 | compose(mcopy(A,length(fs)), otimes(fs)) 156 | end 157 | function pair(fs::Vector{Monocl.Hom}) 158 | A = dom(first(fs)) 159 | @assert all(dom(f) == A for f in fs) 160 | pair(A, fs) 161 | end 162 | pair(A::Monocl.Ob, fs::Vararg{Monocl.Hom}) = pair(A, collect(Monocl.Hom,fs)) 163 | pair(fs::Vararg{Monocl.Hom}) = pair(collect(Monocl.Hom,fs)) 164 | 165 | mcopy(A::Monocl.Ob) = mcopy(A,2) 166 | construct(A::Monocl.Ob) = construct(delete(A)) 167 | 168 | # Monocl category of elements 169 | ############################# 170 | 171 | """ Object in Monocl's category of elements. 172 | """ 173 | @auto_hash_equals struct MonoclElem 174 | ob::Union{Monocl.Ob,Nothing} 175 | value::Any 176 | MonoclElem(ob, value=nothing) = new(ob, value) 177 | end 178 | 179 | # Monocl wiring diagrams 180 | ######################## 181 | 182 | function Box(f::Monocl.Hom) 183 | Box(f, collect(dom(f)), collect(codom(f))) 184 | end 185 | function WiringDiagram(dom::Monocl.Ob, codom::Monocl.Ob) 186 | WiringDiagram(collect(dom), collect(codom)) 187 | end 188 | 189 | function to_wiring_diagram(expr::Monocl.Hom) 190 | functor((Ports, WiringDiagram, Monocl.SubOb), expr; 191 | terms = Dict( 192 | :Ob => (expr) -> Ports([expr]), 193 | :Hom => (expr) -> singleton_diagram(Box(expr)), 194 | :SubOb => identity, 195 | :coerce => (expr) -> to_wiring_diagram(first(expr)), 196 | :construct => (expr) -> singleton_diagram(Box(expr)), 197 | ) 198 | ) 199 | end 200 | 201 | function to_wiring_diagram(sub::Monocl.SubOb) 202 | A, B = collect(dom(sub)), collect(codom(sub)) 203 | @assert length(A) == length(B) 204 | f = WiringDiagram(A, B) 205 | add_wires!(f, ((input_id(f),i) => (output_id(f),i) for i in eachindex(A))) 206 | return f 207 | end 208 | 209 | end 210 | -------------------------------------------------------------------------------- /src/SemanticEnrichment.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ The semantic enrichment algorithm for dataflow graphs. 16 | """ 17 | module SemanticEnrichment 18 | export to_semantic_graph 19 | 20 | using Base.Iterators: product 21 | using Compat 22 | 23 | using LightGraphs, MetaGraphs 24 | 25 | using Catlab.WiringDiagrams 26 | using ..Doctrine 27 | using ..Ontology 28 | using ..RawFlowGraphs 29 | 30 | """ Convert a raw flow graph into a semantic flow graph. 31 | """ 32 | function to_semantic_graph(db::OntologyDB, raw::WiringDiagram)::WiringDiagram 33 | sem = WiringDiagram(to_semantic_ports(db, input_ports(raw)), 34 | to_semantic_ports(db, output_ports(raw))) 35 | 36 | # Add boxes from raw flow graph, marking some for substitution. 37 | to_substitute = Int[] 38 | for v in box_ids(raw) 39 | raw_box = box(raw, v) 40 | sem_box = to_semantic_graph(db, raw_box) 41 | @assert add_box!(sem, sem_box) == v 42 | if isa(raw_box, Box) && isa(sem_box, WiringDiagram) 43 | # If the raw box is atomic but the semantic box is a wiring diagram, 44 | # we should expand it by substitution. We defer the substitution until 45 | # later so that the box IDs on the wires remain valid. 46 | push!(to_substitute, v) 47 | end 48 | end 49 | 50 | # Add wires from raw flow graph. 51 | # FIXME: If a raw box expands to a semantic box that is missing a port with 52 | # incoming or outoging wires in the raw graph (e.g., when an unannotated 53 | # keyword argument is passed), this logic will fail. We should detect this 54 | # situation and either raise an informative error or discard the wire. 55 | add_wires!(sem, wires(raw)) 56 | 57 | # Perform deferred substitutions (see above). 58 | sem = substitute(sem, to_substitute) 59 | 60 | # Simplify the diagram by collapsing adjacent unannotated boxes. 61 | collapse_unannotated_boxes(sem) 62 | end 63 | 64 | function to_semantic_graph(db::OntologyDB, raw_box::Box{RawNode})::AbstractBox 65 | if isnothing(raw_box.value.annotation) 66 | Box(nothing, 67 | to_semantic_ports(db, input_ports(raw_box)), 68 | to_semantic_ports(db, output_ports(raw_box))) 69 | else 70 | expand_annotated_box(db, raw_box) 71 | end 72 | end 73 | 74 | function to_semantic_ports(db::OntologyDB, ports::AbstractVector) 75 | [ isnothing(port.annotation) ? nothing : expand_annotated_port(db, port) 76 | for port in ports ] 77 | end 78 | 79 | """ Expand a single annotated box from a raw flow graph. 80 | """ 81 | function expand_annotated_box(db::OntologyDB, raw_box::Box{RawNode})::WiringDiagram 82 | expand_annotated_box(db, raw_box, Val{raw_box.value.annotation_kind}) 83 | end 84 | 85 | function expand_annotated_box(db::OntologyDB, raw_box::Box{RawNode}, 86 | ::Type{Val{FunctionAnnotation}}) 87 | inputs = input_ports(raw_box) 88 | outputs = output_ports(raw_box) 89 | note = load_annotation(db, raw_box.value.annotation)::HomAnnotation 90 | f = WiringDiagram(inputs, outputs) 91 | v = add_box!(f, to_wiring_diagram(note.definition)) 92 | add_wires!(f, ((input_id(f), i) => (v, port.annotation_index) 93 | for (i, port) in enumerate(inputs) 94 | if !isnothing(port.annotation_index))) 95 | add_wires!(f, ((v, port.annotation_index) => (output_id(f), i) 96 | for (i, port) in enumerate(outputs) 97 | if !isnothing(port.annotation_index))) 98 | substitute(f, v) 99 | end 100 | 101 | function expand_annotated_box(db::OntologyDB, raw_box::Box{RawNode}, 102 | ::Type{Val{ConstructAnnotation}}) 103 | note = load_annotation(db, raw_box.value.annotation)::ObAnnotation 104 | to_wiring_diagram(construct(note.definition)) 105 | end 106 | 107 | function expand_annotated_box(db::OntologyDB, raw_box::Box{RawNode}, 108 | ::Type{Val{SlotAnnotation}}) 109 | note = load_annotation(db, raw_box.value.annotation)::ObAnnotation 110 | index = raw_box.value.annotation_index 111 | to_wiring_diagram(note.slots[index]) 112 | end 113 | 114 | """ Expand a single annotated port from a raw flow graph. 115 | """ 116 | function expand_annotated_port(db::OntologyDB, raw_port::RawPort)::Monocl.Ob 117 | note = load_annotation(db, raw_port.annotation)::ObAnnotation 118 | note.definition 119 | end 120 | 121 | """ Collapse adjacent unannotated boxes into single boxes. 122 | """ 123 | function collapse_unannotated_boxes(diagram::WiringDiagram) 124 | # Find maximal groups of unannotated boxes. 125 | unannotated = filter(box_ids(diagram)) do v 126 | isnothing(box(diagram,v).value) 127 | end 128 | groups = group_blank_vertices(SimpleDiGraph(graph(diagram)), unannotated) 129 | 130 | # Encapsulate the groups, including groups of size 1 because encapsulation 131 | # will simplify the ports. 132 | encapsulate(diagram, groups, discard_boxes=true) 133 | end 134 | 135 | """ Group adjacent blank vertices of a directed graph. 136 | """ 137 | function group_blank_vertices(graph::SimpleDiGraph, blank::Vector{Int})::Vector{Vector{Int}} 138 | # Create transitive closure of graph. 139 | closure = transitiveclosure(graph) 140 | has_path(u::Int, v::Int) = has_edge(closure, u, v) 141 | ancestors(v::Int) = LightGraphs.inneighbors(closure, v) 142 | descendants(v::Int) = LightGraphs.outneighbors(closure, v) 143 | 144 | # Initialize groups as singletons. 145 | graph = MetaDiGraph(graph) 146 | for v in blank 147 | set_prop!(graph, v, :vertices, [v]) 148 | end 149 | get_group(v::Int) = get_prop(graph, v, :vertices) 150 | is_blank(v::Int) = has_prop(graph, v, :vertices) 151 | not_blank(v::Int) = !is_blank(v) 152 | 153 | # Definition: Two adjacent blank vertices are mergeable if their merger 154 | # does not introduce any new dependencies between non-blank vertices. 155 | # That is, if there is no directed path from non-blank vertex `v1` to 156 | # non-blank vertex `v2` before merging, there will not be one afterwards. 157 | # 158 | # This criterion holds iff it's *not* the case that 159 | # there exists a non-blank ancestor of the child that isn't an ancestor of 160 | # the parent and a non-blank descendant of the parent isn't a descendant of 161 | # the child 162 | # (because if there was, merging would create a new dependency) iff 163 | # each non-blank ancestor of the child is also an ancestor of the parent, or 164 | # each non-blank descendant of the parent is also a descendent of the child. 165 | function is_mergable(u::Int, v::Int) 166 | (is_blank(u) && is_blank(v)) || return false 167 | parent, child = if has_edge(graph, u, v); (u, v) 168 | elseif has_edge(graph, v, u); (v, u) 169 | else return false end 170 | all(has_path(v, parent) for v in filter(not_blank, ancestors(child))) || 171 | all(has_path(child, v) for v in filter(not_blank, descendants(parent))) 172 | end 173 | function merge_blank!(u::Int, v::Int) 174 | append!(get_group(min(u,v)), get_group(max(u,v))) 175 | merge_vertices_directed!(graph, [u,v]) 176 | merge_vertices_directed!(closure, [u,v]) 177 | end 178 | 179 | # Merge pairs of mergable vertices until there are no more mergable pairs. 180 | # The loop maintains the invariant that no two vertices less than the current 181 | # one are mergable. 182 | v = 1 183 | while v <= nv(graph) 184 | merged = false 185 | for u in LightGraphs.all_neighbors(graph, v) 186 | if u > v && is_mergable(u, v) 187 | merge_blank!(u, v) 188 | merged = true 189 | break 190 | end 191 | end 192 | v += !merged 193 | end 194 | 195 | Vector{Int}[ get_group(v) for v in vertices(graph) if is_blank(v) ] 196 | end 197 | 198 | """ Merge the vertices into a single vertex, preserving edges. 199 | 200 | Note: LightGraphs.merge_vertices! only supports undirected graphs. 201 | """ 202 | function merge_vertices_directed!(graph::AbstractGraph, vs::Vector{Int}) 203 | @assert is_directed(graph) 204 | vs = sort(vs, rev=true) 205 | v0 = vs[end] 206 | for v in vs[1:end-1] 207 | for u in LightGraphs.inneighbors(graph, v) 208 | if !(u in vs) 209 | add_edge!(graph, u, v0) 210 | end 211 | end 212 | for u in LightGraphs.outneighbors(graph, v) 213 | if !(u in vs) 214 | add_edge!(graph, v0, u) 215 | end 216 | end 217 | rem_vertex!(graph, v) 218 | end 219 | v0 220 | end 221 | 222 | end 223 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /test/data/clustering_kmeans.raw.graphml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | literal 21 | datasets/iris.csv 22 | r/base/character 23 | construct 24 | 25 | output 26 | character 27 | S3 28 | r/base/character 29 | 1 30 | 31 | 32 | 33 | literal 34 | false 35 | r/base/logical 36 | construct 37 | 38 | output 39 | logical 40 | S3 41 | r/base/logical 42 | 1 43 | 44 | 45 | 46 | read.csv 47 | utils 48 | r/utils/read-csv 49 | 50 | input 51 | character 52 | S3 53 | r/base/character 54 | 1 55 | 56 | 57 | input 58 | logical 59 | S3 60 | r/base/logical 61 | 62 | 63 | output 64 | data.frame 65 | S3 66 | r/base/data-frame 67 | 1 68 | 69 | 70 | 71 | names 72 | base 73 | 74 | input 75 | data.frame 76 | S3 77 | r/base/data-frame 78 | 79 | 80 | output 81 | character 82 | S3 83 | r/base/character 84 | 85 | 86 | 87 | literal 88 | Species 89 | r/base/character 90 | construct 91 | 92 | output 93 | character 94 | S3 95 | r/base/character 96 | 1 97 | 98 | 99 | 100 | != 101 | base 102 | 103 | input 104 | character 105 | S3 106 | r/base/character 107 | 108 | 109 | input 110 | character 111 | S3 112 | r/base/character 113 | 114 | 115 | output 116 | logical 117 | S3 118 | r/base/logical 119 | 120 | 121 | 122 | [ 123 | base 124 | 125 | input 126 | data.frame 127 | S3 128 | r/base/data-frame 129 | 130 | 131 | input 132 | logical 133 | S3 134 | r/base/logical 135 | 136 | 137 | output 138 | data.frame 139 | S3 140 | r/base/data-frame 141 | 142 | 143 | 144 | literal 145 | 3 146 | r/base/numeric 147 | construct 148 | 149 | output 150 | numeric 151 | S3 152 | r/base/numeric 153 | 1 154 | 155 | 156 | 157 | kmeans 158 | stats 159 | r/stats/fit-k-means 160 | 161 | input 162 | data.frame 163 | S3 164 | r/base/data-frame 165 | 2 166 | 167 | 168 | input 169 | numeric 170 | S3 171 | r/base/numeric 172 | 1 173 | 174 | 175 | output 176 | kmeans 177 | S3 178 | r/stats/k-means 179 | 1 180 | 181 | 182 | 183 | literal 184 | centers 185 | r/base/character 186 | construct 187 | 188 | output 189 | character 190 | S3 191 | r/base/character 192 | 1 193 | 194 | 195 | 196 | $ 197 | base 198 | centers 199 | r/stats/k-means 200 | slot 201 | 2 202 | 203 | input 204 | kmeans 205 | S3 206 | r/stats/k-means 207 | 208 | 209 | input 210 | character 211 | S3 212 | r/base/character 213 | 214 | 215 | output 216 | matrix 217 | S3 218 | r/base/matrix 219 | 220 | 221 | 222 | literal 223 | cluster 224 | r/base/character 225 | construct 226 | 227 | output 228 | character 229 | S3 230 | r/base/character 231 | 1 232 | 233 | 234 | 235 | $ 236 | base 237 | cluster 238 | r/stats/k-means 239 | slot 240 | 1 241 | 242 | input 243 | kmeans 244 | S3 245 | r/stats/k-means 246 | 247 | 248 | input 249 | character 250 | S3 251 | r/base/character 252 | 253 | 254 | output 255 | integer 256 | S3 257 | r/base/integer 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | -------------------------------------------------------------------------------- /src/CLI.jl: -------------------------------------------------------------------------------- 1 | # Copyright 2018 IBM Corp. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ Command-line interface for raw and semantic flow graphs. 16 | """ 17 | module CLI 18 | export main, invoke, parse 19 | 20 | using ArgParse 21 | using Compat 22 | import DefaultApplication 23 | using Requires 24 | import JSON 25 | import Serd 26 | 27 | using Catlab.WiringDiagrams, Catlab.Graphics 28 | import Catlab.Graphics: Graphviz 29 | using ..RawFlowGraphs 30 | using ..Ontology, ..SemanticEnrichment 31 | using ..Serialization 32 | 33 | # CLI arguments 34 | ############### 35 | 36 | const settings = ArgParseSettings() 37 | 38 | @add_arg_table! settings begin 39 | "record" 40 | help = "record code as raw flow graph" 41 | action = :command 42 | "enrich" 43 | help = "convert raw flow graph to semantic flow graph" 44 | action = :command 45 | "visualize" 46 | help = "visualize flow graph" 47 | action = :command 48 | "ontology" 49 | help = "export ontology" 50 | action = :command 51 | end 52 | 53 | @add_arg_table! settings["record"] begin 54 | "path" 55 | help = "input Python or R file, or directory" 56 | required = true 57 | "-o", "--out" 58 | help = "output raw flow graph or directory" 59 | "-f", "--from" 60 | help = "input format (one of: \"python\", \"r\") (default: from file name)" 61 | default = nothing 62 | "-t", "--to" 63 | help = "output format (one of: \"graphml\", \"json\")" 64 | default = "graphml" 65 | "--graph-outputs" 66 | help = "whether and how to retain outputs of raw flow graph (Python only)" 67 | default = "none" 68 | metavar = "GRAPH-OUT" 69 | "--annotations" 70 | help = "load annotations from JSON file (default: load from remote database)" 71 | metavar = "FILE" 72 | end 73 | 74 | @add_arg_table! settings["enrich"] begin 75 | "path" 76 | help = "input raw flow graph or directory" 77 | required = true 78 | "-o", "--out" 79 | help = "output semantic flow graph or directory" 80 | "-f", "--from" 81 | help = "input format (one of: \"graphml\", \"json\") (default: from file name)" 82 | default = nothing 83 | "-t", "--to" 84 | help = "output format (one of: \"graphml\", \"json\")" 85 | default = "graphml" 86 | "--ontology" 87 | help = "load ontology from JSON file (default: load from remote database)" 88 | metavar = "FILE" 89 | end 90 | 91 | @add_arg_table! settings["visualize"] begin 92 | "path" 93 | help = "input raw or semantic flow graph, or directory" 94 | required = true 95 | "-o", "--out" 96 | help = "output file or directory" 97 | "-f", "--from" 98 | help = "input format (one of: \"raw-graphml\", \"raw-json\", \"semantic-graphml\", \"semantic-json\") (default: from file name)" 99 | default = nothing 100 | "-t", "--to" 101 | help = "Graphviz output format (default: Graphviz input only)" 102 | "--open" 103 | help = "open output using OS default application" 104 | action = :store_true 105 | end 106 | 107 | @add_arg_table! settings["ontology"] begin 108 | "json" 109 | help = "export ontology as JSON" 110 | action = :command 111 | "rdf" 112 | help = "export ontology as RDF/OWL" 113 | action = :command 114 | end 115 | 116 | @add_arg_table! settings["ontology"]["json"] begin 117 | "-o", "--out" 118 | help = "output file (default: stdout)" 119 | "--indent" 120 | help = "number of spaces to indent (default: compact output)" 121 | arg_type = Int 122 | default = nothing 123 | "--no-concepts" 124 | help = "exclude concepts from export" 125 | dest_name = "concepts" 126 | action = :store_false 127 | "--no-annotations" 128 | help = "exclude annotations from export" 129 | dest_name = "annotations" 130 | action = :store_false 131 | end 132 | 133 | @add_arg_table! settings["ontology"]["rdf"] begin 134 | "-o", "--out" 135 | help = "output file (default: stdout)" 136 | "-t", "--to" 137 | help = "output format (one of: \"turtle\", \"ntriples\", \"nquads\", \"trig\")" 138 | default = "turtle" 139 | "--no-concepts" 140 | help = "omit concepts" 141 | dest_name = "concepts" 142 | action = :store_false 143 | "--no-annotations" 144 | help = "omit annotations" 145 | dest_name = "annotations" 146 | action = :store_false 147 | "--no-schema" 148 | help = "omit preamble defining OWL schema for concepts and annotations" 149 | dest_name = "schema" 150 | action = :store_false 151 | "--no-provenance" 152 | help = "omit interoperability with PROV Ontology (PROV-O)" 153 | dest_name = "provenance" 154 | action = :store_false 155 | "--no-wiring-diagrams" 156 | help = "omit wiring diagrams in concepts and annotations" 157 | dest_name = "wiring" 158 | action = :store_false 159 | end 160 | 161 | # Record 162 | ######## 163 | 164 | function record(args::Dict) 165 | paths = parse_io_args(args["path"], args["out"], 166 | format = args["from"], 167 | formats = Dict( 168 | ".py" => "python", 169 | ".r" => "r", 170 | ".R" => "r", 171 | ), 172 | out_ext = format -> ".raw." * args["to"] 173 | ) 174 | for (inpath, lang, outpath) in paths 175 | if args["to"] == "graphml" 176 | record_file(abspath(inpath), abspath(outpath), args, Val(Symbol(lang))) 177 | else 178 | # Both the Python and R packages emit GraphML. 179 | diagram = mktemp() do temp_path, io 180 | record_file(abspath(inpath), temp_path, args, Val(Symbol(lang))) 181 | read_graph_file(temp_path, format="graphml") 182 | end 183 | write_graph_file(diagram, outpath, format=args["to"]) 184 | end 185 | end 186 | end 187 | 188 | function record_file(inpath::AbstractString, outpath::AbstractString, 189 | args::Dict, ::Val{lang}) where lang 190 | if lang == :python 191 | error("PyCall.jl has not been imported") 192 | elseif lang == :r 193 | error("RCall.jl has not been imported") 194 | else 195 | error("Unsupported language: $lang") 196 | end 197 | end 198 | 199 | # Enrich 200 | ######## 201 | 202 | function enrich(args::Dict) 203 | paths = parse_io_args(args["path"], args["out"], 204 | format = args["from"], 205 | formats = Dict( 206 | ".raw.graphml" => "graphml", 207 | ".raw.json" => "json", 208 | ), 209 | out_ext = format -> ".semantic." * args["to"], 210 | ) 211 | 212 | # Load ontology from local or remote source. 213 | db = OntologyDB() 214 | if isnothing(args["ontology"]) 215 | # Only load concepts from remote. Annotations will be loaded on-the-fly. 216 | load_concepts(db) 217 | else 218 | load_ontology_file(db, args["ontology"]) 219 | end 220 | 221 | # Run semantic enrichment. 222 | for (inpath, format, outpath) in paths 223 | raw = read_graph_file(inpath, kind="raw", format=format) 224 | raw = rem_literals(raw) 225 | semantic = to_semantic_graph(db, raw) 226 | write_graph_file(semantic, outpath, format=args["to"]) 227 | end 228 | end 229 | 230 | # Visualize 231 | ########### 232 | 233 | function visualize(args::Dict) 234 | format = isnothing(args["from"]) ? nothing : 235 | Tuple(split(args["from"], "-", limit=2)) 236 | out_format = isnothing(args["to"]) ? "dot" : args["to"] 237 | paths = parse_io_args(args["path"], args["out"], 238 | format = format, 239 | formats = Dict( 240 | ".raw.graphml" => ("raw", "graphml"), 241 | ".raw.json" => ("raw", "json"), 242 | ".semantic.graphml" => ("semantic", "graphml"), 243 | ".semantic.json" => ("semantic", "json"), 244 | ), 245 | out_ext = format -> "." * first(format) * "." * out_format, 246 | ) 247 | for (inpath, (kind, format), outpath) in paths 248 | # Read flow graph and convert to Graphviz AST. 249 | diagram = read_graph_file(inpath, kind=kind, format=format) 250 | graphviz = if kind == "raw" 251 | raw_graph_to_graphviz(diagram) 252 | else 253 | semantic_graph_to_graphviz(diagram) 254 | end 255 | 256 | # Pretty-print Graphviz AST to output file. 257 | if isnothing(args["to"]) 258 | # Default: no output format, yield Graphviz dot input. 259 | open(outpath, "w") do f 260 | Graphviz.pprint(f, graphviz) 261 | end 262 | else 263 | # Run Graphviz with given output format. 264 | open(`dot -T$out_format -o $outpath`, "w", stdout) do f 265 | Graphviz.pprint(f, graphviz) 266 | end 267 | end 268 | 269 | if args["open"] 270 | DefaultApplication.open(outpath) 271 | end 272 | end 273 | end 274 | 275 | function raw_graph_to_graphviz(diagram::WiringDiagram) 276 | to_graphviz(rem_unused_ports(diagram); 277 | graph_name = "raw_flow_graph", 278 | labels = true, 279 | label_attr = :xlabel, 280 | graph_attrs = Graphviz.Attributes( 281 | :fontname => "Courier", 282 | ), 283 | node_attrs = Graphviz.Attributes( 284 | :fontname => "Courier", 285 | ), 286 | edge_attrs = Graphviz.Attributes( 287 | :fontname => "Courier", 288 | :arrowsize => "0.5", 289 | ) 290 | ) 291 | end 292 | 293 | function semantic_graph_to_graphviz(diagram::WiringDiagram) 294 | to_graphviz(diagram; 295 | graph_name = "semantic_flow_graph", 296 | labels = true, 297 | label_attr = :xlabel, 298 | graph_attrs=Graphviz.Attributes( 299 | :fontname => "Helvetica", 300 | ), 301 | node_attrs=Graphviz.Attributes( 302 | :fontname => "Helvetica", 303 | ), 304 | edge_attrs=Graphviz.Attributes( 305 | :fontname => "Helvetica", 306 | :arrowsize => "0.5", 307 | ), 308 | cell_attrs=Graphviz.Attributes( 309 | :style => "rounded", 310 | ) 311 | ) 312 | end 313 | 314 | # Ontology 315 | ########## 316 | 317 | function ontology_as_json(args::Dict) 318 | docs = AbstractDict[] 319 | db = OntologyDB() 320 | if args["concepts"] 321 | append!(docs, OntologyDBs.api_get(db, "/concepts")) 322 | end 323 | if args["annotations"] 324 | append!(docs, OntologyDBs.api_get(db, "/annotations")) 325 | end 326 | if args["out"] != nothing 327 | open(args["out"], "w") do out 328 | JSON.print(out, docs, args["indent"]) 329 | end 330 | else 331 | JSON.print(stdout, docs, args["indent"]) 332 | end 333 | end 334 | 335 | function ontology_as_rdf(args::Dict) 336 | # Load ontology data from remote database. 337 | db = OntologyDB() 338 | if args["concepts"] 339 | load_concepts(db) 340 | end 341 | if args["annotations"] 342 | load_annotations(db) 343 | end 344 | 345 | # Load ontology schemas from filesystem. 346 | stmts = Serd.RDF.Statement[] 347 | if args["schema"] 348 | append!(stmts, [ 349 | read_ontology_rdf_schema("list.ttl"); 350 | args["concepts"] ? read_ontology_rdf_schema("concept.ttl") : []; 351 | args["annotations"] ? read_ontology_rdf_schema("annotation.ttl") : []; 352 | args["wiring"] ? read_ontology_rdf_schema("wiring.ttl") : []; 353 | ]) 354 | end 355 | 356 | # Convert to RDF. 357 | prefix = Serd.RDF.Prefix("dso", "https://www.datascienceontology.org/ns/dso/") 358 | append!(stmts, ontology_to_rdf(db, prefix, 359 | include_provenance=args["provenance"], 360 | include_wiring_diagrams=args["wiring"])) 361 | 362 | # Serialize RDF to file or stdout. 363 | syntax = args["to"] 364 | if args["out"] != nothing 365 | open(args["out"], "w") do out 366 | Serd.write_rdf(out, stmts, syntax=syntax) 367 | end 368 | else 369 | Serd.write_rdf(stdout, stmts, syntax=syntax) 370 | end 371 | end 372 | 373 | function read_ontology_rdf_schema(name::AbstractString) 374 | Serd.read_rdf_file(joinpath(ontology_rdf_schema_dir, name)) 375 | end 376 | 377 | const ontology_rdf_schema_dir = joinpath(@__DIR__, "ontology", "rdf", "schema") 378 | 379 | # CLI main 380 | ########## 381 | 382 | function main(args) 383 | invoke(parse(args)...) 384 | end 385 | 386 | function parse(args) 387 | cmds = String[] 388 | parsed_args = parse_args(args, settings) 389 | while haskey(parsed_args, "%COMMAND%") 390 | cmd = parsed_args["%COMMAND%"] 391 | parsed_args = parsed_args[cmd] 392 | push!(cmds, cmd) 393 | end 394 | return (cmds, parsed_args) 395 | end 396 | 397 | function invoke(cmds, cmd_args) 398 | try 399 | cmd_fun = command_table 400 | for cmd in cmds; cmd_fun = cmd_fun[cmd] end 401 | cmd_fun(cmd_args) 402 | catch err 403 | # Handle further "parsing" errors ala ArgParse.jl. 404 | isa(err, ArgParseError) || rethrow() 405 | settings.exc_handler(settings, err) 406 | end 407 | end 408 | 409 | const command_table = Dict( 410 | "record" => record, 411 | "enrich" => enrich, 412 | "visualize" => visualize, 413 | "ontology" => Dict( 414 | "json" => ontology_as_json, 415 | "rdf" => ontology_as_rdf, 416 | ), 417 | ) 418 | 419 | # CLI extras 420 | ############ 421 | 422 | function __init__() 423 | @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" include("extras/CLI-Python.jl") 424 | @require RCall="6f49c342-dc21-5d91-9882-a32aef131414" include("extras/CLI-R.jl") 425 | end 426 | 427 | # CLI utilities 428 | ############### 429 | 430 | """ Map CLI input/output arguments to pairs of input/output files. 431 | """ 432 | function parse_io_args(input::AbstractString, output::Union{AbstractString,Nothing}; 433 | format::Any=nothing, formats::AbstractDict=Dict(), 434 | out_ext::Function=format->"") 435 | function get_format_and_output(input, output=nothing) 436 | if isnothing(output) || isnothing(format) 437 | name, ext = match_ext(input, keys(formats)) 438 | inferred_format = formats[ext] 439 | inferred_output = name * out_ext(inferred_format) 440 | end 441 | (isnothing(format) ? inferred_format : format, 442 | isnothing(output) ? inferred_output : output) 443 | end 444 | 445 | if isdir(input) 446 | inexts = collect(keys(formats)) 447 | names = filter(name -> any(endswith.(name, inexts)), readdir(input)) 448 | outdir = if isnothing(output); input 449 | elseif isdir(output); output 450 | else; throw(ArgParseError( 451 | "Output must be directory when input is directory")) 452 | end 453 | map(names) do name 454 | format, output = get_format_and_output(name) 455 | (joinpath(input, name), format, joinpath(outdir, output)) 456 | end 457 | elseif isfile(input) 458 | format, output = get_format_and_output(input, output) 459 | [ (input, format, output) ] 460 | else 461 | throw(ArgParseError("Input must be file or directory")) 462 | end 463 | end 464 | 465 | function match_ext(name::AbstractString, exts) 466 | # Don't use splitext because we allow extensions with multiple dots. 467 | for ext in exts 468 | if endswith(name, ext) 469 | return (name[1:end-length(ext)], ext) 470 | end 471 | end 472 | throw(ArgParseError("Cannot match extension in filename: \"$name\"")) 473 | end 474 | 475 | function read_graph_file(filename::AbstractString; 476 | kind::Union{AbstractString,Nothing}=nothing, format::AbstractString="graphml") 477 | reader = get(graph_readers, (kind, format)) do 478 | error("Invalid graph kind \"$kind\" or format \"$format\"") 479 | end 480 | reader(filename) 481 | end 482 | 483 | function write_graph_file(diagram::WiringDiagram, filename::AbstractString; 484 | format::AbstractString="graphml") 485 | writer = get(graph_writers, format) do 486 | error("Invalid graph format \"$format\"") 487 | end 488 | writer(diagram, filename) 489 | end 490 | 491 | const graph_readers = Dict( 492 | (nothing, "graphml") => x -> read_graphml(Dict, Dict, Dict, x), 493 | (nothing, "json") => x -> read_json_graph(Dict, Dict, Dict, x), 494 | ("raw", "graphml") => read_raw_graphml, 495 | ("raw", "json") => read_raw_graph_json, 496 | ("semantic", "graphml") => read_semantic_graphml, 497 | ("semantic", "json") => read_semantic_graph_json, 498 | ) 499 | 500 | const graph_writers = Dict( 501 | "graphml" => write_graphml, 502 | "json" => (x, filename) -> write_json_graph(x, filename, indent=2), 503 | ) 504 | 505 | end 506 | --------------------------------------------------------------------------------