├── src ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── routers.cpython-312.pyc │ ├── routers.cpython-38.pyc │ ├── triage.cpython-38.pyc │ ├── __init__.cpython-312.pyc │ └── myJSONQueryEngine.cpython-38.pyc ├── routers.py ├── triage.py └── myJSONQueryEngine.py ├── .env.example ├── requirements.txt ├── main.py ├── LICENSE ├── readme.md ├── .gitignore └── data └── pdf.json /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY= XX 2 | OPENAI_API_BASE= XX 3 | FILE_PATH = XX # './data/pdf.json' -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | llama-index==0.8.56 2 | openai==0.28.1 3 | jsonpath_ng==1.6.0 4 | python-dotenv==1.0.0 5 | -------------------------------------------------------------------------------- /src/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/routers.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/routers.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/routers.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/routers.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/triage.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/triage.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/__init__.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/__init__.cpython-312.pyc -------------------------------------------------------------------------------- /src/__pycache__/myJSONQueryEngine.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HAMNET-AI/PDFTriage/HEAD/src/__pycache__/myJSONQueryEngine.cpython-38.pyc -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from src.routers import router 2 | from dotenv import load_dotenv 3 | load_dotenv(override=True) 4 | 5 | if __name__ == '__main__': 6 | query = "The second figure of the paper is mainly about related work" 7 | router(query=query) 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 HAMNET.AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/routers.py: -------------------------------------------------------------------------------- 1 | from llama_index.tools import ToolMetadata 2 | from llama_index.selectors.llm_selectors import LLMSingleSelector 3 | from .triage import fetch_figure, fetch_pages, fetch_sections, fetch_table, retrieve 4 | 5 | 6 | def router(query): 7 | choices = [ 8 | ToolMetadata(description="Get the text contained in the pages listed", name="fetch_pages"), 9 | ToolMetadata(description="Get the text contained in the section listed", name="fetch_sections"), 10 | ToolMetadata(description="Get the text contained in the figure caption listed", name="fetch_figure"), 11 | ToolMetadata(description="Get the text contained in the table caption listed", name="fetch_table"), 12 | ToolMetadata(description="Issue a natural language query over the document, and fetch relevant chunks.", 13 | name="retrieve"), 14 | ] 15 | 16 | # choices as a list of strings 17 | # choices = ["fetch_pages", "fetch_sections", "fetch_figure", "fetch_table", "retrieve"] 18 | selector = LLMSingleSelector.from_defaults() 19 | result = selector.select(choices, query=query).selections 20 | flag = result[0].index 21 | print(flag) 22 | if flag == 0: 23 | content = fetch_pages(query=query) 24 | elif flag == 1: 25 | fetch_sections() 26 | elif flag == 2: 27 | fetch_figure(query=query) 28 | elif flag == 3: 29 | fetch_table(query=query) 30 | elif flag == 4: 31 | retrieve() 32 | else: 33 | print("No match found") 34 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## PDFTriage: Question Answering over Long, Structured Documents 2 | 3 | # 4 | 5 | PDFtriage: 用于结构化文档问答的新方法，提供了基于结构和内容的上下文检索，解决了现有模型在处理结构化文档时的局限性 6 | 7 | ## Dependencies 8 | 9 | - llama-index==0.8.56 10 | 11 | ## Usage 12 | 13 | 1. **Scan and Parse PDF Files** 14 | 15 | 首先，你需要将你的PDF文件扫描并解析为JSON格式。你可以使用以下网址将你的PDF文件解析为JSON数据：https://apifox.com/apidoc/shared-a55f1a3d-4871-41b7-8f1a-3af83807410b/api-120356017 16 | 17 | 2. **Set Environment Variables** 18 | 19 | 在使用 PDFtriage 之前，你需要设置一些环境变量。你可以在 .env.example 文件中找到示例，并按照相同的格式创建一个 .env 文件。以下是需要设置的环境变量： 20 | 21 | ```plaintext 22 | OPENAI_API_KEY=Your_OpenAI_API_Key 23 | OPENAI_API_BASE=API_Base_URL 24 | FILE_PATH=Path_to_the_JSON_Data_File_of_the_Parsed_PDF 25 | ## Example 26 | 27 | 在设置好环境变量之后，你可以按照以下示例使用 PDFtriage： 28 | ``` plaintext 29 | query = "What is the summary of the contents of table 1" 30 | result = router(query=query) 31 | print("Output:", result) 32 | ``` 33 | 34 | 运行以上代码，你将得到类似如下的输出： 35 | ``` plaintext 36 | "The summary of the contents of table 1 is as follows: 37 | 38 | - Positive triple: (Los Angeles, LocatedIn, California) 39 | - Negative sampling 40 | - High-quality negative triple: (Los Angeles, LocatedIn, Georgia) 41 | - False-negative triples: (Apple Inc., LocatedIn, California), (Los Angeles, LocatedIn, Apple Pie) 42 | - Low-quality negative triples: (English, LocatedIn, California) 43 | - Challenge 1: Invalid Negative Sampling 44 | - Query: (David, Nationality, ?) 45 | - Correct triple: (David, Nationality, U.S.A.) 46 | - Rank: Commonsense California 11, California Unsatisfy, David+rNationality-eCalifornia 2, U.S.A. Satisfy, David+rNationality-eU.S.A. 3 47 | - Challenge 2: Uncertainty of Fact-View Link Prediction 48 | - Figure 1: Two examples exhibit the challenges that needed to be addressed. 49 | - Challenge 1: Given a positive triple, some generated negative triples are false negative or low-quality. 50 | - Challenge 2: For link prediction, the entity ranks higher than the correct entity U.S.A. due to the uncertainty of KG embeddings, but the correct answer entity should belong to the concept Country in the view of commonsense. 51 | Process finished with exit code 0" 52 | ``` 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Python template 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # poetry 99 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 100 | # This is especially recommended for binary packages to ensure reproducibility, and is more 101 | # commonly ignored for libraries. 102 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 103 | #poetry.lock 104 | 105 | # pdm 106 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 107 | #pdm.lock 108 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 109 | # in version control. 110 | # https://pdm.fming.dev/#use-with-ide 111 | .pdm.toml 112 | 113 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 114 | __pypackages__/ 115 | 116 | # Celery stuff 117 | celerybeat-schedule 118 | celerybeat.pid 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env.example 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | .idea/ 162 | .env 163 | -------------------------------------------------------------------------------- /src/triage.py: -------------------------------------------------------------------------------- 1 | from llama_index.indices.service_context import ServiceContext 2 | from llama_index.llms import OpenAI 3 | from llama_index.llms import AzureOpenAI 4 | import httpx 5 | import json, ast 6 | from jsonpath_ng import jsonpath, parse 7 | from jsonpath_ng.ext import parse 8 | from .myJSONQueryEngine import JSONQueryEngine 9 | from dotenv import load_dotenv 10 | import os 11 | 12 | load_dotenv(override=True) 13 | dataschema = { 14 | "description": "This is a request reply containing a parsed pdf file", 15 | "$schema": "https://json-schema.org/draft/2020-12/schema", 16 | "type": "object", 17 | "properties": { 18 | "code": { 19 | "type": "integer" 20 | }, 21 | "data": { 22 | "description": "data is a structured data book derived from a pdf", 23 | "type": "array", 24 | "items": { 25 | "type": "object", 26 | "description": "List of contents of each page of the pdf", 27 | "properties": { 28 | "page": { 29 | "description": "Page number of each page", 30 | "type": "integer" 31 | }, 32 | "width": { 33 | "description": " Width of each page", 34 | "type": "integer" 35 | }, 36 | "height": { 37 | "description": " Each page of the high", 38 | "type": "integer" 39 | }, 40 | "boxes": { 41 | "description": "Content block of each page", 42 | "type": "array", 43 | "items": { 44 | "type": "object", 45 | "properties": { 46 | "bbox": { 47 | "description": "Bounding box coordinates of the text", 48 | "type": "array", 49 | "items": { 50 | "type": "integer" 51 | } 52 | }, 53 | "text": { 54 | "description": "Content of the block", 55 | "type": "string" 56 | }, 57 | "type": { 58 | "description": "type of the block", 59 | "type": "string" 60 | }, 61 | "index": { 62 | "type": "integer" 63 | }, 64 | "score": { 65 | "type": "number" 66 | }, 67 | "contain_formula": { 68 | "description": "Whether the text contains mathematical formulae", 69 | "type": "boolean" 70 | }, 71 | "font_size": { 72 | "description": "Font size of text", 73 | "type": "number" 74 | }, 75 | "font_name": { 76 | "description": "Font name for text", 77 | "type": "string" 78 | } 79 | }, 80 | "required": [ 81 | "bbox", 82 | "text", 83 | "type", 84 | "index", 85 | "score", 86 | "contain_formula", 87 | "font_size", 88 | "font_name" 89 | ] 90 | } 91 | } 92 | }, 93 | "required": [ 94 | "page", 95 | "width", 96 | "height", 97 | "boxes" 98 | ] 99 | } 100 | }, 101 | "msg": { 102 | "type": "string" 103 | } 104 | }, 105 | "required": [ 106 | "code", 107 | "data", 108 | "msg" 109 | ] 110 | } 111 | llm = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"), api_base=os.environ.get("OPENAI_API_BASE")) 112 | service_context = ServiceContext.from_defaults(llm=llm) 113 | filePath = os.environ.get("FILE_PATH") if os.environ.get("FILE_PATH") else "./data/pdf.json" 114 | with open(filePath, encoding="utf-8") as pdfdata: 115 | data = json.load(pdfdata) 116 | query_engine = JSONQueryEngine( 117 | json_value=data, 118 | json_schema=dataschema, 119 | service_context=service_context, 120 | synthesize_response=False, 121 | ) 122 | 123 | 124 | def get_num(query,type): 125 | prompt = f"""Please indicate in an array form which {type}s are referred to in a question,You just need to output an array 126 | example: 127 | query : What is the summary of the contents of {type} 1 128 | output : [1] 129 | query : What is the summary of the contents of {type} 2 and 6" \ 130 | output : [2,6] 131 | query : What is the summary of the contents of {type} 2 to 5" \ 132 | output : [2,3,4,5] 133 | --------------------------------------------------------------\ 134 | this is the question {query} Please indicate in an array form which {type}s are referred 135 | """ 136 | response = llm.complete(prompt) 137 | return response.__str__() 138 | 139 | 140 | def fetch_pages(query): 141 | # print("pages") 142 | query_prompt = f"What contents to the number of pages mentioned in this question : {query}" 143 | path = query_engine.query(query_prompt).metadata['json_path_response_str'].replace("&&", "&") 144 | # path = "$.data[?(@.page >= 5 & @.page <= 7)].boxes[*].text" 145 | # .replace("&&", "&") 146 | jsonpath_expression = parse(path) 147 | matches = jsonpath_expression.find(data) 148 | content = [match.value for match in matches] 149 | prompt = f"Please answer a question based on something in the pdf\n, this is the question{query}\n, The contents of the pages mentioned in the question are listed in text as follows {content}" 150 | response = llm.complete(prompt) 151 | print(response) 152 | 153 | 154 | def fetch_sections(query): 155 | print("Fetching sections") 156 | 157 | 158 | def fetch_figure(query): 159 | query_prompt = f"What contents mentioned in the figure of this pdf" 160 | path = query_engine.query(query_prompt).metadata['json_path_response_str'].replace("&&", "&") 161 | jsonpath_expression = parse(path) 162 | matches = jsonpath_expression.find(data) 163 | result = [match.value for match in matches] 164 | figure_indexs = get_num(query, type="figure") 165 | print(figure_indexs) 166 | figure_indexs = ast.literal_eval(figure_indexs) 167 | content = [f'figure{i}:{result[i]}' for i in figure_indexs] 168 | prompt = f"Please answer a question based on something in the pdf\n, this is the question{query}\n, The contents of figures, mentioned in the question are listed in text as follows {content}" 169 | response = llm.complete(prompt) 170 | print(response) 171 | 172 | 173 | 174 | def fetch_table(query): 175 | query_prompt = f"What contents mentioned in the table of this pdf" 176 | path = query_engine.query(query_prompt).metadata['json_path_response_str'].replace("&&", "&") 177 | jsonpath_expression = parse(path) 178 | matches = jsonpath_expression.find(data) 179 | result = [match.value for match in matches] 180 | # print("table") 181 | table_indexs = get_num(query,type="table") 182 | table_indexs = ast.literal_eval(table_indexs) 183 | # content = [result[i] for i in table_indexs] 184 | content = [f'table{i}:{result[i]}' for i in table_indexs] 185 | prompt = f"Please answer a question based on something in the pdf\n, this is the question{query}\n, The contents of tables, mentioned in the question are listed in text as follows {content}" 186 | response = llm.complete(prompt) 187 | print(response) 188 | 189 | 190 | def retrieve(): 191 | print("retrieve") 192 | 193 | 194 | def pdf_response(query, content): 195 | print("response") 196 | -------------------------------------------------------------------------------- /src/myJSONQueryEngine.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from typing import Any, Callable, Dict, List, Optional, Union 4 | 5 | from llama_index.indices.query.base import BaseQueryEngine 6 | from llama_index.indices.query.schema import QueryBundle 7 | from llama_index.indices.service_context import ServiceContext 8 | from llama_index.prompts import BasePromptTemplate, PromptTemplate 9 | from llama_index.prompts.default_prompts import DEFAULT_JSON_PATH_PROMPT 10 | from llama_index.prompts.mixin import PromptDictType, PromptMixinType 11 | from llama_index.prompts.prompt_type import PromptType 12 | from llama_index.response.schema import Response 13 | from llama_index.utils import print_text 14 | 15 | logger = logging.getLogger(__name__) 16 | IMPORT_ERROR_MSG = ( 17 | "`jsonpath_ng` package not found, please run `pip install jsonpath-ng`" 18 | ) 19 | 20 | JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] 21 | 22 | 23 | DEFAULT_RESPONSE_SYNTHESIS_PROMPT_TMPL = ( 24 | "Given a query, synthesize a response " 25 | "to satisfy the query using the JSON results. " 26 | "Only include details that are relevant to the query. " 27 | "If you don't know the answer, then say that.\n" 28 | "JSON Schema: {json_schema}\n" 29 | "JSON Path: {json_path}\n" 30 | "Value at path: {json_path_value}\n" 31 | "Query: {query_str}\n" 32 | "Response: " 33 | ) 34 | DEFAULT_RESPONSE_SYNTHESIS_PROMPT = PromptTemplate( 35 | DEFAULT_RESPONSE_SYNTHESIS_PROMPT_TMPL, 36 | prompt_type=PromptType.SQL_RESPONSE_SYNTHESIS, 37 | ) 38 | 39 | 40 | def default_output_processor(llm_output: str, json_value: JSONType) -> JSONType: 41 | """Default output processor that extracts values based on JSON Path expressions.""" 42 | # Split the given string into separate JSON Path expressions 43 | expressions = [expr.strip() for expr in llm_output.split(",")] 44 | 45 | try: 46 | from jsonpath_ng.ext import parse 47 | from jsonpath_ng.jsonpath import DatumInContext 48 | except ImportError as exc: 49 | IMPORT_ERROR_MSG = "You need to install jsonpath-ng to use this function!" 50 | raise ImportError(IMPORT_ERROR_MSG) from exc 51 | 52 | results = {} 53 | for expression in expressions: 54 | expression = expression.replace("&&", "&") 55 | try: 56 | datum: List[DatumInContext] = parse(expression).find(json_value) 57 | if datum: 58 | key = expression.split(".")[ 59 | -1 60 | ] # Extracting "title" from "$.title", for example 61 | results[key] = datum[0].value 62 | except Exception as exc: 63 | raise ValueError(f"Invalid JSON Path: {expression}") from exc 64 | 65 | return results 66 | 67 | 68 | class JSONQueryEngine(BaseQueryEngine): 69 | """GPT JSON Query Engine. 70 | 71 | Converts natural language to JSON Path queries. 72 | 73 | Args: 74 | json_value (JSONType): JSON value 75 | json_schema (JSONType): JSON schema 76 | service_context (ServiceContext): ServiceContext 77 | json_path_prompt (BasePromptTemplate): The JSON Path prompt to use. 78 | output_processor (Callable): The output processor that executes the 79 | JSON Path query. 80 | output_kwargs (dict): Additional output processor kwargs for the 81 | output_processor function. 82 | verbose (bool): Whether to print verbose output. 83 | """ 84 | 85 | def __init__( 86 | self, 87 | json_value: JSONType, 88 | json_schema: JSONType, 89 | service_context: ServiceContext, 90 | json_path_prompt: Optional[BasePromptTemplate] = None, 91 | output_processor: Optional[Callable] = None, 92 | output_kwargs: Optional[dict] = None, 93 | synthesize_response: bool = True, 94 | response_synthesis_prompt: Optional[BasePromptTemplate] = None, 95 | verbose: bool = False, 96 | **kwargs: Any, 97 | ) -> None: 98 | """Initialize params.""" 99 | self._json_value = json_value 100 | self._json_schema = json_schema 101 | self._service_context = service_context 102 | self._json_path_prompt = json_path_prompt or DEFAULT_JSON_PATH_PROMPT 103 | self._output_processor = output_processor or default_output_processor 104 | self._output_kwargs = output_kwargs or {} 105 | self._verbose = verbose 106 | self._synthesize_response = synthesize_response 107 | self._response_synthesis_prompt = ( 108 | response_synthesis_prompt or DEFAULT_RESPONSE_SYNTHESIS_PROMPT 109 | ) 110 | 111 | super().__init__(self._service_context.callback_manager) 112 | 113 | def _get_prompts(self) -> Dict[str, Any]: 114 | """Get prompts.""" 115 | return { 116 | "json_path_prompt": self._json_path_prompt, 117 | "response_synthesis_prompt": self._response_synthesis_prompt, 118 | } 119 | 120 | def _update_prompts(self, prompts: PromptDictType) -> None: 121 | """Update prompts.""" 122 | if "json_path_prompt" in prompts: 123 | self._json_path_prompt = prompts["json_path_prompt"] 124 | if "response_synthesis_prompt" in prompts: 125 | self._response_synthesis_prompt = prompts["response_synthesis_prompt"] 126 | 127 | def _get_prompt_modules(self) -> PromptMixinType: 128 | """Get prompt sub-modules.""" 129 | return {} 130 | 131 | def _get_schema_context(self) -> str: 132 | """Get JSON schema context.""" 133 | return json.dumps(self._json_schema) 134 | 135 | def _query(self, query_bundle: QueryBundle) -> Response: 136 | """Answer a query.""" 137 | schema = self._get_schema_context() 138 | 139 | json_path_response_str = self._service_context.llm_predictor.predict( 140 | self._json_path_prompt, 141 | schema=schema, 142 | query_str=query_bundle.query_str, 143 | ) 144 | 145 | if self._verbose: 146 | print_text( 147 | f"> JSONPath Instructions:\n" f"```\n{json_path_response_str}\n```\n" 148 | ) 149 | 150 | json_path_output = self._output_processor( 151 | json_path_response_str, 152 | self._json_value, 153 | **self._output_kwargs, 154 | ) 155 | 156 | if self._verbose: 157 | print_text(f"> JSONPath Output: {json_path_output}\n") 158 | 159 | if self._synthesize_response: 160 | response_str = self._service_context.llm_predictor.predict( 161 | self._response_synthesis_prompt, 162 | query_str=query_bundle.query_str, 163 | json_schema=self._json_schema, 164 | json_path=json_path_response_str, 165 | json_path_value=json_path_output, 166 | ) 167 | else: 168 | response_str = json.dumps(json_path_output) 169 | 170 | response_metadata = { 171 | "json_path_response_str": json_path_response_str, 172 | } 173 | return Response(response=response_str, metadata=response_metadata) 174 | 175 | async def _aquery(self, query_bundle: QueryBundle) -> Response: 176 | schema = self._get_schema_context() 177 | 178 | json_path_response_str = await self._service_context.llm_predictor.apredict( 179 | self._json_path_prompt, 180 | schema=schema, 181 | query_str=query_bundle.query_str, 182 | ) 183 | 184 | if self._verbose: 185 | print_text( 186 | f"> JSONPath Instructions:\n" f"```\n{json_path_response_str}\n```\n" 187 | ) 188 | 189 | json_path_output = self._output_processor( 190 | json_path_response_str, 191 | self._json_value, 192 | **self._output_kwargs, 193 | ) 194 | 195 | if self._verbose: 196 | print_text(f"> JSONPath Output: {json_path_output}\n") 197 | 198 | if self._synthesize_response: 199 | response_str = await self._service_context.llm_predictor.apredict( 200 | self._response_synthesis_prompt, 201 | query_str=query_bundle.query_str, 202 | json_schema=self._json_schema, 203 | json_path=json_path_response_str, 204 | json_path_value=json_path_output, 205 | ) 206 | else: 207 | response_str = json.dumps(json_path_output) 208 | 209 | response_metadata = { 210 | "json_path_response_str": json_path_response_str, 211 | } 212 | 213 | return Response(response=response_str, metadata=response_metadata) 214 | -------------------------------------------------------------------------------- /data/pdf.json: -------------------------------------------------------------------------------- 1 | { 2 | "code": 200, 3 | "data": [ 4 | { 5 | "page": 0, 6 | "width": 621, 7 | "height": 877, 8 | "boxes": [ 9 | { 10 | "bbox": [ 11 | 79, 12 | 65, 13 | 547, 14 | 595 15 | ], 16 | "text": "CAKE: A Scalable Commonsense-Aware Framework For Multi-ViewKnowledge Graph CompletionGuanglin Niu1,2, Bo Li2,3, Yongfei Zhang1, Shiliang Pu41 Beijing Key Laboratory of Digital Media, Beihang University, Beijing, China2 Institute of Artificial Intelligence, Beihang University, Beijing, China3 Hangzhou Innovation Institute, Beihang University, Hangzhou, China4Hikvision Research Institute, Hangzhou, China{beihangngl, boli, yfzhang}@buaa.edu.cn, pushiliang.hri@hikvision.comAbstractPositive triple:(Los Angeles, LocatedIn, California)(Los Angeles, LocatedIn, California)Negative samplingKnowledge graphs store a large number ofHigh-quality negative triple:(Los Angeles, LocatedIn, (Los Angeles, LocatedIn, GeorgiaGeorgia))((San FranciscoSan Francisco, LocatedIn, California), LocatedIn, California)factual triples while they are still incomplete, inevitably.False-negative triples:((Apple Inc.Apple Inc., LocatedIn, California), LocatedIn, California)The previous knowledge(Los Angeles, LocatedIn, (Los Angeles, LocatedIn, Apple PieApple Pie))graph completion (KGC) models predict missing links between entities merely relying onLow-quality negative triples: ((EnglishEnglish, LocatedIn, California), LocatedIn, California)Challenge 1: Invalid Negative Samplingfact-view data, ignoring the valuable commonsense knowledge.Query:Query:(David, Nationality, ?)(David, Nationality, ?)(David, Nationality, ?)belongs toCountryCorrect triple:Correct triple:(David, Nationality, U.S.A.)(David, Nationality, U.S.A.)(David, Nationality, U.S.A.)The previous knowledge graph embedding (KGE) techniques suffer from invalid negative sampling and theU.S.A.Rank:CommonsenseCalifornia11CaliforniaCaliforniaUnsatisfy||eDavid+rNationality-eCalifornia|| 2U.S.A.Satisfyuncertainty of fact-view link prediction, limiting KGC’s performance.To address theDavid< ||eDavid+rNationality-eU.S.A.||3...CanadaSatisfyChallenge 2: Uncertainty of Fact-View Link Predictionabove challenges, we propose a novel andscalable Commonsense-Aware KnowledgeFigure 1: Two examples exhibit the challenges thatEmbedding (CAKE) framework to automatically extract commonsense from factual triplesneeded to be addressed. Challenge 1: Given a positive triple, some generated negative triples are falsenegative or low-quality.with entity concepts. The generated commonsense augments effective self-supervision to Challenge 2 California: For link prediction, the entityfacilitate both high-quality negative sampling ranks higher than the correct entity(NS) and joint commonsense and fact-view U.S.A. due to the uncertainty of KG embeddings, but the correct answer entity should belong tolink prediction. Experimental results on thethe concept Country in the view of commonsense.KGC task demonstrate that assembling ourframework could enhance the performance ofthe original KGE models, and the proposedmodels mine logic rules for induction reasoning, such as AMIE+ (commonsense-aware NS module is superior toGalárraga et al., 2015other NS techniques. Besides, our proposedDRUM (Sadeghian et al., 2019) and AnyBurl (framework could be easily adaptive to variousMeilicke et al.KGE models and explain the predicted results., 2019). (2) Path-based models (Liu et al.2020; Xiong et al., 2017; Lin et al., 2018) search", 17 | "type": "Header", 18 | "index": 0, 19 | "score": 0.95166015625, 20 | "contain_formula": false, 21 | "font_size": 8.476447105407715, 22 | "font_name": "TimesNewRomanPSMT" 23 | }, 24 | { 25 | "bbox": [ 26 | 310, 27 | 216, 28 | 555, 29 | 506 30 | ], 31 | "text": "Positive triple:(Los Angeles, LocatedIn, California)(Los Angeles, LocatedIn, California)Negative samplingHigh-quality negative triple:(Los Angeles, LocatedIn, (Los Angeles, LocatedIn, GeorgiaGeorgia))((San FranciscoSan Francisco, LocatedIn, California), LocatedIn, California)False-negative triples:((Apple Inc.Apple Inc., LocatedIn, California), LocatedIn, California)(Los Angeles, LocatedIn, (Los Angeles, LocatedIn, Apple PieApple Pie))Low-quality negative triples: ((EnglishEnglish, LocatedIn, California), LocatedIn, California)Challenge 1: Invalid Negative SamplingQuery:Query:(David, Nationality, ?)(David, Nationality, ?)(David, Nationality, ?)belongs toCountryCorrect triple:Correct triple:(David, Nationality, U.S.A.)(David, Nationality, U.S.A.)(David, Nationality, U.S.A.)U.S.A.Rank:CommonsenseCalifornia11CaliforniaCaliforniaUnsatisfy||eDavid+rNationality-eCalifornia|| 2U.S.A.SatisfyDavid< ||eDavid+rNationality-eU.S.A.||3...CanadaSatisfyChallenge 2: Uncertainty of Fact-View Link PredictionFigure 1: Two examples exhibit the challenges thatneeded to be addressed. Challenge 1: Given a positive triple, some generated negative triples are falsenegative or low-quality. Challenge 2 California: For link prediction, the entity ranks higher than the correct entity U.S.A. due to the uncertainty of KG embeddings, but the correct answer entity should belong tothe concept Country in the view of commonsense.", 32 | "type": "Figure", 33 | "index": 1, 34 | "score": 0.94970703125, 35 | "contain_formula": false, 36 | "font_size": 7.10465145111084, 37 | "font_name": "TimesNewRomanPSMT" 38 | }, 39 | { 40 | "bbox": [ 41 | 71, 42 | 595, 43 | 164, 44 | 617 45 | ], 46 | "text": "1Introduction", 47 | "type": "Title", 48 | "index": 2, 49 | "score": 0.89013671875, 50 | "contain_formula": false, 51 | "font_size": 11.9552001953125, 52 | "font_name": "NimbusRomNo9L-Medi" 53 | }, 54 | { 55 | "bbox": [ 56 | 69, 57 | 620, 58 | 306, 59 | 810 60 | ], 61 | "text": "In recent years, knowledge graphs (KGs) suchas Freebase (Bollacker et al., 2008Lehmann et al., 2015), DBpedia () and NELL (Mitchellet al., 2018) have been widely used in manyknowledge-intensive applications, including question answering (Sun et al., 2020; Saxena et al.,2020), dialogue systems (Yang et al., 2020; Zhouet al., 2018) and recommender systems (Wang et al.,2021, 2019a). However, the KGs constructed manually or automatically are inevitably incomplete,requiring KGC to infer new facts.The previous KGC models can be classifiedinto three main streams: (1) Rule learning-based", 62 | "type": "Text", 63 | "index": 3, 64 | "score": 0.95654296875, 65 | "contain_formula": false, 66 | "font_size": 10.968130111694336, 67 | "font_name": "NimbusRomNo9L-Regu" 68 | }, 69 | { 70 | "bbox": [ 71 | 314, 72 | 522, 73 | 554, 74 | 809 75 | ], 76 | "text": "models mine logic rules for induction reasoning, such as AMIE+ (Galárraga et al., 2015),DRUM (Sadeghian et al., 2019) and AnyBurl (Meilicke et al., 2019). (2) Path-based models (Liu et al.,2020; Xiong et al., 2017; Lin et al., 2018) searchpaths for multi-hop reasoning. (3) KGE modelssuch as TransE (Bordes et al., 2013) and its variants (Sun et al., 2019; Zhang et al., 2019a, 2020)learn the embeddings of entities and relations toscore the plausibility of triples for link prediction.Among all the existing KGC models, KGEapproaches achieve higher efficiency and betterperformance. Specifically, the KGE-based KGCpipeline can be divided into two stages: learningknowledge graph (KG) embeddings at the trainingand link prediction at the inference. Learning KGembeddings relies on a basic procedure of negativesampling (Li et al., 2021). Link prediction aims toinfer the missing entity or relation in a triple viaranking the candidate triples’ scores in virtue of the", 77 | "type": "Text", 78 | "index": 4, 79 | "score": 0.962890625, 80 | "contain_formula": false, 81 | "font_size": 10.950942039489746, 82 | "font_name": "NimbusRomNo9L-Regu" 83 | } 84 | ] 85 | }, 86 | { 87 | "page": 1, 88 | "width": 621, 89 | "height": 877, 90 | "boxes": [ 91 | { 92 | "bbox": [ 93 | 315, 94 | 468, 95 | 417, 96 | 488 97 | ], 98 | "text": "2Related Work", 99 | "type": "Title", 100 | "index": 2, 101 | "score": 0.85498046875, 102 | "contain_formula": false, 103 | "font_size": 11.9552001953125, 104 | "font_name": "NimbusRomNo9L-Medi" 105 | }, 106 | { 107 | "bbox": [ 108 | 316, 109 | 489, 110 | 413, 111 | 508 112 | ], 113 | "text": "2.1KGC Models", 114 | "type": "Title", 115 | "index": 3, 116 | "score": 0.8798828125, 117 | "contain_formula": false, 118 | "font_size": 10.909099578857422, 119 | "font_name": "NimbusRomNo9L-Medi" 120 | }, 121 | { 122 | "bbox": [ 123 | 67, 124 | 67, 125 | 308, 126 | 811 127 | ], 128 | "text": "learned KG embeddings.However, the two separate stages both have drawbacks: (1) Invalid negative sampling: all theprevious NS (Wang et al., 2014; Cai and Wang,2018; Sun et al., 2019; Denis et al. Zhang et al., 2019b;, 2015) cannot avoid sampling the falsenegative triples and low-quality negative triples,simultaneously. For instance, given the positivetriple (Los Angeles, LocatedIn, California)as shown in Figure 1, the existing NS strategies might sample the corrupted triples suchas (San Francisco, LocatedIn, California),which is actually a missing correct triple namelyfalse-negative triple. On the other hand, the quality of some generated negative triples such as(San Francisco, LocatedIn, Apple Pie) is toopoor to make little sense for training the KGEmodels. (2) Uncertainty of fact-view link prediction: performing link prediction solely basedon facts in a data-driven fashion suffers from uncertainty due to the deviation of KG embeddingscompared to the symbolic representations, limiting the accuracy of KGC. Take the tail entity prediction (David, Nationality, ?) in Figure 1 as aninstance. The correct tail entity should belong tothe concept Country in the view of commonsense.Whereas the entity California that is inconsistentwith commonsense even ranks highest via scoringthe candidate triples with KG embeddings.Last but not least, although some KGE approaches exploit external information, includingentity types (Xie et al., 2016b), textual descriptions (Xie et al., 2016a) and images of entities (Xieet al., 2017). Such auxiliary information is hard toaccess and enhances the single representation ofentities rather than providing the semantics of commonsense. However, the valuable commonsenseis always acquired by the expensive hand annotation (Rajani et al., 2019), so its high cost leadsto relatively low coverage. Besides, the existinglarge-scale commonsense KGs such as ConceptNet (Speer et al., 2017) only contain the conceptswithout the links to the corresponding entities, causing them unavailable to the KGC task.To address the above challenges, we propose a novel and scalable Commonsense-AwareKnowledge Embedding (CAKE) framework to improve the NS in the training of KGE and boostthe performance of KGC benefited from the selfsupervision of commonsense. In specific, we attempt to automatically construct explicit common-", 129 | "type": "Text", 130 | "index": 0, 131 | "score": 0.9599609375, 132 | "contain_formula": false, 133 | "font_size": 10.948596000671387, 134 | "font_name": "NimbusRomNo9L-Regu" 135 | }, 136 | { 137 | "bbox": [ 138 | 315, 139 | 73, 140 | 555, 141 | 466 142 | ], 143 | "text": "sense via an instance abstraction technique fromKGs. Then, contrary to random sampling, we purposefully generate the high-quality negative triplesby taking advantage of the commonsense togetherwith the characteristics of complex relations. Furthermore, a multi-view link prediction is conductedto determine the entity candidates that belong tothe correct concepts in the commonsense view andpredict the answer entities with the learned KG embeddings from the perspective of fact. In summary,the contributions of our work are three-fold:• We propose a scalable KGC frameworkwith an automatic commonsense generation mechanism to extract valuable commonsense from factual triples and entity concepts.• We develop a commonsense-aware negativesampling strategy for generating valid andhigh-quality negative triples.Meanwhile,a multi-view link prediction mechanism isproposed to improve the accuracy of KGC.• Extensive experiments on four benchmarkdatasets illustrate the effectiveness and thescalability of our whole framework and eachmodule.The source code and datasets ofthis paper can be obtained from https://github.com/ngl567/CAKE.", 144 | "type": "Text", 145 | "index": 1, 146 | "score": 0.96728515625, 147 | "contain_formula": false, 148 | "font_size": 10.92646598815918, 149 | "font_name": "NimbusRomNo9L-Regu" 150 | }, 151 | { 152 | "bbox": [ 153 | 315, 154 | 510, 155 | 555, 156 | 810 157 | ], 158 | "text": "The existing KGC models can be classified intothree main categories: (1) Rule learning-based algorithms such as AMIE+ (Galárraga et al., 2015),DRUM (Sadeghian et al., 2019) and AnyBurl (Meilicke et al., 2019) automatically mine logic rulesfrom KGs and apply these rules for inductive linkprediction. However, these models are inefficientdue to the time-consuming rule searching and evaluation. (2) Path-based models search paths linkingLao et al., 2011head and tail entities, including path ranking approaches (; Liu et al., 2020) andreinforcement learning-based models (Xiong et al.,2017; Lin et al., 2018). Whereas, multi-hop pathbased models also spend much time in path searching. (3) KG embedding (KGE) models such asTransE (Bordes et al., 2013), RESCAL (Nickelet al., 2011), ComplEx (Trouillon et al., 2016), RotatE (Sun et al., 2019) and HAKE (Zhang et al.,2020) learn the embeddings of entities and relations to score the plausibility of triples for predicting the missing triples efficiently. KGE approaches", 159 | "type": "Text", 160 | "index": 4, 161 | "score": 0.96337890625, 162 | "contain_formula": false, 163 | "font_size": 10.948226928710938, 164 | "font_name": "NimbusRomNo9L-Regu" 165 | } 166 | ] 167 | }, 168 | { 169 | "page": 2, 170 | "width": 621, 171 | "height": 877, 172 | "boxes": [ 173 | { 174 | "bbox": [ 175 | 71, 176 | 198, 177 | 234, 178 | 222 179 | ], 180 | "text": "2.2Negative Sampling of KGE", 181 | "type": "Title", 182 | "index": 1, 183 | "score": 0.880859375, 184 | "contain_formula": false, 185 | "font_size": 10.909099578857422, 186 | "font_name": "NimbusRomNo9L-Medi" 187 | }, 188 | { 189 | "bbox": [ 190 | 316, 191 | 69, 192 | 510, 193 | 92 194 | ], 195 | "text": "2.3Commonsense Knowledge Graph", 196 | "type": "Title", 197 | "index": 3, 198 | "score": 0.86767578125, 199 | "contain_formula": false, 200 | "font_size": 10.909099578857422, 201 | "font_name": "NimbusRomNo9L-Medi" 202 | }, 203 | { 204 | "bbox": [ 205 | 315, 206 | 382, 207 | 411, 208 | 405 209 | ], 210 | "text": "3Methodology", 211 | "type": "Title", 212 | "index": 5, 213 | "score": 0.88818359375, 214 | "contain_formula": false, 215 | "font_size": 11.9552001953125, 216 | "font_name": "NimbusRomNo9L-Medi" 217 | }, 218 | { 219 | "bbox": [ 220 | 316, 221 | 713, 222 | 532, 223 | 735 224 | ], 225 | "text": "3.1Notations and Problem Formalization", 226 | "type": "Title", 227 | "index": 7, 228 | "score": 0.849609375, 229 | "contain_formula": false, 230 | "font_size": 10.909099578857422, 231 | "font_name": "NimbusRomNo9L-Medi" 232 | }, 233 | { 234 | "bbox": [ 235 | 69, 236 | 72, 237 | 306, 238 | 193 239 | ], 240 | "text": "achieve higher efficiency and better performance onKGC compared with the others. However, the natural uncertainty of embeddings limits the precisionof KGC relying solely on facts. More specifically,the KGE models generally need a primary negativesampling (NS) procedure to randomly or purposelysample some triples that are not observed in theKG as negative triples for training (Li et al., 2021).", 241 | "type": "Text", 242 | "index": 0, 243 | "score": 0.9482421875, 244 | "contain_formula": false, 245 | "font_size": 10.860613822937012, 246 | "font_name": "NimbusRomNo9L-Regu" 247 | }, 248 | { 249 | "bbox": [ 250 | 68, 251 | 225, 252 | 307, 253 | 809 254 | ], 255 | "text": "Following the local closed-world assumption(Dong et al., 2014), the existing NS techniques forKGE can be classified into five categories: (1) Randomly and uniformly sampling: the majority of theKGE models generate negative triples via randomlyreplacing an entity or relation in a positive triplefrom a uniform distribution (Wang et al., 2014).(2) Adversarial-based sampling: KBGAN (Cai andWang, 2018) integrates the KGE model with softmax probabilities to select the high-quality negativetriples in an adversarial training framework. Selfadversarial sampling (Sun et al., 2019) performssimilar to KBGAN, but it utilizes a self-scoringfunction without a generator. (3) Domain-basedsampling: domain-based NS (Wang et al., 2019b)and type-constrained NS (Denis et al., 2015) bothleverage domain or type constraints on samplingthe corrupted entities that belong to the correct domain. (4) Efficient sampling: NSCaching (Zhanget al., 2019b) employs cache containing candidatesof negative triples to improve the efficiency of sampling. (5) None-sampling: NS-KGE (Li et al.,2021) eliminates the NS procedure by convertingloss functions of KGE into a unified square loss.However, all the previous NS algorithms cannot address the issue of false-negative triples sincethese NS techniques, except for none sampling,would attempt to sample the corrupted triples withhigher probability while they might be correct andjust missing in the KG. Domain-based NS reliesheavily on the constraint of the single type ratherthan the commonsense, limiting the diversity ofnegative triples. KBGAN introduces generativeadversarial networks (GAN) in the NS framework,making the original model more complex and hardto train. None sampling eliminates the negativetriples and has to convert each original KGE modelinto square loss, which weakens the performanceof KGE models. These drawbacks of the NS strategies degrade the training of KGE and further limitthe performance of KGC.", 256 | "type": "Text", 257 | "index": 2, 258 | "score": 0.962890625, 259 | "contain_formula": false, 260 | "font_size": 10.944560050964355, 261 | "font_name": "NimbusRomNo9L-Regu" 262 | }, 263 | { 264 | "bbox": [ 265 | 315, 266 | 93, 267 | 554, 268 | 379 269 | ], 270 | "text": "Different from the factual triples, commonsensecould inject rich abstract knowledge into KGs.However, the valuable commonsense is hard toaccess due to the costly hand annotation. In recentyears, many researches attempt to construct generalcommonsense graphs such as ConceptNet (Speeret al., 2017), Microsoft Concept Graph (Ji et al.,2019) and ATOMIC (Sap et al., 2019). However,these commonsense graphs only contain the concepts without the links to the corresponding entities, causing them inapplicable to the KGC task.On the other hand, although some KGE modelssuch as JOIE (Hao et al., 2019) employ the ontology built-in most of the KGs, i.e., NELL (Mitchellet al., 2018) and DBpedia (Lehmann et al., 2015),the relations in ontology such as isA, partOf andrelatedTo mainly represent the type hierarchy butnot the explicit commonsense. Such relations areuseless for KGC because there are few overlapsbetween the ontological and the factual relations.", 271 | "type": "Text", 272 | "index": 4, 273 | "score": 0.96337890625, 274 | "contain_formula": false, 275 | "font_size": 10.967885971069336, 276 | "font_name": "NimbusRomNo9L-Regu" 277 | }, 278 | { 279 | "bbox": [ 280 | 315, 281 | 409, 282 | 554, 283 | 709 284 | ], 285 | "text": "In this section, we introduce our novel and scalableCAKE framework. As shown in Figure 2, the entirepipeline consists of three developed modules: theautomatic commonsense generation (ACG) module, the commonsense-aware negative sampling(CANS) module and the multi-view link prediction (MVLP) module. Firstly, the ACG moduleextracts the commonsense from the factual tripleswith the entity concepts via an instance abstraction mechanism (§ 3.2). Then, the CANS moduleemploys the generated commonsense to producethe high-quality negative triples, which takes thecharacteristics of complex relations into account(§ 3.3). Afterwards, our approach feeds the positive and the weighted negative triples into the KGEmodel for learning entity and relation embeddings(§ 3.4). Finally, the MVLP module conducts linkprediction in a coarse-to-fine fashion by filteringthe candidates in the view of commonsense andpredicting the answer entities with KG embeddingsfrom the candidates in the view of fact (§ 3.5).", 286 | "type": "Text", 287 | "index": 6, 288 | "score": 0.96337890625, 289 | "contain_formula": false, 290 | "font_size": 10.950066566467285, 291 | "font_name": "NimbusRomNo9L-Regu" 292 | }, 293 | { 294 | "bbox": [ 295 | 316, 296 | 736, 297 | 552, 298 | 809 299 | ], 300 | "text": "Commonsense.Commonsense has gainedwidespread attraction from its successful use inunderstanding high-level semantics, which is generally represented as the concepts with their ontological relations in some well-known commonsense", 301 | "type": "Text", 302 | "index": 8, 303 | "score": 0.927734375, 304 | "contain_formula": false, 305 | "font_size": 10.933244705200195, 306 | "font_name": "NimbusRomNo9L-Regu" 307 | } 308 | ] 309 | }, 310 | { 311 | "page": 3, 312 | "width": 621, 313 | "height": 877, 314 | "boxes": [ 315 | { 316 | "bbox": [ 317 | 62, 318 | 60, 319 | 559, 320 | 259 321 | ], 322 | "text": "Entity ConceptsStateCommonsenseStateConverterAttentive Concept-to-Entity negative tripleshigh-quality +triplespositive CountryPersonCompanyCityCompanyCitycc22t1t1KGE ModelCaliforniaLos Angelescch11h11r1-1cc11t1t1cc22h1h1r1-Ncccc2222t2t2Fact View:entity and relation ......t3t3embeddingsU.S.A.Facebookcc33h1h1cc44h1h1cc44t1t1scoringscoringfilterfiltercandidate candidate PersonFriendOfCountrycccc3333h2h2rN-1cc33t1t1cccc4444h2h2rN-Ncccc4444t2t2...David......h3h3......h3h3......t3t3Commonsense View:NationalityJackFriendOfTomSelecting Candidate Concepts via PersonCountryFactual KGEntity-to-Concept ConverterCommonsense and Complex RelationQuery:DavidNationality?ACG ModuleCANS ModuleMVLK ModuleFigure 2: An overview of the CAKE framework. The orange dotes indicate the entities. The green dotes representthe entity concepts. In the CANS module,j r1−j1, r1−N, rN−1 and rN−N denote the diverse complex relations of1-1, 1-N, N-1 and N-N, respectively. chi and cti indicate the i-th head concept and tail concept that are selected bythe commonsense and the characteristics of complex relations specific to the j-th relation.", 323 | "type": "Figure", 324 | "index": 0, 325 | "score": 0.9453125, 326 | "contain_formula": false, 327 | "font_size": 6.126091957092285, 328 | "font_name": "TimesNewRomanPSMT" 329 | }, 330 | { 331 | "bbox": [ 332 | 316, 333 | 525, 334 | 529, 335 | 549 336 | ], 337 | "text": "3.2Automatic Commonsense Generation", 338 | "type": "Title", 339 | "index": 3, 340 | "score": 0.86083984375, 341 | "contain_formula": false, 342 | "font_size": 10.909099578857422, 343 | "font_name": "NimbusRomNo9L-Medi" 344 | }, 345 | { 346 | "bbox": [ 347 | 67, 348 | 272, 349 | 308, 350 | 815 351 | ], 352 | "text": "graphs such as ConceptNet (Speer et al., 2017) andMicrosoft Concept Graph (Ji et al., 2019). Notably,we extend the commonsense in two forms: the individual form C1 and the set form C2. Both C1 and are the sets of triples while each triple in isC2 C1constituted of a head entity’s concept ch and a tailentity’s concept ct associated with their instancelevel relation r, which can be written as follows:(c, r, c)(1)C1 = {ht}On the contrary, each triple in consists of a C2relation r linking the corresponding head conceptset C and tail concept set C, which is shown as:htC2 = {(Ch, r, Ct)}(2)The detailed description of commonsense generation is introduced in section 3.2.KGE Score Function.We could leverage anyKGE model to learn the entity and relation embeddings owing to our scalable framework independent of the KGE model. Thus, we define a uniformsymbol E(h, r, t) to represent the score function ofany KEG model for evaluating the plausibility ofa triple (h, r, t). More specifically, the three mosttypical score function patterns are given as follows:(1) The translation-based score function, suchas TransE (Bordes et al., 2013):E(h, r, t) =h + r t(3) ∥ −∥where h, r and t denote the embeddings of headentity h, relation r and tail entity t, respectively.(2) The rotation-based score function, such asRotatE (Sun et al., 2019):E(h, r, t) =h r t(4) ∥ ◦ −∥where ◦ indicates the hardmard product.", 353 | "type": "Formula", 354 | "index": 1, 355 | "score": 0.9599609375, 356 | "contain_formula": false, 357 | "font_size": 10.607303619384766, 358 | "font_name": "NimbusRomNo9L-Regu" 359 | }, 360 | { 361 | "bbox": [ 362 | 314, 363 | 271, 364 | 556, 365 | 519 366 | ], 367 | "text": "(3) The tensor decomposition-based score function, such as DistMult (Yang et al., 2015):E(h, r, t) = h⊤diag(M)t(5)rwhere diag(M) represents the diagonal matrix ofrthe relation r.Link Prediction. Following most of the previousKGC models, we regard link prediction as an entityprediction task. Given a triple query with an entitymissing (h, r, ?) or (?, r, t), link prediction takesevery entity as a candidate. It calculates the scoreof each candidate triple by employing the learnedKG embeddings and the score function. Then, werank the candidate entities in light of their scoresand output the top n entities as results.", 368 | "type": "Formula", 369 | "index": 2, 370 | "score": 0.96240234375, 371 | "contain_formula": false, 372 | "font_size": 10.734399795532227, 373 | "font_name": "NimbusRomNo9L-Regu" 374 | }, 375 | { 376 | "bbox": [ 377 | 315, 378 | 551, 379 | 555, 380 | 815 381 | ], 382 | "text": "In terms of the representation of commonsensedefined in section 3.1, our approach could theoretically generate commonsense from any KG automatically as long as there exist some conceptslinked to the entities in the KG. Specifically, wedevelop an entity-to-concept converter to replacethe entities in each factual triple with corresponding concepts. Meanwhile, the relations in commonsense entail the instance-level relations in factual KGs. Take an instance in Figure 2, the factual triple (David, Nationality, U.S.A.) can betransformed to a concept-level triple (Person,.Particularly, the comNationality, Country)monsense in the individual form C1 is achieved bywiping out the reduplicated concept-level triples.Afterwards, we merge the concept-level triples thatcontain the same relation into a set to construct thecommonsense in the set form C2.", 383 | "type": "Text", 384 | "index": 4, 385 | "score": 0.8662109375, 386 | "contain_formula": false, 387 | "font_size": 10.793112754821777, 388 | "font_name": "NimbusRomNo9L-Regu" 389 | } 390 | ] 391 | }, 392 | { 393 | "page": 4, 394 | "width": 621, 395 | "height": 877, 396 | "boxes": [ 397 | { 398 | "bbox": [ 399 | 316, 400 | 708, 401 | 468, 402 | 730 403 | ], 404 | "text": "3.4Traning the KGE Model", 405 | "type": "Title", 406 | "index": 1, 407 | "score": 0.884765625, 408 | "contain_formula": false, 409 | "font_size": 10.909099578857422, 410 | "font_name": "NimbusRomNo9L-Medi" 411 | }, 412 | { 413 | "bbox": [ 414 | 67, 415 | 68, 416 | 307, 417 | 352 418 | ], 419 | "text": "Head EntityN-1 RelationTail EntityPositive atlantacitylocatedingeorgiaTriplestateSelecting cityCandidate citylocatedinstateprovinceConcepts countystatevisualizableislandsceneCommonsense C2Concept-to-EntityConcept-to-EntityHead EntitiesCorrupted Probability(Prob)Tail EntitiesCorrupted greenlandlowhightennesseeAttentive olympiavermontConceptto-Entity cambridgewashingtonConvertingzurichhighlowcalifornia. . . 1-ProbProb. . . weightweightHighquality greenlandgreenlandcitylocatedincitylocatedinstatestategeorgiageorgiaNegative citylocatedincitylocatedinTriplesatlantaatlantastatestatetennesseetennessee. . .Figure 3: An example of generating the high-qualitynegative triples containing an N-1 relation by our designed CANS module on NELL-995.", 420 | "type": "Figure", 421 | "index": 3, 422 | "score": 0.95849609375, 423 | "contain_formula": false, 424 | "font_size": 7.016597270965576, 425 | "font_name": "TimesNewRomanPS-BoldMT" 426 | }, 427 | { 428 | "bbox": [ 429 | 71, 430 | 367, 431 | 301, 432 | 392 433 | ], 434 | "text": "3.3Commonsense-Aware Negative Sampling", 435 | "type": "Title", 436 | "index": 4, 437 | "score": 0.86572265625, 438 | "contain_formula": false, 439 | "font_size": 10.909099578857422, 440 | "font_name": "NimbusRomNo9L-Medi" 441 | }, 442 | { 443 | "bbox": [ 444 | 313, 445 | 73, 446 | 559, 447 | 700 448 | ], 449 | "text": "try to sample the triples conforming to the commonsense2 for high quality. CFor a better understanding, an example of generating high-quality negative triples with an N-1 relation is shown in Figure 3. The whole NS procedurecan be divided into two steps. Step 1: selecting thecandidate concepts with commonsense. The can C2didate head concepts city, county and island aredetermined according to commonsense and nonunique sampling. Besides, based on the uniqueness C2sampling strategy, the candidate tail concept is selected as the same concept as that stateprovinceof Georgia. Step 2: attentive concept-to-entityconverting. To reduce false-negative while ensuring the high quality of the negative triples, the corrupted entities belonging to the candidate conceptsare sampled from the following distribution:w(h′j, r, t) = 1 − p((h′j, r, t)|{(hi, ri, ti)})exp αE(h′j, r, t)= 1 −exp αE(h′, r, t)(6)�i iw(h, r, t′j) = p((h, r, t′j)(hi, ri, ti))|{}exp αE(h, r, t′j)=(7)�i exp αE(h, r, t′i)where h′i and t′i are the corrupted head and tail entities obtained by non-unique sampling and uniqueness sampling. w and p denote the weight and theprobability of the negative triple, respectively. α isthe temperature of sampling motivated by the selfadversarial sampling (Sun et al., 2019). Remarkably, considering that a triple with a higher probability is more likely to be a positive one, the weightof a negative triple containing the corrupted headentity such is defined as Eq. 6 to prevent the issueof false-negative. Besides, the negative triples containing the corrupted tail entities with higher probability are endowed with higher-quality weight sincethere is no false-negative issue. Thus, both the corrupted head entity greenland and the corruptedtail entity tennessee with the high weights are selected to generate high-quality negative triples.", 450 | "type": "Formula", 451 | "index": 0, 452 | "score": 0.95068359375, 453 | "contain_formula": false, 454 | "font_size": 10.38243579864502, 455 | "font_name": "NimbusRomNo9L-Regu" 456 | }, 457 | { 458 | "bbox": [ 459 | 315, 460 | 733, 461 | 553, 462 | 811 463 | ], 464 | "text": "Based on the negative triples obtained by CANS,we train the KGE model to learn the entity andrelation embeddings for enlarging the gap betweenthe scores of the positive and high-quality negativetriples. In this work, we employ the following loss", 465 | "type": "Text", 466 | "index": 2, 467 | "score": 0.94775390625, 468 | "contain_formula": false, 469 | "font_size": 10.906533241271973, 470 | "font_name": "NimbusRomNo9L-Regu" 471 | }, 472 | { 473 | "bbox": [ 474 | 68, 475 | 395, 476 | 307, 477 | 810 478 | ], 479 | "text": "Intuitively, the negative triples satisfying commonsense are more challenging to distinguish from positive triples, contributing to more effective trainingsignals. Therefore, we try to sample the negativetriples that conform to the commonsense.To reduce the false-negative triples, we exploitthe characteristics of complex relations, namely1-1, 1-N, N-1, and N-N defined in TransH (Wanget al., 2014) for negative sampling, where 1 impliesthat the entity is unique when given the relation andanother entity, on the contrary, N denotes that theremight be multiple entities in this case (non-uniqueentity). Based on this observation, two specificsampling strategies are proposed: (1) uniquenesssampling: in terms of corrupting a unique entitysuch as the tail entity of the N-1 relation, the corrupted triples except for the original positive oneare definitely actual negative triples. Furthermore,the corrupted entities that share at least one conceptwith the correct entity are regarded as high-qualitynegative triples, contributing to a more consistenttraining signal. (2) None-unique sampling: forcorrupting a non-unique entity such as a head entity linked by the N-1 relation, the entities belonging to the same concept(s) with the correct entityare more likely to be false-negative due to the nonuniqueness of the head entity. Thus, the weights ofthese negative triples being false-negative shouldbe as low as possible in training. Meanwhile, we", 480 | "type": "Text", 481 | "index": 5, 482 | "score": 0.90869140625, 483 | "contain_formula": false, 484 | "font_size": 10.917460441589355, 485 | "font_name": "NimbusRomNo9L-Regu" 486 | } 487 | ] 488 | }, 489 | { 490 | "page": 5, 491 | "width": 621, 492 | "height": 877, 493 | "boxes": [ 494 | { 495 | "bbox": [ 496 | 313, 497 | 68, 498 | 561, 499 | 191 500 | ], 501 | "text": "Dataset#Rel#Ent#Con#Train#Valid#TestFB15K1,345 14,95189483,142 50,000 59,071FB15K23723714,50589272,115 17,535 20,466NELL-99520075,492270123,370 15,000 15,838DBpedia-24229899,744242592,654 35,851 30,000Table 1: Statistics of the experimental datasets. #Rel,#Ent, #Con represent the number of relations, entitiesand concepts of each dataset, respectively.", 502 | "type": "Table", 503 | "index": 0, 504 | "score": 0.94580078125, 505 | "contain_formula": false, 506 | "font_size": 9.06945514678955, 507 | "font_name": "NimbusRomNo9L-Regu" 508 | }, 509 | { 510 | "bbox": [ 511 | 316, 512 | 204, 513 | 448, 514 | 226 515 | ], 516 | "text": "4.1Experiment Settings", 517 | "type": "Title", 518 | "index": 1, 519 | "score": 0.88330078125, 520 | "contain_formula": false, 521 | "font_size": 10.909099578857422, 522 | "font_name": "NimbusRomNo9L-Medi" 523 | }, 524 | { 525 | "bbox": [ 526 | 315, 527 | 771, 528 | 553, 529 | 811 530 | ], 531 | "text": "1The codes of TransE and RotatE: https://github.com/DeepGraphLearning/KnowledgeGraphEmbedding. The codeof HAKE: https://github.com/MIRALab-USTC/KGE-HAKE.", 532 | "type": "Footnote", 533 | "index": 3, 534 | "score": 0.8759765625, 535 | "contain_formula": false, 536 | "font_size": 8.578521728515625, 537 | "font_name": "NimbusRomNo9L-Regu" 538 | }, 539 | { 540 | "bbox": [ 541 | 71, 542 | 190, 543 | 238, 544 | 211 545 | ], 546 | "text": "3.5Multi-View Link Prediction", 547 | "type": "Title", 548 | "index": 5, 549 | "score": 0.873046875, 550 | "contain_formula": false, 551 | "font_size": 10.909099578857422, 552 | "font_name": "NimbusRomNo9L-Medi" 553 | }, 554 | { 555 | "bbox": [ 556 | 71, 557 | 653, 558 | 230, 559 | 676 560 | ], 561 | "text": "4Experiments and Results", 562 | "type": "Title", 563 | "index": 7, 564 | "score": 0.8818359375, 565 | "contain_formula": false, 566 | "font_size": 11.9552001953125, 567 | "font_name": "NimbusRomNo9L-Medi" 568 | }, 569 | { 570 | "bbox": [ 571 | 315, 572 | 227, 573 | 554, 574 | 766 575 | ], 576 | "text": "Datasets.Four real-world datasets containing ontological concepts are utilized for experiments, including FB15K (Bordes et al., 2013),FB15K237 (Toutanova and Chen, 2015), NELL995 (Xiong et al., 2017) and DBpedia-242. Particularly, DBpedia-242 is extracted from DBpedia (Lehmann et al., 2015) which contains totally242 concepts. The statistics of the datasets aresummarized in Table 1. Notably, the entities inFB15K and FB15K237 always belong to more thanone concept while each entity in NELL-995 andDBpedia-242 has only one concept.Baselines. We compare our CAKE model withthree state-of-the-art KGE models, includingTransE (Bordes et al., 2013), RotatE (Sun et al.,2019) and HAKE (Zhang et al., 2020). Meanwhile,these baselines are also the basic models integratedwith our framework. It is unnecessary to use manybaselines since the focus of this work is to observethe impact of applying our CAKE framework tooriginal KGE models instead of defeating all theSOTA models. We provide the results of baselinesby running their source codes1 with the suggestedparameters. Note that all the existing type-basedand ontology-based models are not chosen as baselines since they are specific to a few KGs and cannot work on most of the datasets in our experiment.Implementation Details. Each complex relationis labelled in the same way as in TransH (Wanget al., 2014). We use Adam optimizer for the training and tune the hyper-parameters of our model bygrid search on the validation sets. Specifically, theembedding size and the batch size are the same asthose of each basic model for a fair comparison.The negative sampling size is set as 2 for all themodels. The learning rate is chosen from 0.0001to 0.01. The margin is tuned in {9, 12, 18, 24, 30}.The sampling temperature is adjusted in {0.5, 1.0}.", 577 | "type": "Text", 578 | "index": 2, 579 | "score": 0.955078125, 580 | "contain_formula": false, 581 | "font_size": 10.90893840789795, 582 | "font_name": "NimbusRomNo9L-Regu" 583 | }, 584 | { 585 | "bbox": [ 586 | 68, 587 | 70, 588 | 306, 589 | 188 590 | ], 591 | "text": "function as our optimization objective:L = − logσ(γ − E(h, r, t))n−�0.5[w(h′i, r, t)logσ(E(h′i, r, t) − γ)(8)i+ w(h, r, t′i)logσ(E(h, r, t′i) γ)] −in which γ is the margin. [x]+ indicates the largervalue between 0 and x. σ is the sigmoid function.", 592 | "type": "Formula", 593 | "index": 4, 594 | "score": 0.9521484375, 595 | "contain_formula": false, 596 | "font_size": 8.887383460998535, 597 | "font_name": "NimbusRomNo9L-Regu" 598 | }, 599 | { 600 | "bbox": [ 601 | 67, 602 | 213, 603 | 308, 604 | 650 605 | ], 606 | "text": "Benefiting from the same relations among commonsense and facts, commonsense could directlyprovide a definite range for link prediction results.Hence we develop a novel multi-view link prediction (MVLK) mechanism in a coarse-to-fineparadigm to facilitate more likely predicted results.Firstly, at the coarse prediction stage, we pick outthe candidate entities in the view of commonsense.Specifically, take a query (h, r, ?) for an example,commonsense is employed for filtering the rea C1sonable concepts of the tail entity. The candidateconcept set of tail entity is defined asC1t =cti(chi, r, cti)1(9) {| ∈ C}where c is the i-th concept of h, and c denoteshitithe tail concept in the commonsense (chi, r, cti).Then, the entities belonging to the concept set C1tcan be determined as the candidate entities sincethey satisfy commonsense and are more likely tobe the correct tail entities from the perspective ofcommonsense compared with other entities.Then, at the fine prediction stage, we score eachcandidate entity ei derived from the coarse prediction stage in the view of fact as followingscore(ei) = E(h, r, ei)(10)where E(h, r, ei) denotes the score function employed for training the KGE model. Subsequently,the prediction results will rank the scores of candidate entities in ascending order and output theentities with higher ranks.", 607 | "type": "Formula", 608 | "index": 6, 609 | "score": 0.9716796875, 610 | "contain_formula": false, 611 | "font_size": 10.362236976623535, 612 | "font_name": "NimbusRomNo9L-Regu" 613 | }, 614 | { 615 | "bbox": [ 616 | 69, 617 | 678, 618 | 307, 619 | 811 620 | ], 621 | "text": "In this section, we perform extensive experimentsof KGC on four widely-used KG datasets containing concepts. We firstly describe datasets, baseline models, implementation details and evaluationprotocol. Then, the effectiveness of our proposedframework CAKE and each module is demonstrated by compared with several baselines. Furthermore, we conduct further experiments, including the ablation study and the case study.", 622 | "type": "Text", 623 | "index": 8, 624 | "score": 0.943359375, 625 | "contain_formula": false, 626 | "font_size": 10.947019577026367, 627 | "font_name": "NimbusRomNo9L-Regu" 628 | } 629 | ] 630 | }, 631 | { 632 | "page": 6, 633 | "width": 621, 634 | "height": 877, 635 | "boxes": [ 636 | { 637 | "bbox": [ 638 | 67, 639 | 62, 640 | 554, 641 | 444 642 | ], 643 | "text": "ModelsFB15KFB15K237MRMRRHits@10Hits@3Hits@1MRMRRHits@10Hits@3Hits@1TransE350.6260.8380.7230.4961950.2680.4540.2980.176TransE+CANS340.6710.8640.7610.5521750.2980.4900.3330.203TransE+MVLP350.6360.8390.7250.5131810.2900.4760.3230.186TransE+CAKE330.6720.8650.7610.5551750.3010.4930.3350.206RotatE350.6570.8500.7460.5372040.2690.4520.2980.179RotatE+CANS330.7020.8770.7900.5881820.2960.4860.3290.202RotatE+MVLP340.6880.8600.7680.5791880.3080.4930.3400.217RotatE+CAKE310.7050.8780.7920.5931810.3180.5110.3540.223HAKE340.6900.8720.7800.5741760.3060.4860.3370.216HAKE+CANS370.7230.8820.8080.6161740.3150.5010.3440.221HAKE+MVLP320.7290.8900.8170.6221720.3200.5080.3520.227HAKE+CAKE300.7410.8960.8250.6461700.3210.5150.3550.226ModelsDBpedia-242NELL-995MRMRRHits@10Hits@3Hits@1MRMRRHits@10Hits@3Hits@1TransE27330.2420.4680.3440.10010810.4290.5570.4770.354TransE+CANS18890.2870.5750.4270.10310220.4330.5910.4950.336TransE+MVLP8810.3220.5850.4500.1523360.5090.6170.5470.444TransE+CAKE8810.3300.5950.4580.1603170.5330.6500.5780.461RotatE19500.3740.5820.4570.24920770.4600.5530.4930.403RotatE+CANS10630.4070.5930.4760.30010970.5310.6440.5730.461RotatE+0.3930.5940.4740.2733560.5190.6280.5640.447MVLP983RotatE+CAKE10270.4230.6030.4860.3203290.5460.6600.5920.474HAKE17570.4080.5790.4630.31211570.5020.6100.5380.437HAKE+CANS11470.4270.5870.4720.34120110.5200.6400.5560.451HAKE+MVLP10830.4110.5800.4630.3194780.5100.6140.5510.444HAKE+CAKE9310.4370.5930.4810.3534330.5430.6550.5830.477Table 2: Link prediction results on four datasets. Bold numbers are the best results for each type of model.", 644 | "type": "Table", 645 | "index": 0, 646 | "score": 0.94287109375, 647 | "contain_formula": false, 648 | "font_size": 8.976009368896484, 649 | "font_name": "NimbusRomNo9L-Regu" 650 | }, 651 | { 652 | "bbox": [ 653 | 72, 654 | 626, 655 | 208, 656 | 648 657 | ], 658 | "text": "4.2Experimental Results", 659 | "type": "Title", 660 | "index": 2, 661 | "score": 0.884765625, 662 | "contain_formula": false, 663 | "font_size": 10.909099578857422, 664 | "font_name": "NimbusRomNo9L-Medi" 665 | }, 666 | { 667 | "bbox": [ 668 | 70, 669 | 459, 670 | 306, 671 | 622 672 | ], 673 | "text": "All the experiments are conducted in Pytorch andon GeForce GTX 2080Ti GPUs.Evaluation Protocol. Following the procedure ofMVLP in Section 3.5, we can obtain the rank ofthe correct entity for each test example. Then, theperformance of link prediction is evaluated by threecommonly-used metrics: mean rank (MR), meanreciprocal rank (MRR), and proportion of the correct entities ranked in the top n (Hits@N). All themetrics are in the filtered setting by wiping out thecandidate triples already exist in the datasets.", 674 | "type": "Text", 675 | "index": 1, 676 | "score": 0.95654296875, 677 | "contain_formula": false, 678 | "font_size": 10.947418212890625, 679 | "font_name": "NimbusRomNo9L-Regu" 680 | }, 681 | { 682 | "bbox": [ 683 | 69, 684 | 650, 685 | 307, 686 | 810 687 | ], 688 | "text": "Table 2 exhibits the evaluation results of link prediction on the four datasets. We can observe thatboth CANS and MVLP modules effectively improve the performance of each basic model on eachdataset. Moreover, the entire CAKE framework further facilitates more performance gains than eachseparate module and outperforms all the baselinesconsistently and significantly. Compared with theperformance average of the three baseline models,our CAKE model improves MRR by 7.2%, 11.5%,16.2% and 16.7% on FB15K, FB15K237, DBpedia-", 689 | "type": "Text", 690 | "index": 3, 691 | "score": 0.95166015625, 692 | "contain_formula": false, 693 | "font_size": 10.903618812561035, 694 | "font_name": "NimbusRomNo9L-Regu" 695 | }, 696 | { 697 | "bbox": [ 698 | 314, 699 | 459, 700 | 555, 701 | 810 702 | ], 703 | "text": "242 and NELL-995. These results demonstrate thesuperiority and effectiveness of integrating commonsense with the original KGE models.We compare our CANS module with varioustypes of NS techniques, including uniform sampling (Bordes et al., 2013), none sampling (Li et al.,2021), NSCaching (Zhang et al., 2019b), domainbased sampling (Wang et al., 2019b) and selfadversarial sampling (Sun et al., 2019). The comparison results are obtained by combining theseNS techniques with the most classical KGE modelTransE(Bordes et al., 2013).From the resultsshown in Table 3, our CANS module significantlyoutperforms all the other NS techniques on allthe datasets. Specifically, domain-based NS, selfadversarial sampling and our CANS module consistently outperform the others due to evaluating thequality of negative triples. Furthermore, our CANSmodule performs better than domain-based NS andself-adversarial sampling since CANS could reduce false-negative. These results illustrate thesuperior ability of our CANS module to generatemore high-quality negative triples for enhancingthe performance of any KGE model.", 704 | "type": "Text", 705 | "index": 4, 706 | "score": 0.9658203125, 707 | "contain_formula": false, 708 | "font_size": 10.948760032653809, 709 | "font_name": "NimbusRomNo9L-Regu" 710 | } 711 | ] 712 | }, 713 | { 714 | "page": 7, 715 | "width": 621, 716 | "height": 877, 717 | "boxes": [ 718 | { 719 | "bbox": [ 720 | 70, 721 | 60, 722 | 554, 723 | 308 724 | ], 725 | "text": "ModelsFB15KFB15K237MRMRR Hits@10 Hits@3 Hits@1MRMRR Hits@10 Hits@3 Hits@1TransE+Unifo1780.3010.5050.3390.2013610.1710.3230.1820.097TransE+NoSamp1440.3500.5780.4150.2273430.2610.4460.2970.168TransE+NSCach2090.2920.5600.3750.1445560.2050.3530.2260.131TransE+Domain350.6190.8390.7150.4891860.2830.4670.3140.190TransE+SAdv350.6260.8380.7230.4961950.2680.4540.2980.176TransE+CANS (Ours)340.6710.8640.7610.5521750.2980.4900.3330.203ModelsDBpedia-242NELL-995MRMRR Hits@10 Hits@3 Hits@1MRMRR Hits@10 Hits@3 Hits@1TransE+Unifo5750 0.1240.2620.1830.03386500.1670.3540.2190.068TransE+None2292 0.2020.3950.2470.10191720.1760.2970.2100.106TransE+NSCach5465 0.1560.3400.2120.05013967 0.1070.2050.1220.107TransE+Domain3415 0.2030.5100.3460.00913190.3810.5490.4680.271TransE+SAdv2733 0.2420.4680.3440.10010810.4290.5570.4770.354TransE+CANS (Ours) 1889 0.2870.5750.4270.10310220.4330.5910.4950.336Table 3: Comparison results of various NS techniques. Unifo, NoSamp, NSCach, Domain and SAdv denoteuniform sampling, none sampling, NSCaching, domain-based NS and self-adversarial NS strategies, respectively.", 726 | "type": "Table", 727 | "index": 0, 728 | "score": 0.9345703125, 729 | "contain_formula": false, 730 | "font_size": 8.980631828308105, 731 | "font_name": "NimbusRomNo9L-Regu" 732 | }, 733 | { 734 | "bbox": [ 735 | 311, 736 | 319, 737 | 562, 738 | 517 739 | ], 740 | "text": "ModelsFB15K237MRMRRHits@10Hits@3Hits@1CAKE1700.3210.5150.3550.226-CRNS1860.3180.5070.3520.223-CSNS1820.3170.5090.3510.222-MVLP1740.3150.5010.3440.221ModelsNELL-995MRMRRHits@10Hits@3Hits@1CAKE4330.5430.6550.5830.477-CRNS6500.5190.6270.5640.453-CSNS4470.5290.6470.5670.463-MVLP20110.5200.6400.5560.451Table 4: Ablation study of integrating each model intothe basic model HAKE on FB15K237 and NELL-995.", 741 | "type": "Table", 742 | "index": 1, 743 | "score": 0.94775390625, 744 | "contain_formula": false, 745 | "font_size": 8.997530937194824, 746 | "font_name": "NimbusRomNo9L-Regu" 747 | }, 748 | { 749 | "bbox": [ 750 | 315, 751 | 696, 752 | 401, 753 | 717 754 | ], 755 | "text": "5Conclusion", 756 | "type": "Title", 757 | "index": 3, 758 | "score": 0.8857421875, 759 | "contain_formula": false, 760 | "font_size": 11.9552001953125, 761 | "font_name": "NimbusRomNo9L-Medi" 762 | }, 763 | { 764 | "bbox": [ 765 | 69, 766 | 316, 767 | 306, 768 | 415 769 | ], 770 | "text": "Query:(rockets, teamplaysinleague, ?)(rockets, teamplaysinleague, ?)Commonsense: (sportsteam, teamplaysinleague,(sportsteam, teamplaysinleague, sportsleaguesportsleague))Top 5 Answers:nbanbancaancaawnbawnbacbacbaaccaccsportsleagueEntity Concept:Figure 4: A case study of explainable link predictionwith commonsense and entity concept on NELL-995.", 771 | "type": "Figure", 772 | "index": 5, 773 | "score": 0.9306640625, 774 | "contain_formula": false, 775 | "font_size": 8.226518630981445, 776 | "font_name": "TimesNewRomanPSMT" 777 | }, 778 | { 779 | "bbox": [ 780 | 71, 781 | 428, 782 | 177, 783 | 449 784 | ], 785 | "text": "4.3Ablation Study", 786 | "type": "Title", 787 | "index": 6, 788 | "score": 0.89501953125, 789 | "contain_formula": false, 790 | "font_size": 10.909099578857422, 791 | "font_name": "NimbusRomNo9L-Medi" 792 | }, 793 | { 794 | "bbox": [ 795 | 71, 796 | 727, 797 | 157, 798 | 748 799 | ], 800 | "text": "4.4Case Study", 801 | "type": "Title", 802 | "index": 8, 803 | "score": 0.8896484375, 804 | "contain_formula": false, 805 | "font_size": 10.909099578857422, 806 | "font_name": "NimbusRomNo9L-Medi" 807 | }, 808 | { 809 | "bbox": [ 810 | 315, 811 | 531, 812 | 554, 813 | 693 814 | ], 815 | "text": "995, our model could output the answer entities and provide the corresponding entity concepts together with the commonsense specificto the query.We can observe that all the top5 entities including the correct entity nba belong to the concept sportsleague which satisfiesthe commonsense (rockets, teamplaysinleague,sportsleague). More interestingly, the commonsense and the entity concepts could explain the rationality of the predicted answer entities to enhancethe users’ credibility of the answers.", 816 | "type": "Text", 817 | "index": 2, 818 | "score": 0.955078125, 819 | "contain_formula": false, 820 | "font_size": 10.938491821289062, 821 | "font_name": "NimbusRomNo9L-Regu" 822 | }, 823 | { 824 | "bbox": [ 825 | 315, 826 | 720, 827 | 553, 828 | 812 829 | ], 830 | "text": "In this paper, we propose a novel and scalablecommonsense-aware knowledge embedding framework, which could automatically generate commonsense from KGs with entity concepts for the KGCtask. We exploit the generated commonsense toproduce effective and high-quality negative triples.", 831 | "type": "Text", 832 | "index": 4, 833 | "score": 0.9365234375, 834 | "contain_formula": false, 835 | "font_size": 10.924032211303711, 836 | "font_name": "NimbusRomNo9L-Regu" 837 | }, 838 | { 839 | "bbox": [ 840 | 69, 841 | 451, 842 | 307, 843 | 723 844 | ], 845 | "text": "We verify the effectiveness of each contributionvia integrating the whole framework CAKE andthe following ablated models into the basic modelHAKE: (1) neglecting the characteristics of complex relations in CANS (-CRNS), (2) removing thecommonsense in CANS while retaining the characteristics of complex relations (-CSNS), and (3)omitting the commonsense-view prediction fromMVLP (-MVLP). The results in Table 4 demonstrate that our whole model CAKE performs betterthan all the ablated models on each dataset. It illustrates that introducing commonsense and the characteristics of complex relations both make sensein the NS process for generating more effectivenegative triples. Besides, MVLP facilitates linkprediction performance benefited from determiningthe reasonable candidate entities by prior commonsense. In general, each contribution plays a pivotalrole in our approach.", 846 | "type": "Text", 847 | "index": 7, 848 | "score": 0.96337890625, 849 | "contain_formula": false, 850 | "font_size": 10.941399574279785, 851 | "font_name": "NimbusRomNo9L-Regu" 852 | }, 853 | { 854 | "bbox": [ 855 | 70, 856 | 749, 857 | 307, 858 | 810 859 | ], 860 | "text": "We provide the case study of explainable linkprediction with commonsense as shown in Figure 4. Given a query with the tail entity missing (rockets, teamplaysinleague, ?) on NELL-", 861 | "type": "Text", 862 | "index": 9, 863 | "score": 0.91845703125, 864 | "contain_formula": false, 865 | "font_size": 10.976943969726562, 866 | "font_name": "NimbusRomNo9L-Regu" 867 | } 868 | ] 869 | }, 870 | { 871 | "page": 8, 872 | "width": 621, 873 | "height": 877, 874 | "boxes": [ 875 | { 876 | "bbox": [ 877 | 313, 878 | 69, 879 | 557, 880 | 811 881 | ], 882 | "text": "L. Ji, Y. Wang, B. Shi, D. Zhang, Z. Wang, and J. Yan.2019. Microsoft concept graph: Mining semanticconcepts for short text understanding., 1:262–294. Data IntelligenceNi Lao, Tom Mitchell, and William W. Cohen. 2011.Random walk inference and learning in a large scaleknowledge base. In Proceedings of the 2011 Conference on Empirical Methods in Natural LanguageProcessing, pages 529–539.Jens Lehmann, Robert Isele, Max Jakob, Anja Jentzsch,Dimitris Kontokostas, Pablo N. Mendes, SebastianHellmann, Mohamed Morsey, Patrick van Kleef,Soren Auer, and Christian Bizer. 2015. Dbpedia-alarge-scale, multilingual knowledge base extractedfrom wikipedia. Semantic Web, 6(2):167–195.Zelong Li, Jianchao Ji, Zuohui Fu, Yingqiang Ge,Shuyuan Xu, Chong Chen, and Yongfeng Zhang.In2021. Efficient non-sampling knowledge graph embedding. Proceedings of the Web Conference2021, page 1727–1736.Xi Victoria Lin, Richard Socher, and Caiming Xiong.2018. Multi-hop knowledge graph reasoning withreward shaping. In Proceedings of the 2018 Conference on Empirical Methods in Natural LanguageProcessing, pages 3243–3253.Weiyu Liu, Angel Daruna, Zsolt Kira, and Sonia Chernova. 2020. Path ranking with attention to type hierarchies. In Proceedings of the Thirty-Fourth AAAIConference on Artificial Intelligence, pages 2893–2900.ChristianMeilicke,MelisachewWudageChekol,Daniel Ruffinelli, and Heiner Stuckenschmidt. 2019.Anytime bottom-up rule learning for knowledgegraph completion.In Proceedings of the TwentyEighth International Joint Conference on ArtificialIntelligence, pages 3137–3143.T. Mitchell, W. Cohen, E. Hruschka, and et al. 2018.Never-ending learning.Communications of theACM, 61(5):103–115.Maximilian Nickel, Volker Tresp, and Hans-PeterKriegel. 2011.A three-way model for collectivelearning on multi-relational data.In Proceedingsof the 28th International Conference on MachineLearning, page 809–816.Nazneen Fatema Rajani, Bryan McCann, CaimingXiong, and Richard Socher. 2019.Explain yourself! leveraging language models for commonsensereasoning. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics,pages 4932–4942.Ali Sadeghian, Mohammadreza Armandpour, PatrickDing, and Daisy Zhe Wang. 2019. Drum: End-toend differentiable rule mining on knowledge graphs.In Proceedings of the Thirty-third Conference onNeural Information Processing Systems.", 883 | "type": "Reference", 884 | "index": 0, 885 | "score": 0.962890625, 886 | "contain_formula": false, 887 | "font_size": 9.962599754333496, 888 | "font_name": "NimbusRomNo9L-ReguItal" 889 | }, 890 | { 891 | "bbox": [ 892 | 69, 893 | 251, 894 | 140, 895 | 270 896 | ], 897 | "text": "References", 898 | "type": "Title", 899 | "index": 2, 900 | "score": 0.7919921875, 901 | "contain_formula": false, 902 | "font_size": 11.9552001953125, 903 | "font_name": "NimbusRomNo9L-Medi" 904 | }, 905 | { 906 | "bbox": [ 907 | 67, 908 | 272, 909 | 307, 910 | 810 911 | ], 912 | "text": "Kurt Bollacker, Georg Gottlob, and Sergio Flesca.2008.Freebase: a collaboratively created graphdatabase for structuring human knowledge. In Proceedings of the 2008 ACM SIGMOD InternationalConference on Management of Data, pages 1247–1250.Antoine Bordes, Nicolas Usunier, Alberto GarciaDuran, Jason Weston, and Oksana Yakhnenko.2013. Translating embeddings for modeling multirelational data.In Processing of the 27th Annual, pages 2787–2795.Conference on Neural Information Processing SystemsLiwei Cai and William Yang Wang. 2018. KBGAN:Adversarial learning for knowledge graph embeddings. In Proceedings of the 2018 Conference of theNorth American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers), pages 1470–1480.KrompaßDenis, Stephan Baier, and Volker Tresp. 2015.Type-constrained representation learning in knowledge graphs.In Proceedings of the 14th International Conference on The Semantic Web - ISWC2015 - Volume 9366, page 640–655.Xin Dong, Evgeniy Gabrilovich, Geremy Heitz, WilkoHorn, Ni Lao, Kevin Murphy, Thomas Strohmann,Shaohua Sun, and Wei Zhang. 2014.Knowledgevault: A web-scale approach to probabilistic knowledge fusion.In Proceedings of the 20th ACMSIGKDD International Conference on KnowledgeDiscovery and Data Mining, page 601–610.Luis Galárraga, Christina Teflioudi, Katja Hose, andFabian Suchanek. 2015. Fast rule mining in ontological knowledge bases with amie+. The VLDB Journal, 24(6):707–730.Junheng Hao, Muhao Chen, Wenchao Yu, Yizhou Sun,and Wei Wang. 2019.Universal representationlearning of knowledge bases by jointly embeddinginstances and ontological concepts. In Proceedingsof the 25th ACM SIGKDD International Conferenceon Knowledge Discovery and Data Mining, pages1709–1719.", 913 | "type": "Reference", 914 | "index": 3, 915 | "score": 0.97314453125, 916 | "contain_formula": false, 917 | "font_size": 9.962599754333496, 918 | "font_name": "NimbusRomNo9L-ReguItal" 919 | }, 920 | { 921 | "bbox": [ 922 | 69, 923 | 71, 924 | 307, 925 | 236 926 | ], 927 | "text": "On the other hand, we design a multi-view linkprediction technique in a coarse-to-fine paradigmto filter the candidate entities in the view of commonsense and output the predicted results fromthe perspective of fact. The experiments on fourdatasets demonstrate the effectiveness and the scalability of our proposed framework and each modulecompared with the state-of-the-art baselines. Furthermore, our framework could explain link prediction results and potentially assemble new KGEmodels to improve their performance.", 928 | "type": "Text", 929 | "index": 1, 930 | "score": 0.95458984375, 931 | "contain_formula": false, 932 | "font_size": 10.946897506713867, 933 | "font_name": "NimbusRomNo9L-Regu" 934 | } 935 | ] 936 | }, 937 | { 938 | "page": 9, 939 | "width": 621, 940 | "height": 877, 941 | "boxes": [ 942 | { 943 | "bbox": [ 944 | 313, 945 | 68, 946 | 559, 947 | 804 948 | ], 949 | "text": "Zhen Wang, Jianwen Zhang, Jianlin Feng, and ZhengChen. 2014.Knowledge graph embedding bytranslating on hyperplanes. In Proceedings of theAAAI Conference on Artificial Intelligence, page1112–1119.Ruobing Xie, Zhiyuan Liu, Jia Jia, Huanbo Luan, andMaosong Sun. 2016a.Representation learning ofknowledge graphs with entity descriptions. In Proceedings of the Thirtieth AAAI Conference on Artificial Intelligence, page 2659–2665.Ruobing Xie,Zhiyuan Liu,Huanbo Luan,andMaosong Sun. 2017. Image-embodied knowledgerepresentation learning.In Proceedings of theTwenty-Sixth International Joint Conference on Artificial Intelligence, pages 3140–3146.Ruobing Xie, Zhiyuan Liu, and Maosong Sun. 2016b.Representation learning of knowledge graphs withhierarchical types.In Proceedings of the TwentyFifth International Joint Conference on Artificial Intelligence, page 2965–2971.Wenhan Xiong, Thien Hoang, , and William YangWang. 2017. DeepPath: A reinforcement learningmethod for knowledge graph reasoning. In Proceedings of the 2017 Conference on Empirical Methodsin Natural Language Processing, pages 564–573.Bishan Yang, Wen tau Yih, Xiaodong He, JianfengGao, and Li Deng. 2015. Embedding entities andrelations for learning and inference in knowledgebases. In Proceedings of International Conferenceon Learning Representations.Shiquan Yang, Rui Zhang, and Sarah Erfani. 2020.GraphDialog: Integrating graph knowledge into endto-end task-oriented dialogue systems. In Proceedings of the 2020 Conference on Empirical Methodsin Natural Language Processing (EMNLP), pages1878–1888.Shuai Zhang, Yi Tay, Lina Yao, and Qi Liu. 2019a.Quaternion knowledge graph embeddings. In Proceedings of the 33rd Conference on Neural Information Processing Systems, pages 2731–2741.Yongqi Zhang, Quanming Yao, Yingxia Shao, and LeiChen. 2019b. Nscaching: Simple and efficient negative sampling for knowledge graph embedding. In2019 IEEE 35th International Conference on DataEngineering, pages 614–625.Zhanqiu Zhang, Jianyu Cai, Yongdong Zhang, and JieWang. 2020. Learning hierarchy-aware knowledgegraph embeddings for link prediction. In Proceedings of the AAAI Conference on Artificial Intelligence, pages 3065–3072.Hao Zhou, Tom Young, Minlie Huang, Haizhou Zhao,and Xiaoyan Zhu. 2018. Commonsense knowledgeaware conversation generation with graph attention.In Proceedings of the Twenty-Seventh InternationalJoint Conference on Artificial Intelligence, pages4623–4629.", 950 | "type": "Reference", 951 | "index": 0, 952 | "score": 0.95361328125, 953 | "contain_formula": false, 954 | "font_size": 9.962599754333496, 955 | "font_name": "NimbusRomNo9L-ReguItal" 956 | }, 957 | { 958 | "bbox": [ 959 | 67, 960 | 68, 961 | 308, 962 | 811 963 | ], 964 | "text": "Maarten Sap, Ronan Le Bras, Emily Allaway, Chandra Bhagavatula, Nicholas Lourie, Hannah Rashkin,Brendan Roof, Noah A. Smith, and Yejin Choi. 2019.ATOMIC: an atlas of machine commonsense for ifthen reasoning. In Proceedings of the Thirty-ThirdAAAI Conference on Artificial Intelligence, pages3027–3035.Apoorv Saxena, Aditay Tripathi, and Partha Talukdar. 2020. Improving multi-hop question answeringover knowledge graphs using knowledge base embeddings. In Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics,pages 4498–4507.Robyn Speer, Joshua Chin, and Catherine Havasi. 2017.ConceptNet 5.5: An open multilingual graph of general knowledge. In Proceedings of the Thirty-FirstAAAI Conference on Artificial Intelligence, page4444–4451.Yawei Sun,Lingling Zhang,Gong Cheng,andYuzhong Qu. 2020.SPARQA: skeleton-based semantic parsing for complex questions over knowledge bases.In Proceedings of the Thirty-FourthAAAI Conference on Artificial Intelligence, pages8952–8959.Zhiqing Sun, Zhi-Hong Deng, Jian-Yun Nie, and JianTang. 2019. RotatE: Knowledge graph embeddingby relational rotation in complex space. In Proceedings of International Conference on Learning Representations.Kristina Toutanova and Danqi Chen. 2015. Observedversus latent features for knowledge base and textinference. In Proceedings of the 3rd Workshop onContinuous Vector Space Models and their Compositionality, pages 57–66.Théo Trouillon, Johannes Welbl, Sebastian Riedel, ÉricGaussier, and Guillaume Bouchard. 2016. Complexembeddings for simple link prediction. In Proceedings of the 33rd International Conference on Machine Learning, page 2071–2080.Hongwei Wang, Fuzheng Zhang, Mengdi Zhang, JureLeskovec, Miao Zhao, Wenjie Li, and ZhongyuanWang. 2019a. Knowledge-aware graph neural networks with label smoothness regularization for recommender systems.In Proceedings of the 25thACM SIGKDD International Conference on Knowledge Discovery and Data Mining, page 968–977.XiangWang,TinglinHuang,DingxianWang,Yancheng Yuan, Zhenguang Liu, Xiangnan He,and Tat-Seng Chua. 2021. Learning intents behindinteractions with knowledge graph for recommendation. In Proceedings of the Web Conference, pages878–887.Y. Wang, Y. Liu, H. Zhang, and H. Xie. 2019b.Leveraging lexical semantic information for learning concept-based multiple embedding representations for knowledge graph completion. In APWebWAIM, pages 382–397.", 965 | "type": "Reference", 966 | "index": 1, 967 | "score": 0.95654296875, 968 | "contain_formula": false, 969 | "font_size": 9.962599754333496, 970 | "font_name": "NimbusRomNo9L-ReguItal" 971 | } 972 | ] 973 | } 974 | ], 975 | "msg": "ok" 976 | } --------------------------------------------------------------------------------