├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── EvalNE-logo.jpg │ ├── acknowledgements.rst │ ├── api.rst │ ├── conf.py │ ├── contributing.rst │ ├── description.rst │ ├── diagram.png │ ├── erc-logo.jpg │ ├── evalne.evaluation.rst │ ├── evalne.methods.rst │ ├── evalne.rst │ ├── evalne.tests.rst │ ├── evalne.utils.rst │ ├── fwo-logo.jpg │ ├── help.rst │ ├── index.rst │ ├── installation.rst │ ├── license.rst │ ├── modules.rst │ ├── nstatic │ └── placeholder │ ├── ntemplates │ └── placeholder │ ├── quickstart.rst │ └── release.rst ├── evalne ├── __init__.py ├── __main__.py ├── evaluation │ ├── __init__.py │ ├── edge_embeddings.py │ ├── evaluator.py │ ├── pipeline.py │ ├── score.py │ └── split.py ├── methods │ ├── __init__.py │ ├── katz.py │ └── similarity.py ├── tests │ ├── __init__.py │ ├── data │ │ ├── network.edgelist │ │ ├── sig_network.edgelist │ │ ├── test.csv │ │ └── test.pkl │ ├── test_baselines.py │ ├── test_edgeembeds.py │ ├── test_nr_sampling.py │ ├── test_preprocess.py │ ├── test_spantree.py │ └── test_visualize.py └── utils │ ├── __init__.py │ ├── preprocess.py │ ├── split_train_test.py │ ├── util.py │ └── viz_utils.py ├── examples ├── api_examples │ ├── evaluator_example.py │ ├── functions_example.py │ ├── output │ │ ├── CN_test_test_PR.pdf │ │ ├── CN_test_test_ROC.pdf │ │ ├── CN_train_train_PR.pdf │ │ ├── CN_train_train_ROC.pdf │ │ ├── eval.pkl │ │ ├── eval_output.txt │ │ ├── lp_train_test_splits │ │ │ └── network_prep_51 │ │ │ │ ├── negTeE_0.csv │ │ │ │ ├── negTeE_1.csv │ │ │ │ ├── negTeE_2.csv │ │ │ │ ├── negTeE_3.csv │ │ │ │ ├── negTeE_4.csv │ │ │ │ ├── negTrE_0.csv │ │ │ │ ├── negTrE_1.csv │ │ │ │ ├── negTrE_2.csv │ │ │ │ ├── negTrE_3.csv │ │ │ │ ├── negTrE_4.csv │ │ │ │ ├── teE_0.csv │ │ │ │ ├── teE_1.csv │ │ │ │ ├── teE_2.csv │ │ │ │ ├── teE_3.csv │ │ │ │ ├── teE_4.csv │ │ │ │ ├── trE_0.csv │ │ │ │ ├── trE_1.csv │ │ │ │ ├── trE_2.csv │ │ │ │ ├── trE_3.csv │ │ │ │ └── trE_4.csv │ │ ├── network_prep.edgelist │ │ ├── prep_graph.edgelist │ │ └── stats.txt │ └── simple-example.py ├── conf.ini └── replicated_setups │ ├── cne │ └── conf_cne.ini │ ├── maxent │ ├── conf_maxent.ini │ ├── conf_maxent2.ini │ ├── conf_maxentNR.ini │ └── conf_maxentbig.ini │ ├── node2vec │ └── conf_node2vec.ini │ ├── others │ ├── conf_cnev2.ini │ ├── conf_opennetest.ini │ └── conf_parTest.ini │ ├── prune │ ├── conf_prune.ini │ └── prep_data_prune.py │ └── sdne │ └── conf_sdne.ini ├── requirements.txt ├── scripts └── set_numpy_threads.sh └── setup.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. Mention if EvalNE was being used as API or CLI. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the error: 15 | 1. OS used: [e.g. Linux] 16 | 2. EvalNE Version [e.g. 0.3.0] 17 | 3. Snippet of code executed (for API) or conf file run (for CLI) 18 | 4. Full error output 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. Linux] 28 | - EvalNE Version [e.g. 0.3.0] 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex1: I would like to be able to evaluate [...]. Ex2: I don't like [some design choice] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea/ 3 | **/__pycache__/ 4 | *.egg-info/ 5 | /build 6 | /dist 7 | /venv 8 | /venv2 9 | /methods 10 | /data 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting alexandru.mara@ugent.be. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How to contribute to EvalNE 2 | 3 | #### **Did you find a bug?** 4 | 5 | * **Ensure the bug was not already reported** by searching the GitHub repo under [Issues](https://github.com/Dru-Mara/EvalNE/issues). 6 | 7 | * If there is no open issue addressing the problem, [open a new one](https://github.com/Dru-Mara/EvalNE/issues/new/). 8 | Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. 9 | 10 | * If possible, use our bug report [**template**](https://github.com/Dru-Mara/EvalNE/issues/new/choose) to create the issue. 11 | 12 | #### **Did you write a patch that fixes a bug?** 13 | 14 | * Open a new GitHub pull request with the patch. 15 | 16 | * Ensure the description clearly summarizes the problem and solution. Include the relevant issue number if applicable. 17 | 18 | #### **Do you intend to add a new feature or change an existing one?** 19 | 20 | * **Ensure that there is no other request for said feature** by searching the GitHub repo under [Issues](https://github.com/Dru-Mara/EvalNE/issues). 21 | 22 | * If there is no open issue addressing the problem, [open a new one](https://github.com/Dru-Mara/EvalNE/issues/new/). 23 | Be sure to include a **title and clear description** of the requested feature. Also motivate why the feature is needed and that it fits the scope of EvalNE. 24 | 25 | * If possible, use our feature report [**template**](https://github.com/Dru-Mara/EvalNE/issues/new/choose) to create the issue. 26 | 27 | #### **Do you have questions about the source code?** 28 | 29 | * Take a look at our more detailed documentation on [Read The Docs](https://evalne.readthedocs.io/en/latest/index.html) 30 | 31 | * Ask any question about EvalNE to our [contact email](https://evalne.readthedocs.io/en/latest/help.html) 32 | 33 | #### **Do you want to contribute to the documentation?** 34 | 35 | * Improvements to the Docstring? to the Read The Docs page? to Code format? 36 | 37 | * Write us to our [contact email](https://evalne.readthedocs.io/en/latest/help.html) describing the changes you propose. 38 | 39 | Thanks! 40 | 41 | EvalNE Team 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2018 Ghent University, Alexandru Mara 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = EvalNE 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=EvalNE 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/source/EvalNE-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/EvalNE-logo.jpg -------------------------------------------------------------------------------- /docs/source/acknowledgements.rst: -------------------------------------------------------------------------------- 1 | Acknowledgements 2 | ================ 3 | 4 | The research leading to these results has received funding from the European Research Council under the European Union's Seventh Framework Programme (FP7/2007-2013) / ERC Grant Agreement no. 615517, from the FWO (project no. G091017N, G0F9816N), and from the European Union's Horizon 2020 research and innovation programme and the FWO under the Marie Sklodowska-Curie Grant Agreement no. 665501. 5 | 6 | .. image:: erc-logo.jpg 7 | :width: 420px 8 | :alt: ERC logo 9 | :align: center 10 | :target: https://erc.europa.eu/ 11 | 12 | .. image:: fwo-logo.jpg 13 | :width: 320px 14 | :alt: FWO logo 15 | :align: center 16 | :target: https://www.fwo.be/en/ 17 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | evalne 8 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | sys.path.insert(0, os.path.abspath('.')) 18 | sys.path.insert(0, os.path.abspath('../')) 19 | 20 | import evalne 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = u'EvalNE' 25 | copyright = u'2019, Alexandru Mara' 26 | author = u'Alexandru Mara' 27 | 28 | # The short X.Y version 29 | version = u'0.3' 30 | # The full version, including alpha/beta/rc tags 31 | release = u'0.3.4' 32 | 33 | 34 | # -- General configuration --------------------------------------------------- 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | # 38 | # needs_sphinx = '1.0' 39 | 40 | # Add any Sphinx extension module names here, as strings. They can be 41 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 42 | # ones. 43 | extensions = [ 44 | 'sphinx.ext.autodoc', 45 | 'sphinx.ext.napoleon', 46 | 'sphinx.ext.doctest', 47 | 'sphinx.ext.todo', 48 | 'sphinx.ext.coverage', 49 | 'sphinx.ext.imgmath', 50 | 'sphinx.ext.viewcode', 51 | 'sphinx.ext.githubpages', 52 | ] 53 | 54 | # Napoleon settings 55 | napoleon_numpy_docstring = True 56 | napoleon_google_docstring = False 57 | napoleon_use_param = True 58 | napoleon_use_ivar = True 59 | 60 | # Add any paths that contain templates here, relative to this directory. 61 | templates_path = ['ntemplates'] 62 | 63 | # The suffix(es) of source filenames. 64 | # You can specify multiple suffix as a list of string: 65 | # 66 | # source_suffix = ['.rst', '.md'] 67 | source_suffix = '.rst' 68 | 69 | # The master toctree document. 70 | master_doc = 'index' 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = None 78 | 79 | # List of patterns, relative to source directory, that match files and 80 | # directories to ignore when looking for source files. 81 | # This pattern also affects html_static_path and html_extra_path . 82 | exclude_patterns = [] 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | 88 | # -- Options for HTML output ------------------------------------------------- 89 | 90 | # The theme to use for HTML and HTML Help pages. See the documentation for 91 | # a list of builtin themes. 92 | # 93 | html_theme = "sphinx_rtd_theme" 94 | 95 | # Theme options are theme-specific and customize the look and feel of a theme 96 | # further. For a list of options available for each theme, see the 97 | # documentation. 98 | # 99 | # html_theme_options = {} 100 | 101 | # Add any paths that contain custom static files (such as style sheets) here, 102 | # relative to this directory. They are copied after the builtin static files, 103 | # so a file named "default.css" will overwrite the builtin "default.css". 104 | html_static_path = ['nstatic'] 105 | 106 | # Custom sidebar templates, must be a dictionary that maps document names 107 | # to template names. 108 | # 109 | # The default sidebars (for documents that don't match any pattern) are 110 | # defined by theme itself. Builtin themes are using these templates by 111 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 112 | # 'searchbox.html']``. 113 | # 114 | # html_sidebars = {} 115 | 116 | 117 | # -- Options for HTMLHelp output --------------------------------------------- 118 | 119 | # Output file base name for HTML help builder. 120 | htmlhelp_basename = 'EvalNEdoc' 121 | 122 | 123 | # -- Options for LaTeX output ------------------------------------------------ 124 | 125 | latex_elements = { 126 | # The paper size ('letterpaper' or 'a4paper'). 127 | # 128 | # 'papersize': 'letterpaper', 129 | 130 | # The font size ('10pt', '11pt' or '12pt'). 131 | # 132 | # 'pointsize': '10pt', 133 | 134 | # Additional stuff for the LaTeX preamble. 135 | # 136 | # 'preamble': '', 137 | 138 | # Latex figure (float) alignment 139 | # 140 | # 'figure_align': 'htbp', 141 | } 142 | 143 | # Grouping the document tree into LaTeX files. List of tuples 144 | # (source start file, target name, title, 145 | # author, documentclass [howto, manual, or own class]). 146 | latex_documents = [ 147 | (master_doc, 'EvalNE.tex', u'EvalNE Documentation', 148 | u'Alexandru Mara', 'manual'), 149 | ] 150 | 151 | 152 | # -- Options for manual page output ------------------------------------------ 153 | 154 | # One entry per manual page. List of tuples 155 | # (source start file, name, description, authors, manual section). 156 | man_pages = [ 157 | (master_doc, 'evalne', u'EvalNE Documentation', 158 | [author], 1) 159 | ] 160 | 161 | 162 | # -- Options for Texinfo output ---------------------------------------------- 163 | 164 | # Grouping the document tree into Texinfo files. List of tuples 165 | # (source start file, target name, title, author, 166 | # dir menu entry, description, category) 167 | texinfo_documents = [ 168 | (master_doc, 'EvalNE', u'EvalNE Documentation', 169 | author, 'EvalNE', 'A framework for Evaluating Network Embeddings Methods.', 170 | 'Miscellaneous'), 171 | ] 172 | 173 | 174 | # -- Extension configuration ------------------------------------------------- 175 | 176 | # -- Options for todo extension ---------------------------------------------- 177 | 178 | # If true, `todo` and `todoList` produce output, else they produce nothing. 179 | todo_include_todos = True 180 | -------------------------------------------------------------------------------- /docs/source/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Contributions in order to improve or extend the capabilities of EvalNE are highly appreciated. There are different ways in which interested users can contribute to the library, these include: 5 | 6 | - Raising issues_ on GitHub 7 | 8 | + Reporting bugs 9 | + Requesting new features 10 | + Questions 11 | 12 | - Pull requests_ for new features on GitHub 13 | - Improving the documentation 14 | 15 | .. _issues: https://github.com/Dru-Mara/EvalNE/issues 16 | .. _requests: https://github.com/Dru-Mara/EvalNE/pulls 17 | 18 | 19 | -------------------------------------------------------------------------------- /docs/source/description.rst: -------------------------------------------------------------------------------- 1 | Features 2 | ======== 3 | 4 | EvalNE has been designed as a pipeline of interconnected and interchangeable building blocks. This structure provides the flexibility to create different evaluation pipelines and, thus, to evaluate methods from node embeddings, node-pair embeddings or similarity scores. The main building blocks that constitute EvalNE as well as the types of tasks and methods it can evaluate are presented in the following diagram. Blocks represented with solid lines correspond to modules provided by the library and those with dashed lines are the user-specified methods to be evaluated. 5 | 6 | .. image:: diagram.png 7 | :width: 600px 8 | :alt: EvalNE diagram 9 | :align: center 10 | 11 | .. note:: 12 | 13 | For node classification (NC) tasks currently only nede embedding methods are supported. 14 | 15 | .. note:: 16 | 17 | The hyper-parameter tuning and evaluation setup functionalities are omitted in this diagram. 18 | 19 | A more detailed description of the library features for the practitioner and for the methodologist are presented below. Further information can be found in our paper_. 20 | 21 | .. _paper: https://www.sciencedirect.com/science/article/pii/S2352711022000139 22 | 23 | For Methodologists 24 | ------------------ 25 | 26 | A command line interface in combination with a configuration file (describing datasets, 27 | methods and evaluation setup) allows the user 28 | to evaluate any embedding method and compare 29 | it to the state of the art or replicate the experimental setup of existing papers without 30 | the need to write additional code. EvalNE does not provide implementations of any NE methods 31 | but offers the necessary environment to evaluate any off-the-shelf algorithm. 32 | Implementations of NE methods can be obtained from libraries 33 | such as OpenNE_ or GEM_ as well as directly from the web pages of the authors e.g. 34 | Deepwalk_, Node2vec_, LINE_, PRUNE_, Metapath2vec_, CNE_. 35 | 36 | .. _OpenNE: https://github.com/thunlp/OpenNE 37 | .. _GEM: https://github.com/palash1992/GEM 38 | .. _Deepwalk: https://github.com/phanein/deepwalk 39 | .. _Node2vec: https://github.com/aditya-grover/node2vec 40 | .. _LINE: https://github.com/tangjianpku/LINE 41 | .. _PRUNE: https://github.com/ntumslab/PRUNE 42 | .. _Metapath2vec: https://ericdongyx.github.io/metapath2vec/m2v.html 43 | .. _CNE: https://bitbucket.org/ghentdatascience/cne/ 44 | 45 | EvalNE also includes the following LP heuristics for both directed and 46 | undirected networks (in and out node neighbourhoods), which can be used as 47 | baselines: 48 | 49 | - Random Prediction 50 | - Common Neighbours 51 | - Jaccard Coefficient 52 | - Adamic Adar Index 53 | - Preferential Attachment 54 | - Resource Allocation Index 55 | - Cosine Similarity 56 | - Leicht-Holme-Newman index 57 | - Topological Overlap 58 | - Katz similarity 59 | - All baselines (a combination of the first 5 heuristics in a 5-dim embedding) 60 | 61 | For Practitioners 62 | ----------------- 63 | 64 | When used as an API, EvalNE provides functions to: 65 | 66 | - Load and preprocess graphs 67 | - Obtain general graph statistics 68 | - Conveniently read node/edge embeddings from files 69 | - Sample nodes/edges to form train/test/validation sets 70 | - Different approaches for edge sampling: 71 | 72 | - Timestamp based sampling: latest nodes are used for testing 73 | - Random sampling: random split of edges in train and test sets 74 | - Spanning tree sampling: train set will contain a spanning tree of the graph 75 | - Fast depth first search sampling: similar to spanning tree but based of DFS 76 | 77 | - Negative sampling or generation of non-edge pairs using: 78 | 79 | - Open world assumption: train non-edges do not overlap with train edges 80 | - Closed world assumption: train non-edges do not overlap with either train nor test edges 81 | 82 | - Evaluate LP, SP and NR for methods that output: 83 | 84 | - Node Embeddings 85 | - Node-pair Embeddings 86 | - Similarity scores (e.g. the ones given by LP heuristics) 87 | 88 | - Implements simple visualization routines for embeddings and graphs 89 | - Includes NC evaluation for node embedding methods 90 | - Provides binary operators to compute edge embeddings from node feature vectors: 91 | 92 | - Average 93 | - Hadamard 94 | - Weighted L1 95 | - Weighted L2 96 | 97 | - Can use any scikit-learn classifier for LP/SP/NR/NC tasks 98 | - Provides routines to run command line commands or functions with a given timeout 99 | - Includes hyperparameter tuning based on grid search 100 | - Implements over 10 different evaluation metrics such as AUC, F-score, etc. 101 | - AUC and PR curves can be provided as output 102 | - Includes routines to generate tabular outputs and directly parse them to Latex tables 103 | 104 | -------------------------------------------------------------------------------- /docs/source/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/diagram.png -------------------------------------------------------------------------------- /docs/source/erc-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/erc-logo.jpg -------------------------------------------------------------------------------- /docs/source/evalne.evaluation.rst: -------------------------------------------------------------------------------- 1 | evalne.evaluation package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | evalne.evaluation.edge\_embeddings module 8 | ----------------------------------------- 9 | 10 | .. automodule:: evalne.evaluation.edge_embeddings 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | evalne.evaluation.evaluator module 16 | ---------------------------------- 17 | 18 | .. automodule:: evalne.evaluation.evaluator 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | evalne.evaluation.pipeline module 24 | --------------------------------- 25 | 26 | .. automodule:: evalne.evaluation.pipeline 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | evalne.evaluation.score module 32 | ------------------------------ 33 | 34 | .. automodule:: evalne.evaluation.score 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | evalne.evaluation.split module 40 | ------------------------------ 41 | 42 | .. automodule:: evalne.evaluation.split 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: evalne.evaluation 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/source/evalne.methods.rst: -------------------------------------------------------------------------------- 1 | evalne.methods package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | evalne.methods.katz module 8 | -------------------------- 9 | 10 | .. automodule:: evalne.methods.katz 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | evalne.methods.similarity module 16 | -------------------------------- 17 | 18 | .. automodule:: evalne.methods.similarity 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: evalne.methods 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/source/evalne.rst: -------------------------------------------------------------------------------- 1 | evalne package 2 | ============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | evalne.evaluation 10 | evalne.methods 11 | evalne.utils 12 | 13 | Module contents 14 | --------------- 15 | 16 | .. automodule:: evalne 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/source/evalne.tests.rst: -------------------------------------------------------------------------------- 1 | evalne.tests package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | evalne.tests.test\_baselines module 8 | ----------------------------------- 9 | 10 | .. automodule:: evalne.tests.test_baselines 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | evalne.tests.test\_edgeembeds module 16 | ------------------------------------ 17 | 18 | .. automodule:: evalne.tests.test_edgeembeds 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | evalne.tests.test\_preprocess module 24 | ------------------------------------ 25 | 26 | .. automodule:: evalne.tests.test_preprocess 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | evalne.tests.test\_spantree module 32 | ---------------------------------- 33 | 34 | .. automodule:: evalne.tests.test_spantree 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | evalne.tests.test\_visualize module 40 | ----------------------------------- 41 | 42 | .. automodule:: evalne.tests.test_visualize 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: evalne.tests 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /docs/source/evalne.utils.rst: -------------------------------------------------------------------------------- 1 | evalne.utils package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | evalne.utils.preprocess module 8 | ------------------------------ 9 | 10 | .. automodule:: evalne.utils.preprocess 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | evalne.utils.split\_train\_test module 16 | -------------------------------------- 17 | 18 | .. automodule:: evalne.utils.split_train_test 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | evalne.utils.util module 24 | ------------------------ 25 | 26 | .. automodule:: evalne.utils.util 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | evalne.utils.viz\_utils module 32 | ------------------------------ 33 | 34 | .. automodule:: evalne.utils.viz_utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: evalne.utils 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/source/fwo-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/fwo-logo.jpg -------------------------------------------------------------------------------- /docs/source/help.rst: -------------------------------------------------------------------------------- 1 | Help 2 | ==== 3 | 4 | For help with setting up or using EvalNE plase contact: 5 | alexandru(dot)mara(at)ugent(dot)be 6 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | EvalNE - A Framework for Evaluating Network Embeddings 2 | ========================================================================= 3 | 4 | .. image:: EvalNE-logo.jpg 5 | :width: 220px 6 | :alt: EvalNE logo 7 | :align: center 8 | 9 | EvalNE is an open source Python library designed for assessing and comparing the performance of Network Embedding (NE) methods on Link Prediction (LP), Sign Prediction (SP), Network Reconstruction (NR), 10 | Node Classification (NC) and vizualization downstream tasks. The library intends to simplify these complex and time consuming evaluation processes by providing automation and abstraction of tasks such as model hyper-parameter tuning and model validation, node and edge sampling, node-pair 11 | embedding computation, results reporting and data visualization, among many others. 12 | EvalNE can be used both as a command line tool and as an API and is compatible with Python 3. 13 | In its current version, EvalNE can evaluate weighted directed and undirected simple networks. 14 | 15 | EvalNE is provided under the MIT_ free software licence and is maintained by Alexandru Mara (alexandru(dot)mara(at)ugent(dot)be). The source code can be found on GitHub_. 16 | 17 | .. _MIT: https://opensource.org/licenses/MIT 18 | .. _GitHub: https://github.com/Dru-Mara/EvalNE 19 | 20 | 21 | See :doc:`the quickstart ` to get started. 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | :caption: Getting started 26 | 27 | description 28 | installation 29 | quickstart 30 | 31 | .. toctree:: 32 | :maxdepth: 2 33 | :caption: Developer 34 | 35 | api 36 | release 37 | contributing 38 | 39 | .. toctree:: 40 | :maxdepth: 1 41 | :caption: More 42 | 43 | license 44 | acknowledgements 45 | help 46 | 47 | 48 | Citation 49 | -------- 50 | 51 | If you have found EvaNE useful in your research, please cite our arXiv paper_ : 52 | 53 | .. _paper: https://arxiv.org/abs/1901.09691 54 | 55 | .. code-block:: console 56 | 57 | @article{MARA2022evalne, 58 | title = {EvalNE: A Framework for Network Embedding Evaluation}, 59 | author = {Alexandru Mara and Jefrey Lijffijt and Tijl {De Bie}}, 60 | journal = {SoftwareX}, 61 | volume = {17}, 62 | pages = {}, 63 | year = {2022}, 64 | issn = {100997}, 65 | doi = {10.1016/j.softx.2022.100997}, 66 | url = {https://www.sciencedirect.com/science/article/pii/S2352711022000139} 67 | } 68 | 69 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | The library has been tested on Python 3.8. The supported platforms 5 | include Linux, Mac OS and Microsoft Windows. 6 | 7 | EvalNE depends on the following open-source packages: 8 | * Numpy 9 | * Scipy 10 | * Scikit-learn 11 | * Matplotlib 12 | * NetworkX 13 | * Pandas 14 | * tqdm 15 | * kiwisolver 16 | 17 | Linux/MacOS 18 | ----------- 19 | 20 | Before installing EvalNE make sure that **pip** and **python-tk** packages are installed 21 | on your system, this can be done by running: 22 | 23 | .. code-block:: console 24 | 25 | foo@bar:~$ sudo apt-get install python3-pip 26 | foo@bar:~$ sudo apt-get install python3-tk 27 | 28 | **Option 1:** Install the library using `pip` 29 | 30 | .. code-block:: console 31 | 32 | foo@bar:~$ pip install evalne 33 | 34 | **Option 2:** Cloning the code and installing 35 | 36 | - Clone the EvalNE repository: 37 | 38 | .. code-block:: console 39 | 40 | foo@bar:~$ git clone https://github.com/Dru-Mara/EvalNE.git 41 | foo@bar:~$ cd EvalNE 42 | 43 | - Install the library: 44 | 45 | .. code-block:: console 46 | 47 | # System-wide install 48 | foo@bar:~$ sudo python setup.py install 49 | 50 | # Alterntive single user install 51 | foo@bar:~$ python setup.py install --user 52 | 53 | - Alternatively, one can first download the required dependencies and then install: 54 | 55 | .. code-block:: console 56 | 57 | foo@bar:~$ pip install -r requirements.txt 58 | foo@bar:~$ sudo python setup.py install 59 | 60 | Check the installation by running `simple_example.py` or `functions_example.py` as shown below. 61 | If you have installed the package using pip, you will need to download the examples folder from 62 | the github repository first. 63 | 64 | .. code-block:: console 65 | 66 | foo@bar:~$ cd examples/ 67 | foo@bar:~$ python simple_example.py 68 | 69 | .. note:: 70 | 71 | In order to run the `evaluator_example.py` script, the OpenNE library, PRUNE and Metapath2Vec are required. Further instructions on where to obtain and how to install these methods/libraries are provided in :doc:`the quickstart ` section. 72 | 73 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ======= 3 | 4 | MIT License 5 | 6 | Copyright 2018 Ghent University, Alexandru Mara 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | evalne 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | evalne 8 | -------------------------------------------------------------------------------- /docs/source/nstatic/placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/nstatic/placeholder -------------------------------------------------------------------------------- /docs/source/ntemplates/placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/docs/source/ntemplates/placeholder -------------------------------------------------------------------------------- /docs/source/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | As a command line tool 5 | ---------------------- 6 | 7 | The library takes as input an *.ini* configuration file. This file allows the user 8 | to specify the evaluation settings, from the task to perform to the networks to use, data preprocessing, methods and baselines to evaluate, and types of output to provide. 9 | 10 | An example `conf.ini` file is provided describing the available options 11 | for each parameter. This file can be either modified to simulate different 12 | evaluation settings or used as a template to generate other *.ini* files. 13 | 14 | Additional configuration (*.ini*) files are provided replicating the experimental 15 | sections of different papers in the NE literature. These can be found in different 16 | folders under `examples/replicated_setups`. One such configuration file is 17 | `examples/replicated_setups/node2vec/conf_node2vec.ini`. This file simulates the link prediction 18 | experiments of the paper "Scalable Feature Learning for Networks" by A. Grover 19 | and J. Leskovec. 20 | 21 | Once the configuration is set, the evaluation can be run as indicated in the next 22 | subsection. 23 | 24 | **Running the conf examples** 25 | 26 | In order to run the evaluations using the provided `conf.ini` or any other *.ini* 27 | file, the following steps are necessary: 28 | 29 | 1. Download/Install the libraries/methods you want to test: 30 | 31 | * For running `conf.ini`: 32 | 33 | * OpenNE_ 34 | * PRUNE_ 35 | 36 | * For running other *.ini* files you may need: 37 | 38 | * Deepwalk_ 39 | * Node2vec_ 40 | * LINE_ 41 | * Metapath2Vec_ 42 | * CNE_ 43 | 44 | 2. Download the datasets used in the examples: 45 | 46 | * For `conf.ini`: 47 | 48 | * StudentDB_ 49 | * Facebook_ combined network 50 | * Arxiv GR-QC_ 51 | 52 | * For other *.ini* files you may need: 53 | 54 | * Facebook-wallpost_ 55 | * Arxiv Astro-Ph_ 56 | * ArXiv Hep-Ph_ (https://snap.stanford.edu/data/cit-HepPh.html) 57 | * BlogCatalog_ (http://socialcomputing.asu.edu/datasets/BlogCatalog3) 58 | * Wikipedia_ (http://snap.stanford.edu/node2vec) 59 | * PPI_ 60 | 61 | 3. Set the correct dataset paths in the INPATHS option of the corresponding *.ini* 62 | file. And the correct method paths under METHODS_OPNE and/or METHODS_OTHER options. 63 | 64 | 4. Run the evaluation: 65 | 66 | .. code-block:: console 67 | 68 | # For conf.ini run: 69 | foo@bar:~$ python -m evalne ./examples/conf.ini 70 | 71 | # For conf_node2vec.ini run: 72 | foo@bar:~$ python -m evalne ./examples/node2vec/conf_node2vec.ini 73 | 74 | .. note:: 75 | 76 | The networks provided as input to EvalNE are required to be in edgelist format. 77 | 78 | .. _OpenNE: https://github.com/thunlp/OpenNE 79 | .. _PRUNE: https://github.com/ntumslab/PRUNE 80 | .. _Deepwalk: https://github.com/phanein/deepwalk 81 | .. _Node2vec: https://github.com/aditya-grover/node2vec 82 | .. _LINE: https://github.com/tangjianpku/LINE 83 | .. _Metapath2Vec: https://www.dropbox.com/s/w3wmo2ru9kpk39n/code_metapath2vec.zip?dl=0 84 | .. _CNE: https://bitbucket.org/ghentdatascience/cne/ 85 | 86 | .. _StudentDB: http://adrem.ua.ac.be/smurfig 87 | .. _GR-QC: https://snap.stanford.edu/data/ca-GrQc.html 88 | .. _Facebook: https://snap.stanford.edu/data/egonets-Facebook.html 89 | .. _Facebook-wallpost: http://socialnetworks.mpi-sws.org/data-wosn2009.html 90 | .. _Astro-Ph: http://snap.stanford.edu/data/ca-AstroPh.html 91 | .. _Hep-Ph: https://snap.stanford.edu/data/cit-HepPh.html 92 | .. _BlogCatalog: http://socialcomputing.asu.edu/datasets/BlogCatalog3 93 | .. _Wikipedia: http://snap.stanford.edu/node2vec 94 | .. _PPI: http://snap.stanford.edu/node2vec/Homo_sapiens.mat 95 | 96 | As an API 97 | --------- 98 | 99 | The library can be imported and used like any other Python module. Next, we 100 | present a very basic LP example, for more complete ones we refer the user to the 101 | `examples` folder and the docstring documentation of the evaluator and the split submodules. 102 | 103 | :: 104 | 105 | from evalne.evaluation.evaluator import LPEvaluator 106 | from evalne.evaluation.split import LPEvalSplit 107 | from evalne.evaluation.score import Scoresheet 108 | from evalne.utils import preprocess as pp 109 | 110 | # Load and preprocess the network 111 | G = pp.load_graph('../evalne/tests/data/network.edgelist') 112 | G, _ = pp.prep_graph(G) 113 | 114 | # Create an evaluator and generate train/test edge split 115 | traintest_split = LPEvalSplit() 116 | traintest_split.compute_splits(G) 117 | nee = LPEvaluator(traintest_split) 118 | 119 | # Create a Scoresheet to store the results 120 | scoresheet = Scoresheet() 121 | 122 | # Set the baselines 123 | methods = ['random_prediction', 'common_neighbours', 'jaccard_coefficient'] 124 | 125 | # Evaluate baselines 126 | for method in methods: 127 | result = nee.evaluate_baseline(method=method) 128 | scoresheet.log_results(result) 129 | 130 | try: 131 | # Check if OpenNE is installed 132 | import openne 133 | 134 | # Set embedding methods from OpenNE 135 | methods = ['node2vec', 'deepwalk', 'GraRep'] 136 | commands = [ 137 | 'python -m openne --method node2vec --graph-format edgelist --p 1 --q 1', 138 | 'python -m openne --method deepWalk --graph-format edgelist --number-walks 40', 139 | 'python -m openne --method grarep --graph-format edgelist --epochs 10'] 140 | edge_emb = ['average', 'hadamard'] 141 | 142 | # Evaluate embedding methods 143 | for i in range(len(methods)): 144 | command = commands[i] + " --input {} --output {} --representation-size {}" 145 | results = nee.evaluate_cmd(method_name=methods[i], method_type='ne', command=command, 146 | edge_embedding_methods=edge_emb, input_delim=' ', output_delim=' ') 147 | scoresheet.log_results(results) 148 | 149 | except ImportError: 150 | print("The OpenNE library is not installed. Reporting results only for the baselines...") 151 | pass 152 | 153 | # Get output 154 | scoresheet.print_tabular() 155 | 156 | 157 | Output 158 | ------ 159 | 160 | The library stores all the output generated in a single folder per execution. The name 161 | of this folder is: `{task}_eval_{month}{day}_{hour}{min}`. Where `{task}` is one of: 162 | lp, sp, nr or nc. 163 | 164 | The library can provide two types of outputs, depending on the value of the SCORES option 165 | of the configuration file. If the keyword *all* is specified, the library will generate a 166 | file named `eval_output.txt` containing for each method and network analysed all the 167 | metrics available (auroc, precision, f-score, etc.). If more than one experiment repeat 168 | is requested the values reported will be the average over all the repeats. 169 | 170 | Setting the SCORES option to `%(maximize)` will generate a similar output file as before. 171 | The content of this file, however, will be a table (Alg. x Networks) containing exclusively 172 | the score specified in the MAXIMIZE option for each combination of method and network 173 | averaged over all experiment repeats. In addition a second table indicating the average 174 | execution time per method and dataset will be generated. 175 | 176 | If the option CURVES is set to a valid option then for each method dataset and experiment 177 | repeat a PR or ROC curve will be generated. If the option SAVE_PREP_NW is set to True, each 178 | evaluated network will be stored, in edgelist format, in a folder with the same name as the 179 | network. 180 | 181 | Finally, the library also generates an `eval.log` file and a `eval.pkl`. The first file 182 | contains important information regarding the evaluation process such as methods whose 183 | execution has failed, or validation scores. The second one encapsulates all the evaluation 184 | results as a pickle file. This file can be conveniently loaded and the results can be 185 | transformed into e.g. pandas dataframes or latex tables. 186 | 187 | Parallelization 188 | --------------- 189 | 190 | EvalNE makes extensive use of numpy for most operations. Numpy, in turn, 191 | uses other libraries such as OpenMP, MKL, etc., to provide parallelization. In order to allow for 192 | certain control on the maximum number of threads used during execution, we include a simple bash 193 | script (`set_numpy_threads.sh`). The script located inside the `scripts` folder can be given execution permissions and run as follows: 194 | 195 | .. code-block:: console 196 | 197 | # Give execution permissions: 198 | chmod +x set_numpy_threads.sh 199 | 200 | # Run the script: 201 | source set_numpy_threads.sh 202 | # The script will then ask for the maximum number of threads to use. 203 | 204 | 205 | -------------------------------------------------------------------------------- /docs/source/release.rst: -------------------------------------------------------------------------------- 1 | Release Log 2 | =========== 3 | 4 | EvalNE v0.4.0 5 | ------------- 6 | 7 | Release date: 04 Jul 2022 8 | 9 | Documentation 10 | ~~~~~~~~~~~~~ 11 | - Release log update. 12 | - Docstring improvements affecting some classes and functions. 13 | - Improved Readme.md and docs files. 14 | - Improved conf.ini inline documentation. 15 | 16 | Miscellaneous 17 | ~~~~~~~~~~~~ 18 | - Transition to python >3.7. 19 | - Support for python 2 has been dropped. 20 | - Dependencies bumped to latest package versions including numpy and networkx. 21 | - Improved imports throughout the library. 22 | - Extended parameter checks for config files. 23 | 24 | 25 | EvalNE v0.3.4 26 | ------------- 27 | 28 | Release date: 05 Dec 2022 29 | 30 | Documentation 31 | ~~~~~~~~~~~~~ 32 | - Release log update. 33 | - Docstring improvements affecting some classes and functions. 34 | - Improved Readme.md and docs files. 35 | 36 | New features 37 | ~~~~~~~~~~~~ 38 | - Included a bash script to control the number of threads used by numpy during execution. 39 | - Included a new label binarization method ('prop') which binarizes predictions based on the number of positive/negative instances in the train data. 40 | - The library now logs the logistic regression coefficients per method when LR or LRCV are used. 41 | - Included a new performance metric, average precision for the LP, SP, and NR tasks. 42 | - Parameter checks in the EvalSetup class for .ini configuration files can now be turned on or off. 43 | - New parallel coordinates plot has been added to visualize method performance from output pickle files. 44 | 45 | Miscellaneous 46 | ~~~~~~~~~~~~ 47 | - Input type errors are now cached and logged and no longer cause the evaluation to crash. 48 | 49 | 50 | EvalNE v0.3.3 51 | ------------- 52 | 53 | Release date: 14 Dec 2020 54 | 55 | Documentation 56 | ~~~~~~~~~~~~~ 57 | - Release log update. 58 | - Extensive docstring improvements affecting all classes and functions. 59 | - Docstring examples included for all important classes and low level functions. 60 | - Improved variable descriptions in conf.ini. 61 | - Improved Readme.md and docs files. 62 | 63 | New features 64 | ~~~~~~~~~~~~ 65 | - Sign prediction added as a downstream task that can be evaluated (using the SPEvaluator class). 66 | - Three new classes (LPEvalSplit, SPEvalSplit and NREvalSplit) added that simplify the computation of evaluation splits for the LP, SP and NR downstream tasks. 67 | - Added three new heuristic baselines: Cosine Similarity, Leicht-Holme-Newman index and Topological Overlap. 68 | - When used as a command line tool, the library now provides both the train and test evaluation scores in the output folder. 69 | - When used as an API the user can now conveniently store the model predictions for any downstream task. 70 | - Added timeout for baselines evaluation 71 | - Added function that can run other functions in a separate process with given timeout. 72 | 73 | Miscellaneous 74 | ~~~~~~~~~~~~ 75 | - Improved requirement specification in requirements.txt and setup.py. 76 | - Improved library and module level imports. 77 | - General improvements on error and warning messages. 78 | - Memory errors are now catched and logged. 79 | - All numerical output is now rounded to 4 decimals. 80 | 81 | Bugs 82 | ~~~~ 83 | - Fixed a bug that would cause a TimeoutError to be raised incorrectly. 84 | 85 | 86 | EvalNE v0.3.2 87 | ------------- 88 | 89 | Release date: 10 Dec 2019 90 | 91 | Documentation 92 | ~~~~~~~~~~~~~ 93 | - Release log update 94 | - Various docstring improvements 95 | - Improved variable descriptions in conf.ini 96 | 97 | New features 98 | ~~~~~~~~~~~~ 99 | - The user can now set a timeout for the execution of each method in the conf files. E.g. TIMEOUT = 1800 100 | - Conf files now support any sklearn binary classifer in the LP_MODEL variable. E.g. LP_MODEL=sklearn.svm.LinearSVC(C=1.0, kernel=’rbf’, degree=3) 101 | - Conf files also support keyword SVM for the LP_MODEL. This uses the sklearn LinearSVC model and tunes the regularization parameter on a grid [0.1, 1, 10, 100, 1000]. 102 | - Method execution is made safer by using Popen communicate instead of subprocess.run(shell=True) 103 | - Removed lp_model coefficient output. This could lead to errors and failed evaluations for certain Sklearn binary classifiers 104 | - Method compute_pred() of LPEvaluator and NREvaluator classes now tries to use lp_model.predict_proba() if the classifier does not have it, the function defaults to lp_model.predict() 105 | - The scoresheet method get_pandas_df() now includes a repeat parameter which denotes the exact experiment repeat results the user wants in the DF. If repeat=None, the DF returned will contain the average metric over all experiment repeats. 106 | 107 | Miscellaneous 108 | ~~~~~~~~~~~~ 109 | - Log file output now shows timeout errors and LR method selected 110 | - Corrected the cases where some warnings were reported as errors 111 | - Added util.py in the utils module 112 | 113 | Bugs 114 | ~~~~ 115 | - Fixed bug which would prevent the library to store the output when executed from Py3 116 | 117 | 118 | EvalNE v0.3.1 119 | ------------- 120 | 121 | Release date: 2 Nov 2019 122 | 123 | Documentation 124 | ~~~~~~~~~~~~~ 125 | - Release log update 126 | - Various docstring improvements 127 | 128 | New features 129 | ~~~~~~~~~~~~ 130 | - New heuristic for LP named `all_baselines`. Generates a 5-dim edge embedding by combining the existing heuristics [CN, JC, AA, PA, RAI]. 131 | - Automated file headder detection (in the output of embedding methods) is now a function 132 | - Functions for reading the embeddings, predictions and node labels have been added 133 | 134 | 135 | Miscellaneous 136 | ~~~~~~~~~~~~ 137 | - General improvements in NC task 138 | - Added NCScores and NCResults classes 139 | - Pickle file containig evaluation results is now saved incrementally, after each networks has been evaluated. If the user stops the process mid-way the results up to the last network will be available 140 | - Coefficients of the binary classifier per evaluated method are now provided for LP and NR tasks 141 | - Improved exception management 142 | - Improved conf file sanity checks 143 | - Evaluated methods now return a single Results object instead of a list 144 | 145 | Bugs 146 | ~~~~ 147 | - Fixed bug related to plotting PR and AUC curves 148 | - Fixed node classification bugs preventing the evaluaition to run properly 149 | 150 | 151 | EvalNE v0.3.0 152 | ------------- 153 | 154 | Release date: 21 Oct 2019 155 | 156 | Documentation 157 | ~~~~~~~~~~~~~ 158 | - Release log update 159 | 160 | New features 161 | ~~~~~~~~~~~~ 162 | - Old Evaluator class is now LPEvaluator 163 | - Added Network Reconstruction evaluation (NREvaluator) 164 | - Added Node Classification evaluation (NCEvaluator) 165 | - Train/validation splits are now required when initializing Evaluator classes 166 | - Added 3 new algorithms for computing train/test splits. One extremely scalable up to millions of nodes/edges 167 | - Improved error management and error logging 168 | - Edge embedding methods are now always tunned as method parameters. Results for the best are given. 169 | - For link prediction and network reconstruction the user can now evaluate the methods exclusively on train data. 170 | - Addes Scoresheet class to simplify output management 171 | - Export results directly to pandas dataframe and latex tables suppored 172 | 173 | Miscellaneous 174 | ~~~~~~~~~~~~ 175 | - Changed default parameters for EvalSplit 176 | - Added new parameter for EvalSplit.set_split() 177 | - Evaluation output is now always stored as pickle file 178 | - Execution time per method and dataset is not provided 179 | - Train/test average time per dataset is registered 180 | - Added `auto` mode for the Results class to decide if train or test data should be logged 181 | 182 | 183 | EvalNE v0.2.3 184 | ------------- 185 | 186 | Release date: 25 Apr 2019 187 | 188 | Documentation 189 | ~~~~~~~~~~~~~ 190 | - Release log update 191 | - Library diagram minor update 192 | 193 | Bugs 194 | ~~~~ 195 | - Corrected parameter tuning rutine which was minimizing the objective metric given instead of maximizing it. 196 | - Corrected evaluate_cmd() function output. 197 | 198 | New features 199 | ~~~~~~~~~~~~ 200 | - Evaluation output file now contains also a table of execution times per evaluated method. 201 | 202 | Miscellaneous 203 | ~~~~~~~~~~~~ 204 | - Changed behaviour of verbosity flag. Now, if Verbose=False it deactivates all stdout for the methods being evaluated (not stderr) but maintains the library stdout. 205 | - Added more conf.ini files for reproducing the experimental section of different papers. 206 | 207 | 208 | EvalNE v0.2.2 209 | ------------- 210 | 211 | Release date: 14 Mar 2019 212 | 213 | Documentation 214 | ~~~~~~~~~~~~~ 215 | - Readme and docs update to include pip installation 216 | 217 | Miscelaneous 218 | ~~~~~~~~~~~~ 219 | - Library is now pip installable 220 | - Minor bugfixes 221 | 222 | 223 | EvalNE v0.2.1 224 | ------------- 225 | 226 | Release date: 13 Mar 2019 227 | 228 | New features 229 | ~~~~~~~~~~~~ 230 | - Added `WRITE_WEIGHTS_OTHER` in conf files which allows the user to specify if the input train network to the NE methods should have weights or not. If True but the original input network is unweighted, weights of 1 are given to each edge. This features is useful for e.g. the original code of LINE, which requires edges to have weights (all 1 if the graph is unweighted). 231 | - Added `WRITE_DIR_OTHER` in conf files which allows the user to specify if the input train network to the NE methods should be specified with both directions of edges or a single one. 232 | - Added `SEED` in the conf file which sets a general random seed for the whole library. If None the system time is used. 233 | - Added a faster method for splitting non-edges in train and test when all non-edges in the graph are required. 234 | 235 | Documentation 236 | ~~~~~~~~~~~~~ 237 | - Readme and docs update 238 | - Descriptions of each option in conf.ini added 239 | 240 | Miscellaneous 241 | ~~~~~~~~~~~~ 242 | - Removed optional seed parameter from all methods in split_train_test.py 243 | - Removed random seed resetting in the edges split methods 244 | - `simple-example.py` now checks if OpenNE is installed, if not it runs only the LP heuristics. 245 | - Sklearn removed from requirements.txt (already satisfied by scikit-learn) 246 | - `setup.py` update. Ready for making EvalNE pip installable. 247 | - Train/validation fraction was 50/50 which caused the train set to be excesively small and parameter validation not accurate. New value is 90/10. 248 | - Improved warnings in evaluator code 249 | - General code cleaning 250 | 251 | Bugs 252 | ~~~~ 253 | - train/validation and train/test splits used the same random seed for generating the edge split which caused correlation between them. Now the train/validation split is random. 254 | - Fixed a bug which would cause the evaluation of any edge embedding method to crash. 255 | - Precitions from edge embeddings were computed using LogisticRegression.predict(). This gives class labels and not class probabilities resulting in worst estimates of method performance. This has been changed to LogisticRegression.predict_proba() 256 | 257 | 258 | EvalNE v0.2.0 259 | ------------- 260 | 261 | Release date: 4 Feb 2019 262 | 263 | API changes 264 | ~~~~~~~~~~~ 265 | - The evaluate_ne_cmd method has been renamed to evaluate_cmd 266 | - evaluate_cmd can now evaluate node, edge or end to end embedding method 267 | - evaluate_cmd a new method_type parameter has been added to indicate how the method should be evaluated (ne, ee or e2e) 268 | - ScoreSheet object has been removed 269 | - Score method removed from Katz and KatzApprox classes 270 | - Method get_parameters() from Evaluator has been removed 271 | 272 | New features 273 | ~~~~~~~~~~~~ 274 | - Added method_type option in *.ini* files to evaluate (ne, ee or e2e) 275 | - compute_results method now takes an optional label binarizer parameter 276 | - evaluate_ne method now takes an optional label binarizer parameter 277 | - save and pretty_print methods in Results now take a precatk_vals parameter which indcates for which k values to compute this score 278 | - When REPORT SCORES = all is selected in the *.ini* file, the library now presents all the available metrics for each algorithm and dataset averaged over the number of repetitions. 279 | 280 | Documentation 281 | ~~~~~~~~~~~~~ 282 | - Docstring updates 283 | - Release log added to Docs 284 | - Contributing added to Docs 285 | 286 | Miscellaneous 287 | ~~~~~~~~~~~~ 288 | - Exception handling improvements 289 | 290 | Bugs 291 | ~~~~ 292 | - Prevented possible infinite loop while generating non-edges by raising a warning if the used-selected values is > that the max possible non-edges. 293 | 294 | 295 | 296 | 297 | 298 | -------------------------------------------------------------------------------- /evalne/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | EvalNE 3 | ====== 4 | 5 | EvalNE is a Python package for the evaluation of network embedding methods on a variety of downstream prediction tasks. 6 | These tasks include link prediction, sign prediction, network reconstruction and node classification. Basic embedding 7 | and graph visualization functions are also provided. 8 | 9 | See https://evalne.readthedocs.io/en/latest/ for complete documentation. 10 | """ 11 | 12 | __author__ = "Alexandru Mara" 13 | __version__ = "0.4.0" 14 | __bibtex__ = """ 15 | @misc{Mara2019, 16 | author = {Alexandru Mara and Jefrey Lijffijt and Tijl De Bie}, 17 | title = {EvalNE: A Framework for Evaluating Network Embeddings on Link Prediction}, 18 | year = {2019}, 19 | archivePrefix = {arXiv}, 20 | eprint = {1901.09691} 21 | } 22 | """ 23 | 24 | from evalne import evaluation 25 | from evalne import methods 26 | from evalne import utils 27 | -------------------------------------------------------------------------------- /evalne/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from evalne.evaluation import edge_embeddings 2 | from evalne.evaluation import evaluator 3 | from evalne.evaluation import pipeline 4 | from evalne.evaluation import score 5 | from evalne.evaluation import split 6 | -------------------------------------------------------------------------------- /evalne/evaluation/edge_embeddings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This file provides implementations of several operators for computing node-pair embeddings from node feature vectors. 8 | 9 | import numpy as np 10 | 11 | 12 | def average(X, ebunch): 13 | """ 14 | Computes the embedding of each node pair (u, v) in ebunch as the element-wise average of the embeddings of 15 | nodes u and v. 16 | 17 | Parameters 18 | ---------- 19 | X : dict 20 | A dictionary of {`nodeID`: embed_vect, `nodeID`: embed_vect, ...}. Dictionary keys are expected to be of type 21 | string and values array_like. 22 | ebunch : iterable 23 | An iterable of node pairs (u,v) for which the embeddings must be computed. 24 | 25 | Returns 26 | ------- 27 | emb : ndarray 28 | A column vector containing node-pair embeddings as rows. In the same order as ebunch. 29 | 30 | Notes 31 | ----- 32 | Formally, if we use x(u) to denote the embedding corresponding to node u and x(v) to denote the embedding 33 | corresponding to node v, and if we use i to refer to the ith position in these vectors, then, the embedding of the 34 | pair (u, v) can be computed element-wise as: :math:`x(u, v)_i = \\frac{x(u)_i + x(v)_i}{2}`. 35 | Also note that all nodeID's in ebunch must exist in X, otherwise, the method will fail. 36 | 37 | Examples 38 | -------- 39 | Simple example of function use and input parameters: 40 | 41 | >>> X = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 42 | >>> ebunch = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 43 | >>> average(X, ebunch) 44 | array([[ 1. , 1. , 1. , 1. ], 45 | [ 0. , 0. , 0. , 0. ], 46 | [ 2. , 2. , 2. , 2. ], 47 | [ 0.5, 0.5, -0.5, -0.5], 48 | [ 0.5, 0.5, -0.5, -0.5], 49 | [ 1.5, 1.5, 0.5, 0.5], 50 | [ 1.5, 1.5, 0.5, 0.5]]) 51 | 52 | """ 53 | edge_embeds = np.zeros((len(ebunch), len(X[list(X.keys())[0]]))) 54 | i = 0 55 | for edge in ebunch: 56 | edge_embeds[i] = (X[str(edge[0])] + X[str(edge[1])]) / 2.0 57 | i += 1 58 | return edge_embeds 59 | 60 | 61 | def hadamard(X, ebunch): 62 | """ 63 | Computes the embedding of each node pair (u, v) in ebunch as the element-wise product between the 64 | embeddings of nodes u and v. 65 | 66 | Parameters 67 | ---------- 68 | X : dict 69 | A dictionary of {`nodeID`: embed_vect, `nodeID`: embed_vect, ...}. Dictionary keys are expected to be of type 70 | string and values array_like. 71 | ebunch : iterable 72 | An iterable of node pairs (u,v) for which the embeddings must be computed. 73 | 74 | Returns 75 | ------- 76 | emb : ndarray 77 | A column vector containing node-pair embeddings as rows. In the same order as ebunch. 78 | 79 | Notes 80 | ----- 81 | Formally, if we use x(u) to denote the embedding corresponding to node u and x(v) to denote the embedding 82 | corresponding to node v, and if we use i to refer to the ith position in these vectors, then, the embedding of the 83 | pair (u, v) can be computed element-wise as: :math:`x(u, v)_i = x(u)_i * x(v)_i`. 84 | Also note that all nodeID's in ebunch must exist in X, otherwise, the method will fail. 85 | 86 | Examples 87 | -------- 88 | Simple example of function use and input parameters: 89 | 90 | >>> X = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 91 | >>> ebunch = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 92 | >>> hadamard(X, ebunch) 93 | array([[ 0., 0., 0., 0.], 94 | [ 0., 0., 0., 0.], 95 | [ 4., 4., 4., 4.], 96 | [ 0., 0., 0., 0.], 97 | [ 0., 0., 0., 0.], 98 | [ 2., 2., -2., -2.], 99 | [ 2., 2., -2., -2.]]) 100 | 101 | """ 102 | edge_embeds = np.zeros((len(ebunch), len(X[list(X.keys())[0]]))) 103 | i = 0 104 | for edge in ebunch: 105 | edge_embeds[i] = X[str(edge[0])] * X[str(edge[1])] 106 | i += 1 107 | return edge_embeds 108 | 109 | 110 | def weighted_l1(X, ebunch): 111 | """ 112 | Computes the embedding of each node pair (u, v) in ebunch as the element-wise weighted L1 distance between the 113 | embeddings of nodes u and v. 114 | 115 | Parameters 116 | ---------- 117 | X : dict 118 | A dictionary of {`nodeID`: embed_vect, `nodeID`: embed_vect, ...}. Dictionary keys are expected to be of type 119 | string and values array_like. 120 | ebunch : iterable 121 | An iterable of node pairs (u,v) for which the embeddings must be computed. 122 | 123 | Returns 124 | ------- 125 | emb : ndarray 126 | A column vector containing node-pair embeddings as rows. In the same order as ebunch. 127 | 128 | Notes 129 | ----- 130 | Formally, if we use x(u) to denote the embedding corresponding to node u and x(v) to denote the embedding 131 | corresponding to node v, and if we use i to refer to the ith position in these vectors, then, the embedding of the 132 | pair (u, v) can be computed element-wise as: :math:`x(u, v)_i = |x(u)_i - x(v)_i|`. 133 | Also note that all nodeID's in ebunch must exist in X, otherwise, the method will fail. 134 | 135 | Examples 136 | -------- 137 | Simple example of function use and input parameters: 138 | 139 | >>> X = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 140 | >>> ebunch = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 141 | >>> weighted_l1(X, ebunch) 142 | array([[2., 2., 2., 2.], 143 | [0., 0., 0., 0.], 144 | [0., 0., 0., 0.], 145 | [1., 1., 1., 1.], 146 | [1., 1., 1., 1.], 147 | [1., 1., 3., 3.], 148 | [1., 1., 3., 3.]]) 149 | 150 | """ 151 | edge_embeds = np.zeros((len(ebunch), len(X[list(X.keys())[0]]))) 152 | i = 0 153 | for edge in ebunch: 154 | edge_embeds[i] = np.abs(X[str(edge[0])] - X[str(edge[1])]) 155 | i += 1 156 | return edge_embeds 157 | 158 | 159 | def weighted_l2(X, ebunch): 160 | """ 161 | Computes the embedding of each node pair (u, v) in ebunch as the element-wise weighted L2 distance between the 162 | embeddings of nodes u and v. 163 | 164 | Parameters 165 | ---------- 166 | X : dict 167 | A dictionary of {`nodeID`: embed_vect, `nodeID`: embed_vect, ...}. Dictionary keys are expected to be of type 168 | string and values array_like. 169 | ebunch : iterable 170 | An iterable of node pairs (u,v) for which the embeddings must be computed. 171 | 172 | Returns 173 | ------- 174 | emb : ndarray 175 | A column vector containing node-pair embeddings as rows. In the same order as ebunch. 176 | 177 | Notes 178 | ----- 179 | Formally, if we use x(u) to denote the embedding corresponding to node u and x(v) to denote the embedding 180 | corresponding to node v, and if we use i to refer to the ith position in these vectors, then, the embedding of the 181 | pair (u, v) can be computed element-wise as: :math:`x(u, v)_i = (x(u)_i - x(v)_i)^2`. 182 | Also note that all nodeID's in ebunch must exist in X, otherwise, the method will fail. 183 | 184 | Examples 185 | -------- 186 | Simple example of function use and input parameters: 187 | 188 | >>> X = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 189 | >>> ebunch = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 190 | >>> weighted_l2(X, ebunch) 191 | array([[4., 4., 4., 4.], 192 | [0., 0., 0., 0.], 193 | [0., 0., 0., 0.], 194 | [1., 1., 1., 1.], 195 | [1., 1., 1., 1.], 196 | [1., 1., 9., 9.], 197 | [1., 1., 9., 9.]]) 198 | 199 | """ 200 | edge_embeds = np.zeros((len(ebunch), len(X[list(X.keys())[0]]))) 201 | i = 0 202 | for edge in ebunch: 203 | edge_embeds[i] = np.power(X[str(edge[0])] - X[str(edge[1])], 2) 204 | i += 1 205 | return edge_embeds 206 | 207 | 208 | def compute_edge_embeddings(X, ebunch, method='hadamard'): 209 | """ 210 | Computes the embedding of each node pair (u, v) in ebunch as an element-wise operation on the embeddings of the end 211 | nodes u and v. The operator used is determined by the `method` parameter. 212 | 213 | Parameters 214 | ---------- 215 | X : dict 216 | A dictionary of {`nodeID`: embed_vect, `nodeID`: embed_vect, ...}. Dictionary keys are expected to be of type 217 | string and values array_like. 218 | ebunch : iterable 219 | An iterable of node pairs (u,v) for which the embeddings must be computed. 220 | method : string, optional 221 | The operator to be used for computing the node-pair embeddings. Options are: `average`, `hadamard`, 222 | `weighted_l1` or `weighted_l2`. Default is `hadamard`. 223 | 224 | Returns 225 | ------- 226 | emb : ndarray 227 | A column vector containing node-pair embeddings as rows. In the same order as ebunch. 228 | 229 | Examples 230 | -------- 231 | Simple example of function use and input parameters: 232 | 233 | >>> X = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 234 | >>> ebunch = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 235 | >>> compute_edge_embeddings(X, ebunch, 'average') 236 | array([[ 1. , 1. , 1. , 1. ], 237 | [ 0. , 0. , 0. , 0. ], 238 | [ 2. , 2. , 2. , 2. ], 239 | [ 0.5, 0.5, -0.5, -0.5], 240 | [ 0.5, 0.5, -0.5, -0.5], 241 | [ 1.5, 1.5, 0.5, 0.5], 242 | [ 1.5, 1.5, 0.5, 0.5]]) 243 | 244 | """ 245 | if method == 'hadamard': 246 | return hadamard(X, ebunch) 247 | elif method == 'average': 248 | return average(X, ebunch) 249 | elif method == 'weighted_l1': 250 | return weighted_l1(X, ebunch) 251 | elif method == 'weighted_l2': 252 | return weighted_l2(X, ebunch) 253 | else: 254 | raise ValueError("Unknown method!") 255 | -------------------------------------------------------------------------------- /evalne/methods/__init__.py: -------------------------------------------------------------------------------- 1 | from evalne.methods.katz import * 2 | from evalne.methods.similarity import * 3 | -------------------------------------------------------------------------------- /evalne/methods/katz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This file provides two different implementations of the katz score. The first, computes the exact score for the 8 | # complete graph using the adjacency matrix. The second, computes the approximated Katz score for each input node-pair. 9 | # Only undirected Graphs and Digraphs are supported. 10 | 11 | import networkx as nx 12 | import numpy as np 13 | 14 | __all__ = ['Katz', 'KatzApprox'] 15 | 16 | 17 | class Katz(object): 18 | """ 19 | Computes the exact katz similarity based on paths between nodes in the graph. Shorter paths will contribute more 20 | than longer ones. This contribution depends of the damping factor 'beta'. The exact katz score is computed using 21 | the adj matrix of the full graph. 22 | 23 | Parameters 24 | ---------- 25 | G : graph 26 | A NetworkX graph or digraph with nodes being consecutive integers starting at 0. 27 | beta = float, optional 28 | The damping factor for the model. Default is 0.005. 29 | 30 | Notes 31 | ----- 32 | The execution is based on dense matrices, so it may run out of memory. 33 | """ 34 | 35 | def __init__(self, G, beta=0.005): 36 | self._G = G 37 | self.beta = beta 38 | self.sim = self._fit() 39 | 40 | def _fit(self): 41 | 42 | # Version using sparse matrices 43 | # adj = nx.adjacency_matrix(self._G, nodelist=range(len(self._G.nodes))) 44 | # ident = sparse.identity(len(self._G.nodes)).tocsc() 45 | # sim = inv(ident - adj.multiply(self.beta).T) - ident 46 | # adj = nx.adjacency_matrix(self._G, nodelist=range(len(self._G.nodes))) 47 | # aux = adj.multiply(-self.beta).T 48 | # aux.setdiag(1+aux.diagonal(), k=0) 49 | # sim = inv(aux) 50 | # sim.setdiag(sim.diagonal()-1) 51 | # print(sim.nnz) 52 | # print(adj.nnz) 53 | 54 | # Version using dense matrices 55 | adj = nx.adjacency_matrix(self._G, nodelist=range(len(self._G.nodes))) 56 | aux = adj.T.multiply(-self.beta).todense() 57 | np.fill_diagonal(aux, 1+aux.diagonal()) 58 | sim = np.linalg.inv(aux) 59 | np.fill_diagonal(sim, sim.diagonal()-1) 60 | return sim 61 | 62 | def predict(self, ebunch): 63 | """ 64 | Computes the katz score for all node-pairs in ebunch. 65 | 66 | Parameters 67 | ---------- 68 | ebunch : iterable 69 | An iterable of node-pairs for which to compute the katz score. 70 | 71 | Returns 72 | ------- 73 | ndarray 74 | An array containing the similarity scores. 75 | """ 76 | ebunch = np.array(ebunch) 77 | return np.array(self.sim[ebunch[:, 0], ebunch[:, 1]]).flatten() 78 | 79 | def save_sim_matrix(self, filename): 80 | """ 81 | Stores the similarity matrix to a file with the given name. 82 | 83 | Parameters 84 | ---------- 85 | filename : string 86 | The name and path of the file where the similarity matrix should be stored. 87 | """ 88 | np.savetxt(filename, self.sim, delimiter=',', fmt='%d') 89 | 90 | def get_params(self): 91 | """ 92 | Returns a dictionary of model parameters. 93 | 94 | Returns 95 | ------- 96 | params : dict 97 | A dictionary of model parameters and their values. 98 | """ 99 | params = {'beta': self.beta} 100 | return params 101 | 102 | 103 | class KatzApprox(object): 104 | """ 105 | Computes the approximated katz similarity based on paths between nodes in the graph. Shorter paths will contribute 106 | more than longer ones. This contribution depends of the damping factor 'beta'. The approximated score is computed 107 | using all paths between nodes of length at most 'path_len'. 108 | 109 | Parameters 110 | ---------- 111 | G : graph 112 | A NetworkX graph or digraph. 113 | beta : float, optional 114 | The damping factor for the model. Default is 0.005. 115 | path_len : int, optional 116 | The maximum path length to consider between each pair of nodes. Default is 3. 117 | 118 | Notes 119 | ----- 120 | The implementation follows the indication in [1]. It becomes extremely slow for large dense graphs. 121 | 122 | References 123 | ---------- 124 | .. [1] R. Laishram "Link Prediction in Dynamic Weighted and Directed Social Network using Supervised Learning" 125 | Dissertations - ALL. 355. 2015 126 | """ 127 | 128 | def __init__(self, G, beta=0.005, path_len=3): 129 | self._G = G 130 | self.beta = beta 131 | self.path_len = path_len 132 | 133 | def fit_predict(self, ebunch): 134 | """ 135 | Computes the katz score for all node-pairs in ebunch. 136 | 137 | Parameters 138 | ---------- 139 | ebunch : iterable 140 | An iterable of node-pairs for which to compute the katz score. 141 | 142 | Returns 143 | ------- 144 | ndarray 145 | An array containing the similarity scores. 146 | """ 147 | res = list() 148 | betas = np.zeros(self.path_len) 149 | for i in range(len(betas)): 150 | betas[i] = np.power(self.beta, i+1) 151 | for u, v in ebunch: 152 | paths = np.zeros(self.path_len) 153 | for path in nx.all_simple_paths(self._G, source=u, target=v, cutoff=self.path_len): 154 | paths[len(path)-2] += 1 # Simple paths output at most path_len+1, plus -1 because indexing at 0 155 | res.append(np.sum(betas * paths)) 156 | return np.array(res).reshape(-1, 1) 157 | 158 | def get_params(self): 159 | """ 160 | Returns a dictionary of model parameters. 161 | 162 | Returns 163 | ------- 164 | params : dict 165 | A dictionary of model parameters and their values. 166 | """ 167 | params = {'beta': self.beta, 'path_len': self.path_len} 168 | return params 169 | -------------------------------------------------------------------------------- /evalne/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/evalne/tests/__init__.py -------------------------------------------------------------------------------- /evalne/tests/data/test.csv: -------------------------------------------------------------------------------- 1 | 1,2 2 | 3,2 3 | 2,4 4 | 3,4 5 | 4,3 6 | -------------------------------------------------------------------------------- /evalne/tests/test_baselines.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | import random 8 | import numpy as np 9 | 10 | from time import time 11 | 12 | from evalne.evaluation import evaluator 13 | from evalne.evaluation import score 14 | from evalne.evaluation import split 15 | from evalne.methods import katz 16 | from evalne.utils import preprocess as pp 17 | 18 | 19 | # TODO: there are big differences between katz exact and approx. Exact probably is wrong. 20 | 21 | 22 | def test_katz(nee): 23 | # Evaluate exact katz implementation 24 | exact = katz.Katz(nee.traintest_split.TG) 25 | train_pred = exact.predict(nee.traintest_split.train_edges) 26 | test_pred = exact.predict(nee.traintest_split.test_edges) 27 | ms = score.Results(method='Katz', params=exact.get_params(), 28 | train_pred=train_pred, train_labels=nee.traintest_split.train_labels, 29 | test_pred=test_pred, test_labels=nee.traintest_split.test_labels) 30 | ms.pretty_print(results='test', precatk_vals=[2, 4, 6, 10, 100, 1000]) 31 | # ms.plot() 32 | 33 | # # Evaluate approx katz implementation 34 | # approx = katz.KatzApprox(nee.traintest_split.TG) 35 | # train_pred = approx.fit_predict(nee.traintest_split.train_edges) 36 | # test_pred = approx.fit_predict(nee.traintest_split.test_edges) 37 | # ms = score.Results(method='Katz', params=approx.get_params(), 38 | # train_pred=train_pred, train_labels=nee.traintest_split.train_labels, 39 | # test_pred=test_pred, test_labels=nee.traintest_split.test_labels) 40 | # ms.pretty_print(results='test', precatk_vals=[2, 4, 6, 10, 100, 1000]) 41 | # # ms.plot() 42 | 43 | 44 | def test_baselines(nee, directed): 45 | """ 46 | Experiment to test the baselines. 47 | """ 48 | print('Evaluating baselines...') 49 | 50 | # Set the baselines 51 | methods = ['random_prediction', 'common_neighbours', 'jaccard_coefficient', 'adamic_adar_index', 52 | 'preferential_attachment', 'resource_allocation_index'] 53 | 54 | # Results list 55 | results = list() 56 | 57 | # Evaluate baseline methods 58 | for method in methods: 59 | if directed: 60 | results.append(nee.evaluate_baseline(method=method, neighbourhood="in")) 61 | results.append(nee.evaluate_baseline(method=method, neighbourhood="out")) 62 | else: 63 | results.append(nee.evaluate_baseline(method=method)) 64 | 65 | for result in results: 66 | result.pretty_print() 67 | 68 | results[0].save_predictions('predictions.txt') 69 | 70 | 71 | def run_test(): 72 | 73 | random.seed(42) 74 | np.random.seed(42) 75 | 76 | # Set some variables 77 | filename = "./data/network.edgelist" 78 | directed = False 79 | 80 | # Load the test graph 81 | G = pp.load_graph(filename, delimiter=",", comments='#', directed=directed) 82 | G, ids = pp.prep_graph(G) 83 | 84 | # Print some stars about the graph 85 | pp.get_stats(G) 86 | 87 | # Generate one train/test split with all edges in train set 88 | start = time() 89 | traintest_split = split.EvalSplit() 90 | # traintest_split.read_splits('./', 0) 91 | traintest_split.compute_splits(G, train_frac=0.9) 92 | end = time() - start 93 | print("\nSplits computed in {} sec".format(end)) 94 | 95 | # Create an evaluator 96 | nee = evaluator.LPEvaluator(traintest_split) 97 | 98 | # Test baselines 99 | start = time() 100 | test_baselines(nee, directed) 101 | end = time() - start 102 | print("\nBaselines computed in {} sec".format(end)) 103 | 104 | # Test Katz 105 | start = time() 106 | test_katz(nee) 107 | end = time() - start 108 | print("\nKatz computed in {} sec".format(end)) 109 | 110 | 111 | if __name__ == "__main__": 112 | run_test() 113 | 114 | -------------------------------------------------------------------------------- /evalne/tests/test_edgeembeds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | import timeit 8 | import numpy as np 9 | 10 | from evalne.evaluation import edge_embeddings 11 | 12 | 13 | def test(): 14 | 15 | a = {'1': np.array([0, 0, 0, 0]), '2': np.array([2, 2, 2, 2]), '3': np.array([1, 1, -1, -1])} 16 | e = ((2, 1), (1, 1), (2, 2), (1, 3), (3, 1), (2, 3), (3, 2)) 17 | ee = edge_embeddings.compute_edge_embeddings(a, e, "average") 18 | print('Input node embeddings:') 19 | print(a) 20 | print('Ebunch:') 21 | print(e) 22 | print('Output edge embeddings:') 23 | print(ee) 24 | 25 | 26 | def time_test(): 27 | 28 | # Create a dictionary simulating the node embeddings 29 | keys = map(str, range(100)) 30 | vals = np.random.randn(100, 10) 31 | d = dict(zip(keys, vals)) 32 | 33 | # Create set of edges 34 | num_edges = 1000000 35 | edges = list(zip(np.random.randint(0, 100, num_edges), np.random.randint(0, 100, num_edges))) 36 | 37 | start = timeit.default_timer() 38 | res = edge_embeddings.compute_edge_embeddings(d, edges, "average") 39 | end = timeit.default_timer() - start 40 | 41 | print("Processed in: {}".format(end)) 42 | 43 | 44 | if __name__ == "__main__": 45 | 46 | test() 47 | time_test() 48 | -------------------------------------------------------------------------------- /evalne/tests/test_nr_sampling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 24/03/2020 6 | 7 | import time 8 | 9 | from scipy.sparse import rand 10 | 11 | from evalne.utils import split_train_test as stt 12 | 13 | 14 | def test(): 15 | a = rand(1000, 1000, 0.3) 16 | a = a.tocsr() 17 | 18 | start = time.time() 19 | p1, n1 = stt.random_edge_sample(a, samp_frac=0.01) 20 | end = time.time() - start 21 | print("Exec time Impl. 1: {}".format(end)) 22 | 23 | start = time.time() 24 | p2, n2 = stt.random_edge_sample(a) 25 | end = time.time() - start 26 | print("Exec time Impl. 2: {}".format(end)) 27 | 28 | print('Results Impl. 1: {} pos, {} neg.'.format(p1.shape, n1.shape)) 29 | print('Results Impl. 2: {} pos, {} neg.'.format(p2.shape, n2.shape)) 30 | 31 | 32 | if __name__ == "__main__": 33 | test() 34 | -------------------------------------------------------------------------------- /evalne/tests/test_preprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | import time 8 | import random 9 | import networkx as nx 10 | 11 | from evalne.utils import preprocess as pp 12 | from evalne.utils import split_train_test as stt 13 | 14 | 15 | def test(): 16 | # Variables 17 | dataset_path = "./data/" 18 | output_path = "./data/" 19 | test_name = "network.edgelist" 20 | 21 | # Load a graph 22 | G = pp.load_graph(dataset_path + test_name, delimiter=',', comments='#', directed=True) 23 | 24 | # Print some stats 25 | print("") 26 | print("Original graph stats:") 27 | print("-----------------------------------------") 28 | pp.get_stats(G) 29 | 30 | # Save the graph 31 | pp.save_graph(G, output_path + "orig_graph.edgelist", delimiter=",") 32 | 33 | # Load the saved graph 34 | G2 = pp.load_graph(output_path + "orig_graph.edgelist", delimiter=",", comments='#', directed=True) 35 | 36 | # Stats comparison 37 | print("Has the same stats after being loaded?:") 38 | print("-----------------------------------------") 39 | pp.get_stats(G2) 40 | 41 | # Preprocess the graph 42 | GP, ids = pp.prep_graph(G2, del_self_loops=False, relabel=True) 43 | 44 | print("Preprocessed graph stats (restricted to main cc):") 45 | print("-----------------------------------------") 46 | pp.get_stats(GP) 47 | 48 | pp.save_graph(GP, output_path + "prep_graph.edgelist", delimiter=",") 49 | 50 | print("Sample of 10 (oldNodeID, newNodeID):") 51 | print("-----------------------------------------") 52 | print(ids[0:10]) 53 | 54 | pp.get_redges_false(GP, output_path + "redges_false.csv") 55 | 56 | 57 | def test_split(): 58 | # Variables 59 | dataset_path = "./data/" 60 | output_path = "./data/" 61 | test_name = "network.edgelist" 62 | subgraph_size = 400 63 | train_frac = 0.5 64 | directed = False 65 | 66 | # Load a graph 67 | G = pp.load_graph(dataset_path + test_name, delimiter=",", comments='#', directed=directed) 68 | 69 | # Restrict graph to a sub-graph of 'subgraph_size' nodes 70 | SG = G.subgraph(random.sample(G.nodes, subgraph_size)).copy() 71 | 72 | # Preprocess the graph 73 | PSG, ids = pp.prep_graph(SG, relabel=True, del_self_loops=True, maincc=True) 74 | 75 | # Save the preprocessed graph 76 | pp.save_graph(PSG, output_path + "prep_graph.edgelist", delimiter=",") 77 | 78 | # Compute train/test splits 79 | start = time.time() 80 | train_stt, test_stt = stt.split_train_test(PSG, train_frac=train_frac) 81 | end = time.time() - start 82 | print("Exec time stt: {}".format(end)) 83 | 84 | # Check that the train graph generated with stt has one single cc 85 | if directed: 86 | TG_stt = nx.DiGraph() 87 | TG_stt.add_edges_from(train_stt) 88 | print("Number of weakly CCs with stt: {}".format(nx.number_weakly_connected_components(TG_stt))) 89 | else: 90 | TG_stt = nx.Graph() 91 | TG_stt.add_edges_from(train_stt) 92 | print("Number of CCs with stt: {}".format(nx.number_connected_components(TG_stt))) 93 | print("Number train edges stt: {}".format(len(train_stt))) 94 | print("Number test edges stt: {}".format(len(test_stt))) 95 | print("Number of nodes in train graph: {}".format(len(TG_stt.nodes))) 96 | 97 | # Preprocess the graph 98 | PSG, ids = pp.prep_graph(SG, relabel=True, del_self_loops=True, maincc=False) 99 | 100 | # Compute train/test splits 101 | start = time.time() 102 | train_rstt, test_rstt = stt.rand_split_train_test(PSG, train_frac=train_frac) 103 | end = time.time() - start 104 | print("\nExec time rand_stt: {}".format(end)) 105 | 106 | # Check that the train graph generated with rstt has one single cc 107 | if directed: 108 | TG_rstt = nx.DiGraph() 109 | TG_rstt.add_edges_from(train_rstt) 110 | print("Number of weakly CCs with rstt: {}".format(nx.number_weakly_connected_components(TG_rstt))) 111 | else: 112 | TG_rstt = nx.Graph() 113 | TG_rstt.add_edges_from(train_rstt) 114 | print("Number of CCs with rstt: {}".format(nx.number_connected_components(TG_rstt))) 115 | print("Number train edges rstt: {}".format(len(train_rstt))) 116 | print("Number test edges rstt: {}".format(len(test_rstt))) 117 | print("Number of nodes in train graph: {}".format(len(TG_rstt.nodes))) 118 | 119 | # Compute set of false edges 120 | # train_E_false, test_E_false = stt.generate_false_edges_owa(SG, train_E=train_E, test_E=test_E, num_fe_train=0, 121 | # num_fe_test=100) 122 | # train_E_false, test_E_false = stt.generate_false_edges_owa(G, train_E=train_E, test_E=test_E, num_fe_train=None, 123 | # num_fe_test=None, seed=99) 124 | 125 | # Store the edge splits generated 126 | # stt.store_train_test_splits('./', train_E, train_E_false, test_E, test_E_false, split_id=0) 127 | 128 | 129 | if __name__ == "__main__": 130 | 131 | test() 132 | test_split() 133 | 134 | 135 | -------------------------------------------------------------------------------- /evalne/tests/test_spantree.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | import time 8 | import networkx as nx 9 | 10 | from evalne.utils import preprocess as pp 11 | from evalne.utils import split_train_test as stt 12 | 13 | 14 | def test_stt(): 15 | # Variables 16 | dataset_path = "./data/" 17 | test_name = "network.edgelist" 18 | frac = 0.5 19 | 20 | # Load a graph 21 | G = pp.load_graph(dataset_path + test_name, delimiter=",", comments='#', directed=False) 22 | 23 | # Preprocess the graph for stt alg. 24 | SG, ids = pp.prep_graph(G, relabel=True, del_self_loops=True, maincc=True) 25 | 26 | # Split train/test using stt 27 | start = time.time() 28 | train_E, test_E = stt.split_train_test(SG, train_frac=frac) 29 | end1 = time.time() - start 30 | 31 | # Compute the false edges 32 | train_E_false, test_E_false = stt.generate_false_edges_owa(SG, train_E=train_E, test_E=test_E, 33 | num_fe_train=None, num_fe_test=None) 34 | # Store data to file 35 | _ = stt.store_train_test_splits(dataset_path + "stt_frac_" + str(frac), 36 | train_E=train_E, train_E_false=train_E_false, test_E=test_E, 37 | test_E_false=test_E_false, split_id=0) 38 | 39 | # Split train/test using rstt 40 | start = time.time() 41 | tr_E, te_E = stt.rand_split_train_test(G, train_frac=frac) 42 | end2 = time.time() - start 43 | 44 | train_E, test_E, J, mp = pp.relabel_nodes(tr_E, te_E, G.is_directed()) 45 | 46 | print("Number of nodes in G: {}".format(len(G.nodes()))) 47 | print("Number of nodes in J: {}".format(len(J.nodes()))) 48 | print("Are nodes in J sequential integers? {}".format(not len(set(J.nodes()) - set(range(len(J.nodes())))))) 49 | 50 | checks = list() 51 | queries = 200 52 | # Check if the mapping is correct 53 | for i in range(queries): 54 | ag = tr_E.pop() # a random element from train 55 | aj = (mp[ag[0]], mp[ag[1]]) # check what it maps to in J 56 | checks.append(aj in train_E) 57 | # print("Random tuple from G: {}".format(ag)) 58 | # print("The tuple maps in J to: {}".format(aj)) 59 | # print("Is that tuple in the new train?: {}".format(aj in train_E)) 60 | 61 | print("For train edges out of {} samples, {} were in the relabeled train_E".format(queries, sum(checks))) 62 | 63 | checks = list() 64 | # Check if the mapping is correct 65 | for i in range(queries): 66 | ag = te_E.pop() # a random element from test 67 | aj = (mp[ag[0]], mp[ag[1]]) # check what it maps to in J 68 | checks.append(aj in test_E) 69 | # print("Random tuple from G: {}".format(ag)) 70 | # print("The tuple maps in J to: {}".format(aj)) 71 | # print("Is that tuple in the new train?: {}".format(aj in train_E)) 72 | 73 | print("For test edges out of {} samples, {} were in the relabeled test_E".format(queries, sum(checks))) 74 | 75 | # Compute the false edges 76 | train_E_false, test_E_false = stt.generate_false_edges_owa(J, train_E=train_E, test_E=test_E, 77 | num_fe_train=None, num_fe_test=None) 78 | # Store data to file 79 | _ = stt.store_train_test_splits(dataset_path + "rstt_frac_" + str(frac), 80 | train_E=train_E, train_E_false=train_E_false, test_E=test_E, 81 | test_E_false=test_E_false, split_id=0) 82 | 83 | 84 | def test_split(): 85 | # Variables 86 | dataset_path = "./data/" 87 | test_name = "network.edgelist" 88 | 89 | # Load a graph 90 | SG = pp.load_graph(dataset_path + test_name, delimiter=",", comments='#', directed=False) 91 | 92 | # Preprocess the graph 93 | SG, ids = pp.prep_graph(SG, relabel=True, del_self_loops=True) 94 | print("Number of CCs input: {}".format(nx.number_connected_components(SG))) 95 | 96 | # Store the edges in the graphs as a set E 97 | E = set(SG.edges()) 98 | 99 | # Use LERW approach to get the ST 100 | start = time.time() 101 | train_lerw = stt.wilson_alg(SG, E) 102 | end1 = time.time() - start 103 | 104 | # Use BRO approach to get the ST 105 | start = time.time() 106 | train_bro = stt.broder_alg(SG, E) 107 | end2 = time.time() - start 108 | 109 | print("LERW time: {}".format(end1)) 110 | print("Bro time: {}".format(end2)) 111 | 112 | print("Num tr_e lerw: {}".format(len(train_lerw))) 113 | print("Num tr_e bro: {}".format(len(train_bro))) 114 | 115 | print("All tr_e in E for lerw?: {}".format(train_lerw - E)) 116 | print("All tr_e in E for bro?: {}".format(train_bro - E)) 117 | 118 | # Check that the graph generated with lerw has indeed one single cc 119 | TG_lerw = nx.Graph() 120 | TG_lerw.add_edges_from(train_lerw) 121 | print("Number of CCs with lerw: {}".format(nx.number_connected_components(TG_lerw))) 122 | 123 | # Check that the graph generated with broder algorithm has indeed one single cc 124 | TG_bro = nx.Graph() 125 | TG_bro.add_edges_from(train_bro) 126 | print("Number of CCs with lerw: {}".format(nx.number_connected_components(TG_bro))) 127 | 128 | 129 | if __name__ == "__main__": 130 | 131 | # Run the tests 132 | test_split() 133 | test_stt() 134 | 135 | -------------------------------------------------------------------------------- /evalne/tests/test_visualize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | import pickle 8 | import numpy as np 9 | import pandas as pd 10 | 11 | from evalne.utils.viz_utils import * 12 | 13 | 14 | def test(): 15 | # Generate set of nodes 16 | ids = range(10) 17 | 18 | # Set some random node embeddings 19 | emb = np.array([[0.82420207, 0.93905952, 0.0443836, 0.54250611, 0.30456824], 20 | [0.16079168, 0.15119187, 0.40094691, 0.79790962, 0.84341248], 21 | [0.57813155, 0.97857005, 0.65691974, 0.32131624, 0.22398546], 22 | [0.12097439, 0.91100631, 0.13567747, 0.55608758, 0.46882953], 23 | [0.81526756, 0.39482937, 0.57112954, 0.73773972, 0.93670739], 24 | [0.59268631, 0.87080881, 0.73983155, 0.31100985, 0.77501675], 25 | [0.53125864, 0.60695178, 0.91817668, 0.2321715, 0.19028287], 26 | [0.13669606, 0.69945586, 0.59937428, 0.08156526, 0.21188543], 27 | [0.64635913, 0.01367627, 0.90677346, 0.28922694, 0.59633913], 28 | [0.06707187, 0.33893169, 0.36597878, 0.0011946, 0.07324235]]) 29 | 30 | # Create a graph 31 | g = nx.Graph() 32 | 33 | # Add some nodes and random edges 34 | g.add_nodes_from(ids) 35 | aux1 = np.random.choice(ids, 10) 36 | aux2 = np.random.choice(ids, 10) 37 | g.add_edges_from(zip(aux1, aux2)) 38 | 39 | # Create a set of labels 40 | labels = dict(zip(ids, ids)) 41 | 42 | # Test embedding plotting 43 | plot_emb2d(emb) 44 | 45 | # Test graph plotting with spring layout 46 | plot_graph2d(g) 47 | plot_graph2d(g, labels=labels, colors=ids) 48 | 49 | # Test graph plotting with given positions 50 | plot_graph2d(g, emb, labels=labels, colors=ids) 51 | plot_graph2d(g, emb=emb[:, 0:2]) 52 | 53 | 54 | def test_parallel_coord(): 55 | features = ['eval_time', 'auroc'] 56 | scoresheet = pickle.load(open('data/test.pkl', 'rb')) 57 | parallel_coord(scoresheet, features, class_col='methods') 58 | 59 | 60 | if __name__ == "__main__": 61 | test() 62 | #test_parallel_coord() 63 | -------------------------------------------------------------------------------- /evalne/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from evalne.utils import preprocess 2 | from evalne.utils import split_train_test 3 | from evalne.utils import util 4 | from evalne.utils import viz_utils 5 | -------------------------------------------------------------------------------- /evalne/utils/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This file provides a set of functions for importing binary classifiers from a given sting and for running command line 8 | # calls and python functions in independent processes with given pre-defined timeouts. 9 | 10 | import os 11 | import queue 12 | import shlex 13 | import importlib 14 | 15 | from subprocess import Popen 16 | from threading import Timer 17 | from multiprocessing import Process, Queue 18 | 19 | 20 | class TimeoutExpired(Exception): 21 | pass 22 | 23 | 24 | def auto_import(classpath): 25 | """ 26 | Imports any Sklearn binary classifier from a string. 27 | 28 | Parameters 29 | ---------- 30 | classpath : string 31 | A string indicating the full path the any Sklearn classifier and its parameters. 32 | 33 | Returns 34 | ------- 35 | clf : object 36 | The classifier instance. 37 | 38 | Raises 39 | ------ 40 | ValueError 41 | If the classifier could not be imported. 42 | 43 | Examples 44 | -------- 45 | Importing the SVC classifier with user-defined parameters: 46 | 47 | >>> auto_import("sklearn.svm.SVC(C=1.0, kernel='rbf')") 48 | SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, 49 | decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf', 50 | max_iter=-1, probability=False, random_state=None, shrinking=True, 51 | tol=0.001, verbose=False) 52 | 53 | Importing a decision tree classifier with no parameters: 54 | 55 | >>> auto_import("sklearn.ensemble.ExtraTreesClassifier()") 56 | ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini', 57 | max_depth=None, max_features='auto', max_leaf_nodes=None, 58 | min_impurity_decrease=0.0, min_impurity_split=None, 59 | min_samples_leaf=1, min_samples_split=2, 60 | min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1, 61 | oob_score=False, random_state=None, verbose=0, warm_start=False) 62 | 63 | """ 64 | comps = classpath.split('.') 65 | if len(comps) > 1: 66 | for i, c in enumerate(reversed(comps)): 67 | if c[0] not in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']: 68 | break 69 | class_str = '.'.join(comps[-(i+1):]) 70 | # params = comps[-1].split('(')[1].replace(')', '') 71 | # param_dict = dict(map(lambda x: x.strip().split('='), params.split(','))) 72 | module_str = '.'.join(comps[:-(i+1)]) 73 | module = importlib.import_module(module_str) 74 | clf = eval("module." + class_str) 75 | else: 76 | raise ValueError("Classifier has to be an Sklearn class given as e.g. `sklearn.svm.SVC(C=1.0, kernel='rbf')`") 77 | return clf 78 | 79 | 80 | def run(cmd, timeout, verbose): 81 | """ 82 | Runs the cmd command provided as input in a new process. If execution time exceeds timeout, the process is killed 83 | and a TimeoutExpired exception is raised. 84 | 85 | Parameters 86 | ---------- 87 | cmd : string 88 | A string indicating the command to run on the command line. 89 | timeout : int or float 90 | A value indicating the maximum number of second the process is allowed to run for. 91 | verbose : bool 92 | Boolean indicating if the execution output should be shown or not (pipes stdout and stderr to devnull). 93 | 94 | Raises 95 | ------ 96 | TimeoutExpired 97 | If the execution time exceeds the number of second indicated by timeout. 98 | 99 | Notes 100 | ----- 101 | The method additionally raises ImportError, IOError and AttributeError if these are encountered during execution 102 | of the cmd command. 103 | 104 | Examples 105 | -------- 106 | Runs a command that prints Start, sleeps for 5 seconds and prints Done 107 | 108 | >>> util.run("python -c 'import time; print(\"Start\"); time.sleep(5); print(\"Done\")'", 7, True) 109 | Start 110 | Done 111 | 112 | Same as previous command but now it does not print Done because it times out after 2 seconds 113 | 114 | >>> util.run("python -c 'import time; print(\"Start\"); time.sleep(5); print(\"Done\")'", 2, True) 115 | Start 116 | Traceback (most recent call last): 117 | File "", line 1, in 118 | File "EvalNE/evalne/utils/util.py", line 84, in run 119 | A string indicating the command to run on the command line. 120 | TimeoutExpired: Command `python -c 'import time; print("Start"); time.sleep(5); print("Done")'` timed out 121 | 122 | """ 123 | if verbose: 124 | sto = None 125 | ste = None 126 | else: 127 | devnull = open(os.devnull, 'w') 128 | sto = devnull 129 | ste = devnull 130 | # Alternative without timeout 131 | # subprocess.run(cmd, shell=True, stdout=sto, stderr=ste) 132 | proc = Popen(shlex.split(cmd), stdout=sto, stderr=ste) 133 | timer = Timer(timeout, proc.kill) 134 | try: 135 | timer.start() 136 | proc.communicate() 137 | 138 | except (ImportError, IOError, AttributeError) as e: 139 | raise e 140 | 141 | finally: 142 | if not timer.is_alive() and proc.poll() != 0: 143 | raise TimeoutExpired('Command `{}` timed out'.format(cmd)) 144 | timer.cancel() 145 | 146 | 147 | def run_function(timeout, func, *args): 148 | """ 149 | Runs the function provided as input in a new process. If execution time exceeds timeout, the process is killed 150 | and a TimeoutExpired exception is raised. 151 | 152 | Parameters 153 | ---------- 154 | timeout : int or float 155 | A value indicating the maximum number of seconds the process is allowed to run for or None. 156 | func : object 157 | A function to be executed. First parameter must be a queue object. Check notes section for more details. 158 | *args 159 | Variable length argument list for function func. The list shloud **not** include the queue object. 160 | 161 | Raises 162 | ------ 163 | TimeoutExpired 164 | If the execution time exceeds the number of second indicated by timeout. 165 | 166 | Notes 167 | ----- 168 | The input function func must take as a first parameter a queue object q. This is used to communicate results 169 | between the process where the function is running and the main thread. The list of args does not need to include 170 | the queue object, it is automatically inserted by this function. 171 | 172 | Examples 173 | -------- 174 | Runs function foo for at most 100 seconds and returns the result. Foo must put the result in q. 175 | 176 | >>> def foo(q, a, b): 177 | ... q.put(a+b) 178 | ... 179 | >>> run_function(100, foo, *[1, 2]) 180 | 3 181 | 182 | """ 183 | # Initialize a process to run the function and a queue to communicate results 184 | q = Queue() 185 | p = Process(target=func, args=(q,) + args) 186 | 187 | try: 188 | # Start the process and join with given timeout 189 | p.start() 190 | # Block until results are available in the queue and start consuming them 191 | res = q.get(block=True, timeout=timeout) 192 | # Finish the child process 193 | p.join() 194 | return res 195 | 196 | except queue.Empty: 197 | raise TimeoutExpired('Function `{}` has timed out'.format(func.__name__)) 198 | 199 | finally: 200 | # Make sure we kill the process after execution if its still alive 201 | if p.is_alive(): 202 | p.terminate() 203 | -------------------------------------------------------------------------------- /evalne/utils/viz_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 16/04/2019 6 | 7 | # This file provides simple methods for embedding and graph visualization. 8 | # T-SNE is applied to embeddings with more than two dimensions in order to plot them in a 2d space. 9 | 10 | import os 11 | import pandas as pd 12 | import networkx as nx 13 | import matplotlib as mpl 14 | from sklearn.manifold import TSNE 15 | 16 | if os.environ.get('DISPLAY', '') == '': 17 | mpl.use('Agg') 18 | import matplotlib.pyplot as plt 19 | 20 | 21 | def plot_emb2d(emb, colors=None, filename=None): 22 | """ 23 | Generates a scatter plot of the given embeddings. Optional colors for the nodes can be provided as well as 24 | a filename to store the results. If dim of embeddings > 2, uses t-SNE to reduce it to 2. 25 | 26 | Parameters 27 | ---------- 28 | emb : matrix 29 | A Numpy matrix containing the node or edge embeddings. 30 | colors : array, optional 31 | A Numpy array containing the colors of each node. Default is None. 32 | filename : string, optional 33 | A string indicating the path and name of the file where to store the scatter plot. 34 | If not provided the plot is shown on screen. Default is None. 35 | """ 36 | print('Generating embedding scatter plot...') 37 | 38 | # Get the size of the embedding 39 | n, dim = emb.shape 40 | 41 | # If needed, reduce dimensionality to 2 using t-SNE 42 | if dim > 2: 43 | print("Embedding dimension is {}, using t-SNE to reduce it to 2.".format(dim)) 44 | emb = TSNE(n_components=2).fit_transform(emb) 45 | 46 | # Plot embeddings 47 | if colors is None: 48 | plt.scatter(emb[:, 0], emb[:, 1], alpha=0.6) 49 | else: 50 | plt.scatter(emb[:, 0], emb[:, 1], alpha=0.6, c=colors) 51 | 52 | # Store or show the scatter plot 53 | if filename is None: 54 | plt.show() 55 | else: 56 | plt.savefig(filename, dpi=300, format='pdf', bbox_inches='tight') 57 | 58 | 59 | def plot_graph2d(G, emb=None, labels=None, colors=None, filename=None): 60 | """ 61 | Plots the given graph in 2d. If the embeddings of nodes are provided, they are used to place the nodes on the 62 | 2d plane. If dim of embeddings > 2, then its reduced to 2 using t-SNE. Optional labels and colors for the nodes 63 | can be provided, as well as a filename to store the results. 64 | 65 | Parameters 66 | ---------- 67 | G : graph 68 | A NetworkX graph or digraph. 69 | emb : matrix, optional 70 | A Numpy matrix containing the node embeddings. Default is None. 71 | labels : dict, optional 72 | A dictionary containing nodeIDs as keys and node labels as values. Default is None. 73 | colors : array, optional 74 | A Numpy array containing the colors of each graph node. Default is None. 75 | filename : string, optional 76 | A string indicating the path and name of the file where to store the scatter plot. 77 | If not provided the plot is showed on screen. Default is None. 78 | """ 79 | print('Generating embedding visualization...') 80 | 81 | if emb is not None: 82 | # Get the size of the embedding 83 | n, dim = emb.shape 84 | 85 | # If needed, reduce dimensionality to 2 using t-SNE 86 | if dim > 2: 87 | print("Embedding dimension is {}, using t-SNE to reduce it to 2.".format(dim)) 88 | emb = TSNE(n_components=2).fit_transform(emb) 89 | else: 90 | # If no embeddings provided, use the spring layout to position nodes 91 | emb = nx.spring_layout(G) 92 | 93 | # Plot nodes and edges 94 | nx.draw_networkx_nodes(G, emb, node_size=100, alpha=0.6, node_color=colors) 95 | nx.draw_networkx_edges(G, emb, width=1.0, arrows=False, alpha=0.1) 96 | 97 | # Plot the labels if provided 98 | if labels is not None: 99 | nx.draw_networkx_labels(G, emb, labels=labels, font_size=6) 100 | 101 | # Store or show the scatter plot 102 | if filename is None: 103 | plt.show() 104 | else: 105 | plt.savefig(filename, dpi=300, format='pdf', bbox_inches='tight') 106 | 107 | 108 | def plot_curve(filename, x, y, x_label, y_label, title=None): 109 | """ 110 | Plots y coordinates against x coordinates as a line. 111 | 112 | Parameters 113 | ---------- 114 | filename : string 115 | A file or filename where to store the plot. 116 | x : array_like 117 | The x coordinates of the plot. 118 | y : array_like 119 | The y coordinates of the plot. 120 | x_label : string 121 | The label of the x axis. 122 | y_label : string 123 | The label of the y axis. 124 | title : string or None, optional 125 | The title of the plot. Default is None (no title). 126 | """ 127 | plt.plot(x, y) 128 | plt.xlabel(x_label) 129 | plt.ylabel(y_label) 130 | plt.ylim([0.0, 1.0]) 131 | plt.xlim([0.0, 1.0]) 132 | if title is not None: 133 | plt.title(title) 134 | if filename is not None: 135 | plt.savefig(filename) 136 | plt.close() 137 | else: 138 | plt.show() 139 | 140 | 141 | def parallel_coord(scoresheet, features, class_col='methods'): 142 | """ 143 | Generates a parallel coordinate plot from the given Scoresheet object and the set of features specified. 144 | 145 | Parameters 146 | ---------- 147 | scoresheet : evalne.Scoresheet 148 | A Scoresheet object containing the results of an evaluation. 149 | features : list 150 | A list of strings indicating the features to show in the plot (in addition to methods and networks). 151 | Accepted features are: 'auroc', 'average_precision', 'precision', 'recall', 152 | 'fallout', 'miss', 'accuracy', 'f_score', `eval_time` and `edge_embed_method`. 153 | class_col : string, optional 154 | Indicates the class to highlight. Options are `methods` and `networks`. Default is `methods`. 155 | """ 156 | # Get dfs per feature and stack them 157 | f_dfs = [] 158 | for f in features: 159 | f_dfs.append(scoresheet.get_pandas_df(metric=f).stack()) 160 | 161 | # Concatenate dfs and reset indexing 162 | df = pd.concat(f_dfs, axis=1, join="inner") 163 | df.reset_index(inplace=True) 164 | 165 | # Set correct column names 166 | new_names = ['methods_str', 'networks_str'] 167 | new_names.extend(features) 168 | df.set_axis(new_names, axis=1, inplace=True) 169 | 170 | # Make networks and methods numerical 171 | df['methods_str'] = pd.Categorical(df['methods_str']) 172 | df['methods'] = df['methods_str'].cat.codes 173 | df['methods'] = (df['methods'] - df['methods'].min()) / (df['methods'].max() - df['methods'].min()) 174 | df['networks_str'] = pd.Categorical(df['networks_str']) 175 | df['networks'] = df['networks_str'].cat.codes 176 | df['networks'] = (df['networks'] - df['networks'].min()) / (df['networks'].max() - df['networks'].min()) 177 | if 'edge_embed_method' in features: 178 | df['edge_embed_method'] = pd.Categorical(df['edge_embed_method']) 179 | df['edge_embed_method'] = df['edge_embed_method'].cat.codes # TODO: fix this 180 | df['edge_embed_method'] = (df['edge_embed_method'] - df['edge_embed_method'].min()) / \ 181 | (df['edge_embed_method'].max() - df['edge_embed_method'].min()) 182 | if 'eval_time' in features: 183 | df['eval_time'] = (df['eval_time'] - df['eval_time'].min()) / \ 184 | (df['eval_time'].max() - df['eval_time'].min()) 185 | 186 | # Select all numerical cols 187 | num = ['methods', 'networks'] 188 | num.extend(features) 189 | 190 | # Generate the plot 191 | pd.plotting.parallel_coordinates(df[num], class_col) 192 | ax = plt.gca() 193 | 194 | # Add labels 195 | if class_col == 'methods': 196 | for i, (label, val) in df.ix[:, ['networks_str', 'networks']].drop_duplicates().iterrows(): 197 | ax.annotate(label, xy=(0, val), ha='left', va='center') 198 | aux = df.ix[:, ['methods_str', 'methods']].drop_duplicates() 199 | plt.legend(aux['methods_str']) 200 | elif class_col == 'networks': 201 | for i, (label, val) in df.ix[:, ['methods_str', 'methods']].drop_duplicates().iterrows(): 202 | ax.annotate(label, xy=(0, val), ha='left', va='center') 203 | aux = df.ix[:, ['networks_str', 'networks']].drop_duplicates() 204 | plt.legend(aux['networks_str']) 205 | 206 | # Some changes to plot axis 207 | ax.yaxis.set_label_position("right") 208 | ax.yaxis.tick_right() 209 | plt.show() 210 | -------------------------------------------------------------------------------- /examples/api_examples/evaluator_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This example shows how to use the LPEvaluator class to analyse several algorithms on the input networks. 8 | # Takes raw graph as input, preprocesses it, computes tr/te splits and runs the algorithms on these splits. 9 | # In this case the parameters are provided by the user and no config file is used. 10 | # Similar evaluations can be run for network reconstruction and sign prediction by importing the appropriate classes. 11 | 12 | from __future__ import division 13 | 14 | import os 15 | import random 16 | 17 | import numpy as np 18 | 19 | from evalne.evaluation.evaluator import LPEvaluator 20 | from evalne.evaluation.score import Scoresheet 21 | from evalne.evaluation.split import LPEvalSplit 22 | from evalne.utils import preprocess as pp 23 | 24 | # NOTE: The example `as is`, only evaluates baseline methods. To evaluate the OpenNE methods, PRUNE and Metapath2vec 25 | # these must be first installed. Then the correct paths must be set in the commands_other variable. 26 | # Finally, the following parameter can be set to True. 27 | run_other_methods = False 28 | 29 | 30 | def main(): 31 | # Initialize some parameters 32 | inpath = list() 33 | nw_names = ['test_network', 'blogCatalog'] # Stores the names of the networks evaluated 34 | inpath.append("../../evalne/tests/data/network.edgelist") 35 | # inpath.append("../../../data/BlogCatalog/blog.edgelist") 36 | outpath = "./output/" 37 | if not os.path.exists(outpath): 38 | os.makedirs(outpath) 39 | directed = False # indicates if the graphs are directed or undirected 40 | delimiters = (',', '\t') # indicates the delimiter in the original graph 41 | repeats = 2 # number of time the experiment will be repeated 42 | 43 | # Create a scoresheet to store the results 44 | scoresheet = Scoresheet(tr_te='test') 45 | 46 | for i in range(len(inpath)): 47 | 48 | # Create folders for the evaluation results (one per input network) 49 | if not os.path.exists(outpath): 50 | os.makedirs(outpath) 51 | 52 | # Load and preprocess the graph 53 | G = preprocess(inpath[i], outpath, delimiters[i], directed) 54 | 55 | # For each repeat of the experiment generate new data splits 56 | for repeat in range(repeats): 57 | print('Repetition {} of experiment'.format(repeat)) 58 | 59 | # Generate one train/test split with default parameters 60 | traintest_split = LPEvalSplit() 61 | traintest_split.compute_splits(G, nw_name=nw_names[i], train_frac=0.8, split_id=repeat) 62 | 63 | trainvalid_split = LPEvalSplit() 64 | trainvalid_split.compute_splits(traintest_split.TG, nw_name=nw_names[i], train_frac=0.9, split_id=repeat) 65 | 66 | # Create an evaluator 67 | nee = LPEvaluator(traintest_split, trainvalid_split) 68 | 69 | # Evaluate baselines 70 | eval_baselines(nee, directed, scoresheet) 71 | 72 | # Evaluate other NE methods 73 | if run_other_methods: 74 | eval_other(nee, scoresheet) 75 | 76 | print("\nEvaluation results:") 77 | print("-------------------") 78 | 79 | # Print results averaged over exp repeats 80 | scoresheet.print_tabular(metric='auroc') 81 | 82 | # Write results averaged over exp repeats to a single file 83 | scoresheet.write_tabular(filename=os.path.join(outpath, 'eval_output.txt'), metric='auroc') 84 | 85 | # Store the Scoresheet object for later analysis 86 | scoresheet.write_pickle(os.path.join(outpath, 'eval.pkl')) 87 | 88 | print("Evaluation results are also stored in a folder named `output` in the current directory.") 89 | print("End of evaluation") 90 | 91 | 92 | def preprocess(inpath, outpath, delimiter, directed): 93 | """ 94 | Graph preprocessing routine. 95 | """ 96 | print('Preprocessing graph...') 97 | 98 | # Load a graph 99 | G = pp.load_graph(inpath, delimiter=delimiter, comments='#', directed=directed) 100 | 101 | # Preprocess the graph 102 | G, ids = pp.prep_graph(G, relabel=True, del_self_loops=True) 103 | 104 | # Store preprocessed graph to a file 105 | pp.save_graph(G, output_path=outpath + "prep_graph.edgelist", delimiter=',', write_stats=True) 106 | 107 | # Return the preprocessed graph 108 | return G 109 | 110 | 111 | def eval_baselines(nee, directed, scoresheet): 112 | """ 113 | Experiment to test the baselines. 114 | """ 115 | print('Evaluating baselines...') 116 | 117 | # Set the baselines 118 | methods = ['common_neighbours', 'jaccard_coefficient', 'cosine_similarity', 'lhn_index', 'topological_overlap', 119 | 'adamic_adar_index', 'resource_allocation_index', 'preferential_attachment', 'random_prediction', 120 | 'all_baselines'] 121 | 122 | # Evaluate baseline methods 123 | for method in methods: 124 | if directed: 125 | result = nee.evaluate_baseline(method=method, neighbourhood="in") 126 | scoresheet.log_results(result) 127 | result = nee.evaluate_baseline(method=method, neighbourhood="out") 128 | scoresheet.log_results(result) 129 | else: 130 | result = nee.evaluate_baseline(method=method) 131 | scoresheet.log_results(result) 132 | 133 | 134 | def eval_other(nee, scoresheet): 135 | """ 136 | Experiment to test other embedding methods not integrated in the library. 137 | """ 138 | print('Evaluating Embedding methods...') 139 | 140 | # Set edge embedding methods 141 | # Other options: 'weighted_l1', 'weighted_l2' 142 | edge_embedding_methods = ['average', 'hadamard'] 143 | 144 | # Evaluate non OpenNE method 145 | # ------------------------------- 146 | # Set the methods 147 | methods_other = ['PRUNE', 'metapath2vec++'] 148 | 149 | # Set the method types 150 | method_type = ['ne', 'ne'] 151 | 152 | # Set the commands 153 | commands_other = [ 154 | 'python ../../methods/PRUNE/src/main.py --inputgraph {} --output {} --dimension {}', 155 | '../../methods/metapath2vec/metapath2vec -min-count 1 -iter 20 -samples 100 -train {} -output {} -size {}'] 156 | 157 | # Set delimiters for the in and out files required by the methods 158 | input_delim = [' ', ' '] 159 | output_delim = [',', ' '] 160 | 161 | for i in range(len(methods_other)): 162 | # Evaluate the method 163 | results = nee.evaluate_cmd(method_name=methods_other[i], method_type=method_type[i], command=commands_other[i], 164 | edge_embedding_methods=edge_embedding_methods, 165 | input_delim=input_delim[i], output_delim=output_delim[i]) 166 | # Log the list of results 167 | scoresheet.log_results(results) 168 | 169 | # Evaluate methods from OpenNE 170 | # ---------------------------- 171 | # Set the methods 172 | methods = ['node2vec', 'deepwalk', 'line'] 173 | 174 | # Set the commands 175 | commands = [ 176 | 'python -m openne --method node2vec --graph-format edgelist --epochs 100 --number-walks 10', 177 | 'python -m openne --method deepWalk --graph-format edgelist --epochs 100 --number-walks 10 --walk-length 80', 178 | 'python -m openne --method line --graph-format edgelist --epochs 100'] 179 | 180 | # Set parameters to be tuned 181 | tune_params = ['--p 0.5 1 --q 0.5 1', None, None] 182 | 183 | # For each method evaluate 184 | for i in range(len(methods)): 185 | command = commands[i] + " --input {} --output {} --representation-size {}" 186 | results = nee.evaluate_cmd(method_name=methods[i], method_type='ne', command=command, 187 | edge_embedding_methods=edge_embedding_methods, input_delim=' ', output_delim=' ', 188 | tune_params=tune_params[i]) 189 | # Log the list of results 190 | scoresheet.log_results(results) 191 | 192 | 193 | if __name__ == "__main__": 194 | random.seed(42) 195 | np.random.seed(42) 196 | main() 197 | -------------------------------------------------------------------------------- /examples/api_examples/functions_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # Note: This is just an example of how to use some of the low level functions in EvalNE. 8 | # If possible the Evaluator and Split classes should be used as they simplify the pipeline. 9 | # Diagram of the evaluation: 10 | # preprocess_data -> split_train_test -> compute_node/edge_emb -> predict_edges -> evaluate_accuracy 11 | # preprocess_data -> split_train_test -> link_prediction -> evaluate_accuracy 12 | 13 | import os 14 | import random 15 | from time import time 16 | 17 | import networkx as nx 18 | import numpy as np 19 | 20 | from evalne.evaluation import score 21 | from evalne.methods import similarity as sim 22 | from evalne.utils import preprocess as pp 23 | from evalne.utils import split_train_test as stt 24 | 25 | ########################### 26 | # Variables 27 | ########################### 28 | 29 | 30 | dataset_path = "../../evalne/tests/data/network.edgelist" 31 | output_path = "./output/" 32 | directed = False 33 | random.seed(99) 34 | 35 | ########################### 36 | # Main 37 | ########################### 38 | 39 | # Time count 40 | start = time() 41 | 42 | # Create folders for the results if these do not exist 43 | if not os.path.exists(output_path): 44 | os.makedirs(output_path) 45 | 46 | traintest_path = os.path.join(output_path, "lp_train_test_splits") 47 | if not os.path.exists(traintest_path): 48 | os.makedirs(traintest_path) 49 | 50 | # --------------- 51 | # Preprocess data 52 | # --------------- 53 | 54 | # Load the data as a directed graph 55 | G = pp.load_graph(dataset_path, delimiter=",", comments='#', directed=directed) 56 | 57 | # Get some graph statistics 58 | pp.get_stats(G) 59 | 60 | # Or store them to a file 61 | pp.get_stats(G, os.path.join(output_path, "stats.txt")) 62 | 63 | # Preprocess the graph 64 | SG, ids = pp.prep_graph(G, relabel=True, del_self_loops=True) 65 | 66 | # Get non-edges so that the reversed edge exists in the graph 67 | if directed: 68 | redges = pp.get_redges_false(SG, output_path=os.path.join(output_path, "redges.csv")) 69 | 70 | # Store the graph to a file 71 | pp.save_graph(SG, output_path=os.path.join(output_path, "network_prep.edgelist"), delimiter=',', write_stats=True) 72 | 73 | # ---------------- 74 | # Split train test 75 | # ---------------- 76 | 77 | # Compute train/test splits and false edges in parallel 78 | stt.compute_splits_parallel(SG, os.path.join(traintest_path, "network_prep_51"), owa=True, 79 | train_frac=0.51, num_fe_train=None, num_fe_test=None, num_splits=5) 80 | 81 | # The overlap between the 5 generated sets can be easily checked 82 | print("Overlap check for train sets: ") 83 | stt.check_overlap(filename=os.path.join(traintest_path, "network_prep_51", "trE"), num_sets=5) 84 | print("Overlap check for test sets: ") 85 | stt.check_overlap(filename=os.path.join(traintest_path, "network_prep_51", "teE"), num_sets=5) 86 | 87 | # The same computations can be performed for the sets of non-edges 88 | # print "Overlap check for negative train sets: " 89 | # stt.check_overlap(filename=output_path + "lp_train_test_splits/network_prep_51_negTrE", num_sets=5) 90 | # print "Overlap check for negative test sets: " 91 | # stt.check_overlap(filename=output_path + "lp_train_test_splits/network_prep_51_negTeE", num_sets=5) 92 | 93 | # Alternatively, train/test splits can be computed one at a time 94 | train_E, test_E = stt.split_train_test(SG, train_frac=0.50) 95 | 96 | # Compute set of false edges 97 | # train_E_false, test_E_false = stt.generate_false_edges_owa(SG, train_E=train_E, test_E=test_E, num_fe_train=None, 98 | # num_fe_test=None) 99 | train_E_false, test_E_false = stt.generate_false_edges_cwa(SG, train_E=train_E, test_E=test_E, num_fe_train=None, 100 | num_fe_test=None) 101 | 102 | # Store the computed edge sets to a file 103 | filenames = stt.store_train_test_splits(os.path.join(output_path, "lp_train_test_splits", "network_prep_51"), 104 | train_E=train_E, train_E_false=train_E_false, test_E=test_E, 105 | test_E_false=test_E_false, split_id=0) 106 | 107 | # ------------------------------------------- 108 | # Link prediction (LP) using baseline methods 109 | # ------------------------------------------- 110 | 111 | # Create a graph using only the train edges 112 | if directed: 113 | TG = nx.DiGraph() 114 | else: 115 | TG = nx.Graph() 116 | TG.add_edges_from(train_E) 117 | 118 | # Stack the true and false edges together. 119 | train_edges = np.vstack((list(train_E), list(train_E_false))) 120 | test_edges = np.vstack((list(test_E), list(test_E_false))) 121 | 122 | # Create labels vectors with 1s for true edges and 0s for false edges 123 | train_labels = np.hstack((np.ones(len(train_E)), np.zeros(len(train_E_false)))) 124 | test_labels = np.hstack((np.ones(len(test_E)), np.zeros(len(test_E_false)))) 125 | 126 | # Run common neighbours and obtain the probability of links 127 | # Other methods are: jaccard_coefficient, adamic_adar_index, preferential_attachment, resource_allocation_index 128 | train_pred = np.array(sim.common_neighbours(TG, ebunch=train_edges, neighbourhood='in')) 129 | test_pred = np.array(sim.common_neighbours(TG, ebunch=test_edges, neighbourhood='in')) 130 | 131 | # ------------------------------- 132 | # Evaluate LP using Results class 133 | # ------------------------------- 134 | # The results class automatically binarizes the method predictions and provides useful functions for plotting and 135 | # outputting the method results. 136 | 137 | # Instantiate a Results class 138 | results = score.Results(method='CN', params={}, train_pred=train_pred, train_labels=train_labels, 139 | test_pred=test_pred, test_labels=test_labels) 140 | 141 | # Compute auroc 142 | train_auroc = results.train_scores.auroc() 143 | test_auroc = results.test_scores.auroc() 144 | 145 | # Visualize the results 146 | print("Train AUROC {}: {}".format("common neighbours", train_auroc)) 147 | print("Test AUROC {}: {}".format("common neighbours", test_auroc)) 148 | 149 | # Compute precision recall curves 150 | # Options are: pr, roc, all 151 | results.plot(filename=os.path.join(output_path, 'CN_train'), results='train', curve='all') 152 | results.plot(filename=os.path.join(output_path, 'CN_test'), results='test', curve='all') 153 | 154 | # ----------------------------- 155 | # Evaluate LP using Score class 156 | # ----------------------------- 157 | # Alternatively, the user can directly use the Scores class. This class acts as an interface providing different scores. 158 | # The Scores class requires the user to provide binary representations of the predictions in the initialization. 159 | 160 | # Binarize predictions using a simple threshold 161 | train_bin = np.where(train_pred >= 0.5, 1, 0) 162 | test_bin = np.where(test_pred >= 0.5, 1, 0) 163 | 164 | # Compute the area under the AUC curve 165 | train_scores = score.Scores(y_true=train_labels, y_pred=train_pred, y_bin=train_bin) 166 | test_scores = score.Scores(y_true=test_labels, y_pred=test_pred, y_bin=test_bin) 167 | 168 | # Compute auroc 169 | train_auroc = train_scores.auroc() 170 | test_auroc = test_scores.auroc() 171 | 172 | # Visualize the results 173 | print("Train AUROC {}: {}".format("common neighbours", train_auroc)) 174 | print("Test AUROC {}: {}".format("common neighbours", test_auroc)) 175 | 176 | # Get the execution time 177 | end = time()-start 178 | print("Processed in: {}".format(end)) 179 | -------------------------------------------------------------------------------- /examples/api_examples/output/CN_test_test_PR.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/examples/api_examples/output/CN_test_test_PR.pdf -------------------------------------------------------------------------------- /examples/api_examples/output/CN_test_test_ROC.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/examples/api_examples/output/CN_test_test_ROC.pdf -------------------------------------------------------------------------------- /examples/api_examples/output/CN_train_train_PR.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/examples/api_examples/output/CN_train_train_PR.pdf -------------------------------------------------------------------------------- /examples/api_examples/output/CN_train_train_ROC.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/examples/api_examples/output/CN_train_train_ROC.pdf -------------------------------------------------------------------------------- /examples/api_examples/output/eval.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dru-Mara/EvalNE/4c827e38771c5539642ecb50ff2d7f90a43acf9e/examples/api_examples/output/eval.pkl -------------------------------------------------------------------------------- /examples/api_examples/output/eval_output.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | Evaluation results (auroc): 4 | ----------------------- 5 | test_network 6 | common_neighbours 0.8034 7 | jaccard_coefficient 0.711 8 | cosine_similarity 0.7074 9 | lhn_index 0.6906 10 | topological_overlap 0.7312 11 | adamic_adar_index 0.8149 12 | resource_allocation_index 0.8146 13 | preferential_attachment 0.9344 14 | random_prediction 0.5194 15 | all_baselines 0.9345 16 | -------------------------------------------------------------------------------- /examples/api_examples/output/lp_train_test_splits/network_prep_51/teE_2.csv: -------------------------------------------------------------------------------- 1 | 439,605 2 | 8,647 3 | 497,546 4 | 465,68 5 | 127,189 6 | 615,272 7 | 486,127 8 | 45,12 9 | 134,615 10 | 270,359 11 | 108,68 12 | 575,357 13 | 429,123 14 | 72,114 15 | 137,131 16 | 357,363 17 | 456,405 18 | 188,182 19 | 83,654 20 | 672,411 21 | 348,579 22 | 85,434 23 | 373,492 24 | 147,265 25 | 45,254 26 | 492,565 27 | 497,680 28 | 85,61 29 | 579,224 30 | 114,124 31 | 488,451 32 | 68,502 33 | 227,675 34 | 108,166 35 | 185,678 36 | 147,157 37 | 45,146 38 | 575,46 39 | 329,399 40 | 615,527 41 | 45,411 42 | 607,34 43 | 83,114 44 | 127,300 45 | 465,444 46 | 497,513 47 | 85,159 48 | 465,300 49 | 497,369 50 | 227,508 51 | 615,504 52 | 163,609 53 | 654,352 54 | 357,245 55 | 270,326 56 | 1,201 57 | 163,645 58 | 163,236 59 | 391,243 60 | 227,629 61 | 48,254 62 | 213,587 63 | 227,256 64 | 497,611 65 | 188,41 66 | 41,516 67 | 45,486 68 | 492,388 69 | 432,257 70 | 227,233 71 | 85,149 72 | 627,60 73 | 227,498 74 | 394,495 75 | 654,342 76 | 575,543 77 | 598,181 78 | 85,41 79 | 289,185 80 | 45,499 81 | 497,71 82 | 329,523 83 | 679,492 84 | 432,391 85 | 497,457 86 | 270,594 87 | 45,67 88 | 330,465 89 | 254,97 90 | 679,469 91 | 329,127 92 | 45,188 93 | 497,614 94 | 330,357 95 | 465,136 96 | 330,213 97 | 85,525 98 | 188,129 99 | 212,486 100 | 45,80 101 | 83,192 102 | 212,77 103 | 163,72 104 | 459,479 105 | 147,621 106 | 504,221 107 | 654,574 108 | 487,397 109 | 387,459 110 | 465,149 111 | 147,68 112 | 357,552 113 | 149,523 114 | 85,358 115 | 465,270 116 | 647,247 117 | 227,478 118 | 575,487 119 | 357,673 120 | 497,604 121 | 163,435 122 | 663,492 123 | 83,182 124 | 227,455 125 | 527,487 126 | 412,357 127 | 645,479 128 | 227,311 129 | 680,702 130 | 85,227 131 | 615,572 132 | 85,492 133 | 147,58 134 | 198,140 135 | 45,47 136 | 439,352 137 | 227,23 138 | 572,472 139 | 330,157 140 | 163,425 141 | 9,367 142 | 329,457 143 | 497,41 144 | 291,201 145 | 157,350 146 | 694,60 147 | 227,674 148 | 436,439 149 | 72,391 150 | 148,157 151 | 163,402 152 | 497,656 153 | 694,181 154 | 212,178 155 | 212,672 156 | 663,50 157 | 364,391 158 | 163,258 159 | 330,435 160 | 227,687 161 | 348,568 162 | 647,492 163 | 227,134 164 | 654,616 165 | 85,50 166 | 144,136 167 | 1,465 168 | 469,570 169 | 45,508 170 | 140,22 171 | 83,391 172 | 465,227 173 | 335,346 174 | 227,255 175 | 198,457 176 | 679,272 177 | 311,254 178 | 291,47 179 | 87,397 180 | 163,127 181 | 157,196 182 | 456,492 183 | 357,221 184 | 225,367 185 | 357,486 186 | 432,400 187 | 85,472 188 | 72,502 189 | 227,147 190 | 563,218 191 | 254,286 192 | 679,658 193 | 45,112 194 | 147,388 195 | 227,268 196 | 198,61 197 | 85,449 198 | 227,124 199 | 45,4 200 | 227,389 201 | 85,305 202 | 613,690 203 | 227,654 204 | 163,346 205 | 357,211 206 | 45,619 207 | 391,389 208 | 497,227 209 | 32,456 210 | 72,492 211 | 488,178 212 | 111,191 213 | 270,364 214 | 497,492 215 | 680,613 216 | 465,279 217 | 85,174 218 | 291,279 219 | 497,384 220 | 227,523 221 | 85,439 222 | 227,114 223 | 83,515 224 | 497,649 225 | 9,157 226 | 45,79 227 | 188,164 228 | 45,344 229 | 83,227 230 | 148,356 231 | 198,293 232 | 45,200 233 | 357,466 234 | 45,465 235 | 598,677 236 | 163,228 237 | 227,621 238 | 374,640 239 | 658,435 240 | 85,393 241 | 227,68 242 | 13,145 243 | 694,357 244 | 680,459 245 | 357,443 246 | 651,661 247 | 136,215 248 | 145,300 249 | 329,646 250 | 9,147 251 | 227,598 252 | 663,262 253 | 188,118 254 | 515,247 255 | 257,397 256 | 658,412 257 | 435,483 258 | 575,414 259 | 677,282 260 | 411,46 261 | 497,580 262 | 357,191 263 | 140,342 264 | 575,270 265 | 615,571 266 | 45,455 267 | 497,472 268 | 267,275 269 | 575,391 270 | 357,577 271 | 497,328 272 | 85,383 273 | 497,593 274 | 497,449 275 | 204,196 276 | 70,232 277 | 329,492 278 | 465,236 279 | 227,444 280 | 329,83 281 | 85,131 282 | 565,439 283 | 45,180 284 | 225,147 285 | 658,523 286 | 497,691 287 | 504,586 288 | 492,203 289 | 85,373 290 | 267,350 291 | 57,540 292 | 45,157 293 | 497,583 294 | 584,551 295 | 163,149 296 | 492,59 297 | 405,356 298 | 198,106 299 | 26,408 300 | 188,98 301 | 212,46 302 | 45,314 303 | 300,275 304 | 272,442 305 | 679,680 306 | 227,699 307 | 575,479 308 | 147,181 309 | 198,492 310 | 594,492 311 | 455,154 312 | 615,551 313 | 275,488 314 | 256,209 315 | 497,272 316 | 225,258 317 | 57,265 318 | 432,62 319 | 677,504 320 | 212,553 321 | 515,325 322 | 163,548 323 | 492,458 324 | 227,159 325 | 497,694 326 | 94,389 327 | 181,457 328 | 387,562 329 | 575,204 330 | 256,487 331 | 587,178 332 | 575,469 333 | 163,381 334 | 364,635 335 | 679,647 336 | 108,157 337 | 23,378 338 | 45,137 339 | 497,154 340 | 225,189 341 | 94,523 342 | 330,391 343 | 497,10 344 | 198,580 345 | 504,543 346 | 598,699 347 | 163,250 348 | 497,504 349 | 388,597 350 | 282,350 351 | 148,270 352 | 85,559 353 | 694,523 354 | 163,515 355 | 45,114 356 | 163,106 357 | 83,118 358 | 227,620 359 | 147,282 360 | 287,442 361 | 497,108 362 | 45,356 363 | 85,163 364 | 45,127 365 | 527,508 366 | 163,348 367 | 270,65 368 | 587,675 369 | 654,585 370 | 198,305 371 | 654,356 372 | 105,528 373 | 213,61 374 | 405,411 375 | 163,240 376 | 497,494 377 | 416,420 378 | 497,85 379 | 188,153 380 | 436,483 381 | 492,680 382 | 330,358 383 | 45,225 384 | 364,350 385 | 598,666 386 | 270,343 387 | 497,471 388 | 363,391 389 | 45,490 390 | 198,439 391 | 291,487 392 | 127,415 393 | 356,387 394 | 227,93 395 | 188,22 396 | 267,660 397 | 227,358 398 | 0,87 399 | 140,66 400 | 45,94 401 | 330,492 402 | 227,479 403 | 163,580 404 | 93,664 405 | 330,348 406 | 412,502 407 | 45,71 408 | 330,469 409 | 497,88 410 | 201,175 411 | 267,120 412 | 87,598 413 | 575,272 414 | 487,388 415 | 478,568 416 | 465,369 417 | 1,679 418 | 45,48 419 | 157,289 420 | 357,543 421 | 300,683 422 | 515,491 423 | 329,72 424 | 198,406 425 | 85,120 426 | 527,236 427 | 497,330 428 | 181,502 429 | 45,169 430 | 75,457 431 | 225,136 432 | 289,300 433 | 465,117 434 | 198,118 435 | 256,388 436 | 497,42 437 | 267,254 438 | 188,110 439 | 227,181 440 | 20,117 441 | 181,214 442 | 68,44 443 | 68,309 444 | 329,579 445 | 607,178 446 | 497,163 447 | 198,504 448 | 198,275 449 | 45,182 450 | 329,615 451 | 85,483 452 | 305,523 453 | 594,360 454 | 157,508 455 | 276,236 456 | 212,300 457 | 357,389 458 | 469,630 459 | 227,459 460 | 85,375 461 | 267,123 462 | 357,654 463 | 257,487 464 | 575,504 465 | 619,659 466 | 163,416 467 | 198,108 468 | 359,62 469 | 300,421 470 | 163,272 471 | 227,436 472 | 45,136 473 | 227,701 474 | 46,546 475 | 615,288 476 | 45,401 477 | 680,683 478 | 85,208 479 | 575,72 480 | 497,9 481 | 227,148 482 | 188,77 483 | 198,350 484 | 679,574 485 | 166,318 486 | 83,140 487 | 598,469 488 | 227,413 489 | 701,598 490 | 85,329 491 | 267,77 492 | 85,221 493 | 575,350 494 | 287,441 495 | 198,363 496 | 45,270 497 | 83,382 498 | 672,368 499 | 456,627 500 | 1,83 501 | 615,242 502 | 679,672 503 | 497,552 504 | 658,469 505 | 83,9 506 | 587,445 507 | 575,62 508 | 497,637 509 | 487,479 510 | 198,196 511 | 46,513 512 | 147,415 513 | 677,352 514 | 188,44 515 | 22,649 516 | 198,353 517 | 497,506 518 | 227,645 519 | 481,561 520 | 227,236 521 | 329,140 522 | 411,587 523 | 188,57 524 | 198,330 525 | 598,678 526 | 497,254 527 | 188,142 528 | 4,92 529 | 109,523 530 | 227,658 531 | 127,391 532 | 9,701 533 | 357,624 534 | 166,239 535 | 69,342 536 | 117,239 537 | 9,148 538 | 465,427 539 | 227,635 540 | 85,551 541 | 148,82 542 | 45,335 543 | 108,391 544 | 575,680 545 | 618,233 546 | 227,82 547 | 329,624 548 | 45,636 549 | 227,118 550 | 45,227 551 | 465,175 552 | 227,612 553 | 443,502 554 | 329,251 555 | 45,492 556 | 267,47 557 | 45,83 558 | 432,492 559 | 621,279 560 | 497,329 561 | 330,72 562 | 497,594 563 | 227,504 564 | 658,282 565 | 492,515 566 | 85,276 567 | 225,436 568 | 694,469 569 | 157,386 570 | 318,340 571 | 256,279 572 | 267,145 573 | 680,692 574 | 178,445 575 | 45,181 576 | 198,45 577 | 497,607 578 | 607,478 579 | 112,127 580 | 163,438 581 | 85,518 582 | 658,645 583 | 163,330 584 | 70,102 585 | 94,459 586 | 270,541 587 | 227,579 588 | 357,545 589 | 4,134 590 | 291,227 591 | 465,492 592 | 85,387 593 | 429,197 594 | 254,165 595 | 85,508 596 | 575,372 597 | 694,701 598 | 487,259 599 | 598,504 600 | 497,574 601 | 145,680 602 | 198,506 603 | 196,582 604 | 291,325 605 | 1,406 606 | 291,37 607 | 227,281 608 | 436,46 609 | 198,254 610 | 492,472 611 | 658,504 612 | 523,459 613 | 240,256 614 | 615,133 615 | 497,672 616 | 3,694 617 | 50,397 618 | 330,680 619 | 465,459 620 | 163,274 621 | 497,528 622 | 188,187 623 | 672,645 624 | 459,272 625 | 572,478 626 | 147,414 627 | 497,155 628 | 93,214 629 | 227,559 630 | 492,570 631 | 11,382 632 | 654,259 633 | 465,472 634 | 608,637 635 | 148,127 636 | 514,221 637 | 654,344 638 | 45,416 639 | 11,274 640 | 157,68 641 | 497,24 642 | 487,468 643 | 358,503 644 | 212,40 645 | 227,657 646 | 201,147 647 | 157,598 648 | 163,120 649 | 497,639 650 | 227,549 651 | 458,302 652 | 127,282 653 | 694,658 654 | 546,680 655 | 357,515 656 | 647,439 657 | 225,72 658 | 527,293 659 | 291,282 660 | 492,272 661 | 497,387 662 | 26,127 663 | 358,457 664 | 364,387 665 | 72,99 666 | 465,439 667 | 486,233 668 | 497,508 669 | 329,515 670 | 206,444 671 | 227,9 672 | 45,118 673 | 527,270 674 | 497,364 675 | 188,23 676 | 357,469 677 | 0,497 678 | 465,151 679 | 85,46 680 | 267,23 681 | 329,672 682 | 227,624 683 | 166,385 684 | 572,435 685 | 387,389 686 | 249,684 687 | 680,498 688 | 391,215 689 | 497,233 690 | 45,481 691 | 85,288 692 | 364,606 693 | 256,664 694 | 497,498 695 | 188,157 696 | 45,108 697 | 658,680 698 | 163,329 699 | 329,397 700 | 163,185 701 | 270,576 702 | 465,406 703 | 411,435 704 | 227,614 705 | 188,134 706 | 45,579 707 | 492,661 708 | 67,109 709 | 85,157 710 | 45,615 711 | 449,290 712 | 1,163 713 | 497,187 714 | 575,515 715 | 45,206 716 | 465,383 717 | 163,198 718 | 9,140 719 | 481,507 720 | 267,255 721 | 188,111 722 | 615,178 723 | 615,443 724 | 465,504 725 | 45,592 726 | 198,276 727 | 61,572 728 | 492,350 729 | 188,124 730 | 198,662 731 | 615,456 732 | 382,254 733 | 527,492 734 | 65,318 735 | 265,343 736 | 556,492 737 | 45,196 738 | 679,333 739 | 465,373 740 | 163,188 741 | 497,33 742 | 227,172 743 | 213,274 744 | 679,598 745 | 22,68 746 | 507,491 747 | 85,124 748 | 72,154 749 | 465,265 750 | 227,702 751 | 188,487 752 | 256,621 753 | 497,46 754 | 679,22 755 | 267,114 756 | 227,41 757 | 45,150 758 | 497,576 759 | 508,60 760 | 45,415 761 | 357,272 762 | 329,439 763 | 492,582 764 | 330,669 765 | 45,307 766 | 469,189 767 | 497,553 768 | 227,692 769 | 45,657 770 | 188,68 771 | 679,565 772 | 163,420 773 | 225,215 774 | 488,445 775 | 330,8 776 | 163,276 777 | 213,133 778 | 45,140 779 | 565,435 780 | 188,225 781 | 406,267 782 | 674,692 783 | 267,225 784 | 487,457 785 | 85,68 786 | 289,621 787 | 227,646 788 | 594,439 789 | 163,518 790 | 45,117 791 | 127,595 792 | 497,543 793 | 275,350 794 | 468,336 795 | 85,189 796 | 198,66 797 | 57,127 798 | 486,124 799 | 275,435 800 | 499,588 801 | 654,611 802 | 45,238 803 | 45,9 804 | 607,270 805 | 270,356 806 | 374,593 807 | 45,274 808 | 397,433 809 | 527,17 810 | 329,298 811 | 330,263 812 | 497,147 813 | 45,395 814 | 157,456 815 | 137,69 816 | 85,58 817 | 291,163 818 | 357,337 819 | 497,268 820 | 85,552 821 | 330,11 822 | 45,107 823 | 457,689 824 | 357,373 825 | 1,329 826 | 497,618 827 | 295,254 828 | 85,444 829 | 654,601 830 | 492,539 831 | 144,350 832 | 117,253 833 | 456,356 834 | 227,649 835 | 329,288 836 | 108,140 837 | 357,615 838 | 546,551 839 | 163,112 840 | 654,578 841 | 157,446 842 | 1,198 843 | 163,233 844 | 465,9 845 | 72,343 846 | 83,618 847 | 291,238 848 | 181,515 849 | 497,114 850 | 382,276 851 | 598,574 852 | 270,300 853 | 654,591 854 | 291,130 855 | 227,603 856 | 188,123 857 | 72,91 858 | 497,500 859 | 227,639 860 | 181,492 861 | 497,585 862 | 357,196 863 | 300,565 864 | 201,214 865 | 85,411 866 | 137,49 867 | 83,78 868 | 497,212 869 | 575,540 870 | 468,414 871 | 45,231 872 | 607,492 873 | 658,574 874 | 465,408 875 | 227,616 876 | 492,627 877 | 227,472 878 | 85,388 879 | 497,598 880 | 546,374 881 | 198,530 882 | 388,282 883 | 47,397 884 | 631,683 885 | 607,60 886 | 87,61 887 | 497,45 888 | 254,58 889 | 227,449 890 | 148,305 891 | 140,72 892 | 405,492 893 | 357,415 894 | 227,305 895 | 112,504 896 | 465,133 897 | 85,486 898 | 587,504 899 | 45,41 900 | 497,58 901 | 465,663 902 | 45,77 903 | 465,254 904 | 227,691 905 | 69,254 906 | 188,391 907 | 85,378 908 | 497,588 909 | 227,318 910 | 134,127 911 | 187,282 912 | 607,459 913 | 148,439 914 | 523,581 915 | 645,463 916 | 580,599 917 | 157,645 918 | 45,175 919 | 163,432 920 | 391,439 921 | 163,23 922 | 256,214 923 | 414,279 924 | 225,492 925 | 198,654 926 | 575,461 927 | 45,152 928 | 598,364 929 | 134,117 930 | 157,478 931 | 497,663 932 | 204,181 933 | 615,389 934 | 329,32 935 | 270,391 936 | 329,68 937 | 465,486 938 | 256,492 939 | 291,41 940 | 198,487 941 | 615,546 942 | 169,200 943 | 614,587 944 | 311,140 945 | 329,45 946 | 679,652 947 | 487,73 948 | 357,372 949 | 457,459 950 | 188,191 951 | 148,118 952 | 163,543 953 | 654,106 954 | 85,443 955 | 267,191 956 | 239,272 957 | 267,227 958 | 254,257 959 | 225,230 960 | 140,680 961 | 148,504 962 | 357,614 963 | 357,385 964 | 198,68 965 | 492,286 966 | 408,472 967 | 72,257 968 | 492,551 969 | 85,47 970 | 157,72 971 | 497,257 972 | 335,78 973 | 457,449 974 | 22,112 975 | 188,181 976 | 680,643 977 | 112,42 978 | 227,517 979 | 267,181 980 | 492,528 981 | 465,165 982 | 198,575 983 | 85,289 984 | 405,416 985 | 497,499 986 | 85,554 987 | 212,106 988 | 465,286 989 | 227,494 990 | 227,265 991 | 289,60 992 | 198,58 993 | 378,254 994 | 585,504 995 | 387,488 996 | 504,515 997 | 87,492 998 | 157,62 999 | 497,247 1000 | 163,487 1001 | 465,443 1002 | 163,343 1003 | 391,350 1004 | 227,507 1005 | 329,375 1006 | 198,300 1007 | 227,363 1008 | 134,172 1009 | 198,565 1010 | 596,611 1011 | 85,544 1012 | 679,645 1013 | 198,192 1014 | 270,446 1015 | 465,276 1016 | 163,91 1017 | 72,610 1018 | 330,88 1019 | 85,400 1020 | 45,364 1021 | 227,75 1022 | 188,4 1023 | 198,313 1024 | 607,481 1025 | 85,27 1026 | 679,357 1027 | 227,605 1028 | 87,73 1029 | 227,196 1030 | 680,178 1031 | 163,68 1032 | 136,114 1033 | 465,24 1034 | 405,504 1035 | 497,587 1036 | 465,289 1037 | 85,413 1038 | 212,374 1039 | 71,106 1040 | 330,366 1041 | 306,646 1042 | 163,45 1043 | 267,282 1044 | 45,318 1045 | 45,89 1046 | 163,575 1047 | 198,267 1048 | 329,607 1049 | 45,210 1050 | 9,144 1051 | 497,456 1052 | 147,486 1053 | 188,115 1054 | 45,66 1055 | 147,342 1056 | 163,552 1057 | 131,254 1058 | 497,577 1059 | 597,664 1060 | 11,454 1061 | 45,187 1062 | 227,343 1063 | 443,233 1064 | 188,92 1065 | 9,615 1066 | 163,444 1067 | 198,136 1068 | 487,504 1069 | 23,549 1070 | 408,360 1071 | 163,300 1072 | 204,72 1073 | 663,357 1074 | 85,380 1075 | 213,157 1076 | 357,659 1077 | 378,224 1078 | 572,504 1079 | 497,181 1080 | 607,690 1081 | 615,136 1082 | 679,157 1083 | 85,501 1084 | 198,378 1085 | 45,56 1086 | 257,613 1087 | 436,435 1088 | 300,282 1089 | 551,538 1090 | 465,89 1091 | 198,270 1092 | 357,672 1093 | 365,267 1094 | 0,291 1095 | 87,259 1096 | 227,683 1097 | 47,78 1098 | 615,270 1099 | 382,68 1100 | 490,504 1101 | 227,539 1102 | 114,253 1103 | 9,353 1104 | 163,411 1105 | 4,9 1106 | 163,267 1107 | 8,658 1108 | 497,148 1109 | 615,512 1110 | 257,50 1111 | 478,543 1112 | 330,565 1113 | 227,552 1114 | 414,235 1115 | 598,464 1116 | 677,236 1117 | 654,481 1118 | 487,75 1119 | 645,661 1120 | 465,56 1121 | 329,577 1122 | 68,127 1123 | 329,433 1124 | 575,574 1125 | 227,385 1126 | 649,523 1127 | 497,17 1128 | 85,72 1129 | 329,289 1130 | 353,646 1131 | 654,494 1132 | 83,4 1133 | 45,386 1134 | 619,621 1135 | 163,378 1136 | 47,166 1137 | 357,508 1138 | 108,563 1139 | 227,663 1140 | 227,254 1141 | 45,363 1142 | 663,147 1143 | 45,134 1144 | 552,459 1145 | 198,47 1146 | 546,385 1147 | 497,380 1148 | 204,127 1149 | 45,628 1150 | 85,435 1151 | 330,388 1152 | 227,375 1153 | 658,598 1154 | 163,247 1155 | 204,248 1156 | 40,405 1157 | 198,204 1158 | 85,147 1159 | 575,11 1160 | 598,552 1161 | 497,357 1162 | 147,387 1163 | 85,412 1164 | 353,492 1165 | 527,528 1166 | 680,478 1167 | 198,60 1168 | 439,272 1169 | 456,504 1170 | 127,265 1171 | 9,575 1172 | 658,575 1173 | 618,539 1174 | 163,224 1175 | 497,478 1176 | 465,180 1177 | 329,256 1178 | 1,45 1179 | 188,137 1180 | 163,80 1181 | 598,300 1182 | 227,473 1183 | 45,353 1184 | 414,336 1185 | 497,599 1186 | 387,503 1187 | 679,611 1188 | 227,221 1189 | 188,150 1190 | 83,213 1191 | 227,486 1192 | 198,279 1193 | 465,134 1194 | 227,342 1195 | 481,487 1196 | 497,59 1197 | 575,387 1198 | 213,300 1199 | 188,127 1200 | 397,646 1201 | 523,605 1202 | 456,435 1203 | 85,379 1204 | 197,211 1205 | 45,199 1206 | 11,57 1207 | 465,147 1208 | 267,248 1209 | 45,55 1210 | 497,72 1211 | 188,140 1212 | 198,4 1213 | 672,683 1214 | 357,33 1215 | 61,565 1216 | 443,487 1217 | 57,680 1218 | 188,117 1219 | 504,582 1220 | 163,289 1221 | 85,369 1222 | 680,435 1223 | 329,357 1224 | 267,382 1225 | 487,385 1226 | 163,410 1227 | 136,456 1228 | 83,72 1229 | 227,574 1230 | 267,238 1231 | 227,165 1232 | 213,267 1233 | 198,367 1234 | 157,515 1235 | 300,680 1236 | 127,343 1237 | 497,291 1238 | 227,695 1239 | 45,575 1240 | 267,359 1241 | 198,259 1242 | 45,166 1243 | 357,661 1244 | 497,412 1245 | 45,22 1246 | 108,487 1247 | 148,443 1248 | 575,596 1249 | 694,696 1250 | 381,236 1251 | 227,443 1252 | 225,519 1253 | 374,233 1254 | 291,464 1255 | 198,236 1256 | 9,486 1257 | 594,236 1258 | 497,569 1259 | 391,142 1260 | 367,50 1261 | 181,597 1262 | 26,394 1263 | 163,256 1264 | 486,680 1265 | 45,300 1266 | 508,439 1267 | -------------------------------------------------------------------------------- /examples/api_examples/output/lp_train_test_splits/network_prep_51/teE_4.csv: -------------------------------------------------------------------------------- 1 | 439,605 2 | 227,11 3 | 41,415 4 | 227,276 5 | 615,272 6 | 694,565 7 | 239,250 8 | 486,127 9 | 45,12 10 | 270,359 11 | 429,123 12 | 357,363 13 | 497,523 14 | 182,282 15 | 456,405 16 | 188,182 17 | 227,24 18 | 83,654 19 | 357,399 20 | 348,579 21 | 227,289 22 | 157,459 23 | 147,265 24 | 497,680 25 | 85,61 26 | 579,224 27 | 68,502 28 | 227,675 29 | 108,166 30 | 465,58 31 | 701,680 32 | 45,146 33 | 615,527 34 | 45,411 35 | 157,472 36 | 16,204 37 | 127,300 38 | 497,513 39 | 45,123 40 | 227,508 41 | 188,28 42 | 159,230 43 | 357,245 44 | 157,40 45 | 1,201 46 | 330,148 47 | 227,364 48 | 163,645 49 | 136,282 50 | 227,629 51 | 85,545 52 | 497,526 53 | 227,256 54 | 560,587 55 | 149,687 56 | 41,516 57 | 357,78 58 | 282,313 59 | 492,388 60 | 432,257 61 | 497,94 62 | 267,306 63 | 188,162 64 | 227,233 65 | 85,149 66 | 291,254 67 | 329,546 68 | 627,60 69 | 227,498 70 | 394,495 71 | 654,571 72 | 367,249 73 | 654,342 74 | 157,439 75 | 575,543 76 | 85,41 77 | 356,504 78 | 289,185 79 | 577,588 80 | 329,523 81 | 497,372 82 | 432,391 83 | 270,594 84 | 45,67 85 | 330,465 86 | 330,236 87 | 85,548 88 | 658,181 89 | 679,469 90 | 523,630 91 | 680,699 92 | 497,614 93 | 465,136 94 | 330,213 95 | 225,276 96 | 83,192 97 | 212,77 98 | 163,72 99 | 314,204 100 | 459,479 101 | 167,191 102 | 647,270 103 | 654,574 104 | 487,397 105 | 429,47 106 | 147,68 107 | 149,523 108 | 356,98 109 | 85,358 110 | 465,270 111 | 147,598 112 | 575,487 113 | 225,145 114 | 406,40 115 | 575,523 116 | 198,127 117 | 318,458 118 | 85,515 119 | 136,72 120 | 267,263 121 | 387,328 122 | 41,594 123 | 227,455 124 | 85,227 125 | 85,492 126 | 67,215 127 | 198,140 128 | 45,47 129 | 439,352 130 | 45,541 131 | 497,149 132 | 357,254 133 | 330,157 134 | 163,425 135 | 488,630 136 | 497,270 137 | 147,300 138 | 497,41 139 | 291,201 140 | 227,674 141 | 72,391 142 | 181,563 143 | 163,402 144 | 694,181 145 | 212,178 146 | 232,272 147 | 663,50 148 | 330,435 149 | 598,478 150 | 492,433 151 | 375,442 152 | 372,553 153 | 227,134 154 | 85,50 155 | 469,570 156 | 45,508 157 | 163,500 158 | 465,227 159 | 227,664 160 | 198,457 161 | 679,272 162 | 311,254 163 | 87,397 164 | 157,196 165 | 357,221 166 | 147,411 167 | 497,646 168 | 432,400 169 | 1,213 170 | 575,565 171 | 227,147 172 | 679,393 173 | 497,502 174 | 615,637 175 | 254,286 176 | 607,553 177 | 679,658 178 | 497,538 179 | 227,268 180 | 329,136 181 | 45,148 182 | 198,61 183 | 85,449 184 | 45,4 185 | 227,389 186 | 85,305 187 | 497,515 188 | 227,654 189 | 187,438 190 | 227,101 191 | 163,611 192 | 391,389 193 | 140,627 194 | 497,227 195 | 72,492 196 | 598,687 197 | 267,295 198 | 680,613 199 | 45,102 200 | 445,566 201 | 227,523 202 | 330,356 203 | 85,439 204 | 227,114 205 | 497,649 206 | 9,157 207 | 83,227 208 | 188,20 209 | 198,293 210 | 357,466 211 | 0,85 212 | 163,228 213 | 227,621 214 | 374,640 215 | 663,514 216 | 85,537 217 | 227,212 218 | 658,435 219 | 465,40 220 | 227,68 221 | 13,145 222 | 680,459 223 | 357,443 224 | 497,603 225 | 136,215 226 | 329,646 227 | 497,459 228 | 227,598 229 | 615,594 230 | 481,514 231 | 188,118 232 | 679,615 233 | 515,247 234 | 257,397 235 | 62,387 236 | 435,483 237 | 411,46 238 | 492,236 239 | 465,282 240 | 357,191 241 | 227,490 242 | 329,358 243 | 140,342 244 | 575,270 245 | 330,359 246 | 227,575 247 | 615,571 248 | 45,455 249 | 497,472 250 | 227,611 251 | 348,492 252 | 267,275 253 | 357,577 254 | 45,82 255 | 654,275 256 | 497,328 257 | 45,576 258 | 497,593 259 | 497,449 260 | 204,196 261 | 85,504 262 | 188,108 263 | 144,181 264 | 227,444 265 | 134,253 266 | 329,83 267 | 85,131 268 | 565,439 269 | 227,300 270 | 225,147 271 | 497,691 272 | 329,469 273 | 504,586 274 | 492,203 275 | 85,373 276 | 267,350 277 | 45,157 278 | 497,583 279 | 584,551 280 | 147,204 281 | 198,106 282 | 497,439 283 | 227,578 284 | 26,408 285 | 461,336 286 | 188,98 287 | 9,212 288 | 212,46 289 | 227,434 290 | 300,275 291 | 272,442 292 | 198,227 293 | 492,445 294 | 357,400 295 | 147,181 296 | 198,492 297 | 594,492 298 | 140,142 299 | 85,206 300 | 163,391 301 | 551,387 302 | 575,335 303 | 45,26 304 | 497,272 305 | 225,258 306 | 677,504 307 | 515,325 308 | 213,140 309 | 497,658 310 | 227,159 311 | 212,674 312 | 497,694 313 | 85,75 314 | 94,389 315 | 181,457 316 | 677,481 317 | 528,272 318 | 575,204 319 | 256,487 320 | 227,689 321 | 575,469 322 | 163,381 323 | 157,486 324 | 364,635 325 | 330,270 326 | 45,137 327 | 497,154 328 | 45,631 329 | 85,438 330 | 378,282 331 | 225,189 332 | 330,391 333 | 381,436 334 | 45,258 335 | 504,543 336 | 598,699 337 | 505,508 338 | 497,275 339 | 388,597 340 | 282,350 341 | 85,559 342 | 694,523 343 | 163,515 344 | 45,114 345 | 163,106 346 | 363,280 347 | 85,415 348 | 45,379 349 | 492,546 350 | 83,118 351 | 287,442 352 | 163,83 353 | 497,108 354 | 45,356 355 | 85,163 356 | 45,127 357 | 270,65 358 | 166,502 359 | 654,585 360 | 198,305 361 | 654,356 362 | 492,523 363 | 497,494 364 | 416,420 365 | 85,549 366 | 497,85 367 | 188,153 368 | 436,483 369 | 497,206 370 | 45,225 371 | 598,666 372 | 270,343 373 | 363,391 374 | 267,274 375 | 204,254 376 | 147,357 377 | 198,439 378 | 45,346 379 | 300,78 380 | 291,487 381 | 127,415 382 | 497,363 383 | 356,387 384 | 227,93 385 | 188,22 386 | 465,150 387 | 227,358 388 | 0,87 389 | 267,22 390 | 330,227 391 | 45,94 392 | 504,608 393 | 227,479 394 | 45,359 395 | 93,664 396 | 329,383 397 | 330,348 398 | 465,127 399 | 157,11 400 | 270,562 401 | 188,120 402 | 412,502 403 | 85,107 404 | 45,71 405 | 330,469 406 | 134,265 407 | 436,221 408 | 468,290 409 | 201,175 410 | 267,120 411 | 497,582 412 | 575,272 413 | 487,388 414 | 541,508 415 | 85,228 416 | 478,568 417 | 1,679 418 | 546,479 419 | 300,683 420 | 497,65 421 | 329,72 422 | 198,406 423 | 150,515 424 | 527,236 425 | 163,305 426 | 181,502 427 | 329,108 428 | 227,60 429 | 45,169 430 | 225,136 431 | 227,325 432 | 44,469 433 | 198,118 434 | 497,42 435 | 267,254 436 | 227,181 437 | 147,72 438 | 20,117 439 | 412,492 440 | 68,44 441 | 492,457 442 | 329,579 443 | 607,178 444 | 497,163 445 | 92,114 446 | 329,615 447 | 357,268 448 | 305,523 449 | 594,360 450 | 679,584 451 | 157,508 452 | 198,131 453 | 276,236 454 | 212,300 455 | 357,389 456 | 227,459 457 | 429,185 458 | 267,123 459 | 357,654 460 | 45,159 461 | 257,487 462 | 270,506 463 | 357,16 464 | 163,416 465 | 359,62 466 | 227,580 467 | 8,663 468 | 329,340 469 | 46,546 470 | 615,288 471 | 45,401 472 | 85,208 473 | 10,443 474 | 575,72 475 | 227,148 476 | 188,77 477 | 198,350 478 | 645,581 479 | 585,387 480 | 83,140 481 | 227,413 482 | 267,77 483 | 256,476 484 | 227,40 485 | 80,515 486 | 432,149 487 | 85,221 488 | 163,635 489 | 163,406 490 | 9,348 491 | 497,251 492 | 575,350 493 | 287,441 494 | 581,659 495 | 198,363 496 | 83,382 497 | 148,511 498 | 456,627 499 | 1,83 500 | 47,50 501 | 225,502 502 | 615,242 503 | 679,672 504 | 227,17 505 | 163,527 506 | 560,492 507 | 497,552 508 | 83,9 509 | 587,445 510 | 575,62 511 | 497,637 512 | 188,67 513 | 487,479 514 | 198,196 515 | 330,272 516 | 46,513 517 | 227,259 518 | 497,385 519 | 147,415 520 | 677,352 521 | 166,514 522 | 22,649 523 | 456,352 524 | 198,353 525 | 497,506 526 | 357,117 527 | 481,561 528 | 227,236 529 | 147,127 530 | 357,611 531 | 85,152 532 | 227,272 533 | 329,140 534 | 497,627 535 | 188,57 536 | 198,330 537 | 465,414 538 | 497,254 539 | 45,502 540 | 4,92 541 | 45,93 542 | 109,523 543 | 127,391 544 | 9,701 545 | 654,587 546 | 69,342 547 | 9,148 548 | 329,9 549 | 188,155 550 | 148,82 551 | 457,279 552 | 227,82 553 | 357,457 554 | 329,624 555 | 108,247 556 | 227,118 557 | 694,227 558 | 465,175 559 | 227,612 560 | 227,203 561 | 267,47 562 | 621,279 563 | 432,492 564 | 497,329 565 | 415,439 566 | 330,72 567 | 497,594 568 | 227,504 569 | 658,282 570 | 4,432 571 | 492,515 572 | 85,276 573 | 32,270 574 | 694,469 575 | 519,388 576 | 267,289 577 | 607,357 578 | 157,386 579 | 256,279 580 | 163,582 581 | 45,181 582 | 198,45 583 | 497,607 584 | 198,539 585 | 254,211 586 | 112,127 587 | 163,438 588 | 85,518 589 | 658,645 590 | 198,660 591 | 267,122 592 | 163,330 593 | 270,541 594 | 391,422 595 | 148,435 596 | 22,66 597 | 198,408 598 | 225,282 599 | 4,134 600 | 291,227 601 | 254,165 602 | 45,436 603 | 486,178 604 | 227,592 605 | 45,701 606 | 85,508 607 | 694,701 608 | 487,259 609 | 598,504 610 | 134,486 611 | 93,597 612 | 289,508 613 | 405,82 614 | 163,140 615 | 145,680 616 | 198,506 617 | 45,184 618 | 196,582 619 | 291,325 620 | 22,429 621 | 1,406 622 | 227,569 623 | 388,523 624 | 382,363 625 | 502,279 626 | 163,32 627 | 575,614 628 | 497,551 629 | 532,243 630 | 615,686 631 | 291,37 632 | 227,281 633 | 436,46 634 | 198,254 635 | 492,472 636 | 658,504 637 | 504,446 638 | 523,459 639 | 227,317 640 | 240,256 641 | 615,133 642 | 497,443 643 | 227,402 644 | 329,41 645 | 50,397 646 | 330,680 647 | 465,459 648 | 497,528 649 | 672,645 650 | 459,272 651 | 679,46 652 | 497,155 653 | 238,309 654 | 11,382 655 | 654,259 656 | 465,472 657 | 457,468 658 | 608,637 659 | 163,372 660 | 45,416 661 | 454,435 662 | 607,677 663 | 11,274 664 | 108,701 665 | 654,645 666 | 313,581 667 | 85,79 668 | 497,518 669 | 201,147 670 | 157,598 671 | 163,120 672 | 185,287 673 | 458,302 674 | 127,282 675 | 198,342 676 | 546,680 677 | 148,490 678 | 607,546 679 | 647,439 680 | 672,612 681 | 72,387 682 | 291,282 683 | 551,587 684 | 26,127 685 | 4,225 686 | 358,457 687 | 364,387 688 | 72,99 689 | 486,233 690 | 497,508 691 | 227,647 692 | 85,563 693 | 227,238 694 | 45,118 695 | 527,270 696 | 374,702 697 | 497,364 698 | 227,503 699 | 300,344 700 | 227,94 701 | 188,23 702 | 87,465 703 | 85,275 704 | 85,46 705 | 329,672 706 | 17,177 707 | 694,504 708 | 198,188 709 | 188,180 710 | 387,389 711 | 249,684 712 | 212,357 713 | 329,649 714 | 391,215 715 | 465,164 716 | 45,481 717 | 112,391 718 | 256,664 719 | 188,157 720 | 45,108 721 | 658,680 722 | 128,371 723 | 163,185 724 | 270,576 725 | 465,406 726 | 411,435 727 | 227,614 728 | 45,494 729 | 276,247 730 | 188,134 731 | 198,178 732 | 196,254 733 | 45,579 734 | 515,528 735 | 492,661 736 | 67,109 737 | 85,157 738 | 45,615 739 | 497,187 740 | 94,62 741 | 342,313 742 | 487,402 743 | 9,140 744 | 481,507 745 | 188,111 746 | 465,504 747 | 575,672 748 | 45,592 749 | 188,3 750 | 198,276 751 | 497,609 752 | 198,541 753 | 267,124 754 | 527,492 755 | 198,253 756 | 556,492 757 | 679,333 758 | 163,188 759 | 204,189 760 | 227,172 761 | 213,274 762 | 679,598 763 | 225,248 764 | 85,353 765 | 85,124 766 | 72,154 767 | 497,563 768 | 465,265 769 | 227,702 770 | 253,202 771 | 163,574 772 | 188,487 773 | 188,78 774 | 147,40 775 | 679,575 776 | 85,510 777 | 497,46 778 | 197,571 779 | 157,391 780 | 497,576 781 | 45,415 782 | 198,99 783 | 497,23 784 | 85,78 785 | 198,629 786 | 45,307 787 | 497,553 788 | 134,92 789 | 227,54 790 | 83,46 791 | 45,657 792 | 300,389 793 | 188,68 794 | 679,565 795 | 163,11 796 | 497,265 797 | 654,657 798 | 488,445 799 | 330,8 800 | 85,356 801 | 159,391 802 | 45,140 803 | 188,225 804 | 382,319 805 | 406,267 806 | 267,225 807 | 487,457 808 | 227,646 809 | 163,518 810 | 127,595 811 | 497,543 812 | 330,286 813 | 300,523 814 | 468,336 815 | 57,127 816 | 127,680 817 | 45,238 818 | 45,9 819 | 607,270 820 | 374,593 821 | 436,388 822 | 397,433 823 | 527,17 824 | 85,575 825 | 679,676 826 | 85,166 827 | 83,507 828 | 137,69 829 | 608,328 830 | 83,363 831 | 357,337 832 | 497,268 833 | 148,492 834 | 45,107 835 | 357,373 836 | 552,203 837 | 1,329 838 | 227,492 839 | 683,543 840 | 497,618 841 | 497,389 842 | 85,444 843 | 654,601 844 | 497,654 845 | 456,356 846 | 267,48 847 | 373,358 848 | 227,649 849 | 329,288 850 | 108,140 851 | 357,615 852 | 45,120 853 | 163,112 854 | 157,446 855 | 330,374 856 | 1,198 857 | 163,233 858 | 329,265 859 | 83,618 860 | 45,97 861 | 291,238 862 | 497,114 863 | 267,146 864 | 497,608 865 | 598,574 866 | 270,300 867 | 627,574 868 | 163,619 869 | 85,254 870 | 163,210 871 | 227,603 872 | 188,123 873 | 357,340 874 | 497,500 875 | 188,159 876 | 85,411 877 | 137,49 878 | 575,540 879 | 468,414 880 | 45,231 881 | 607,492 882 | 373,325 883 | 658,574 884 | 465,408 885 | 227,616 886 | 85,532 887 | 188,136 888 | 405,479 889 | 343,275 890 | 527,504 891 | 497,598 892 | 388,282 893 | 497,45 894 | 188,113 895 | 198,157 896 | 357,559 897 | 227,449 898 | 148,305 899 | 46,294 900 | 405,492 901 | 497,575 902 | 631,166 903 | 465,133 904 | 85,257 905 | 45,41 906 | 575,615 907 | 254,300 908 | 45,77 909 | 465,254 910 | 227,691 911 | 69,254 912 | 85,378 913 | 198,255 914 | 78,110 915 | 497,588 916 | 134,127 917 | 187,282 918 | 45,427 919 | 607,459 920 | 527,350 921 | 497,444 922 | 267,247 923 | 198,147 924 | 523,581 925 | 497,300 926 | 68,37 927 | 645,463 928 | 45,584 929 | 157,645 930 | 45,175 931 | 188,80 932 | 236,645 933 | 198,124 934 | 391,439 935 | 256,214 936 | 73,358 937 | 414,279 938 | 225,492 939 | 575,461 940 | 497,578 941 | 598,364 942 | 134,117 943 | 45,417 944 | 204,181 945 | 329,32 946 | 45,44 947 | 405,207 948 | 127,342 949 | 575,618 950 | 329,68 951 | 256,492 952 | 291,41 953 | 663,587 954 | 465,342 955 | 497,411 956 | 572,325 957 | 614,587 958 | 311,140 959 | 329,45 960 | 300,247 961 | 487,73 962 | 148,118 963 | 163,543 964 | 654,106 965 | 239,272 966 | 157,59 967 | 201,282 968 | 198,356 969 | 254,257 970 | 225,230 971 | 270,381 972 | 163,255 973 | 140,680 974 | 575,608 975 | 198,212 976 | 198,68 977 | 497,630 978 | 492,286 979 | 615,127 980 | 492,551 981 | 85,47 982 | 157,72 983 | 163,3 984 | 45,276 985 | 66,70 986 | 166,386 987 | 457,449 988 | 329,300 989 | 22,112 990 | 680,643 991 | 357,627 992 | 112,42 993 | 267,181 994 | 198,575 995 | 85,289 996 | 405,416 997 | 497,499 998 | 83,136 999 | 306,293 1000 | 45,109 1001 | 212,106 1002 | 465,57 1003 | 330,98 1004 | 85,410 1005 | 289,60 1006 | 157,435 1007 | 585,504 1008 | 378,254 1009 | 87,227 1010 | 227,350 1011 | 504,515 1012 | 497,247 1013 | 163,487 1014 | 227,507 1015 | 198,300 1016 | 596,611 1017 | 85,544 1018 | 679,645 1019 | 198,192 1020 | 163,91 1021 | 329,123 1022 | 330,88 1023 | 140,107 1024 | 267,148 1025 | 227,75 1026 | 127,253 1027 | 607,481 1028 | 85,27 1029 | 679,357 1030 | 227,605 1031 | 87,73 1032 | 680,178 1033 | 163,68 1034 | 465,24 1035 | 187,425 1036 | 497,587 1037 | 46,571 1038 | 85,413 1039 | 45,197 1040 | 306,646 1041 | 497,479 1042 | 227,618 1043 | 56,62 1044 | 267,282 1045 | 45,318 1046 | 198,267 1047 | 519,502 1048 | 329,607 1049 | 381,388 1050 | 497,456 1051 | 147,486 1052 | 267,259 1053 | 188,115 1054 | 329,499 1055 | 45,560 1056 | 163,552 1057 | 131,254 1058 | 40,72 1059 | 680,60 1060 | 357,44 1061 | 443,233 1062 | 85,488 1063 | 188,92 1064 | 198,136 1065 | 492,354 1066 | 497,469 1067 | 408,360 1068 | 163,300 1069 | 204,72 1070 | 663,357 1071 | 45,164 1072 | 378,224 1073 | 572,504 1074 | 83,47 1075 | 497,181 1076 | 615,136 1077 | 497,675 1078 | 679,157 1079 | 198,378 1080 | 357,551 1081 | 257,613 1082 | 603,692 1083 | 85,357 1084 | 163,133 1085 | 465,89 1086 | 46,551 1087 | 365,267 1088 | 85,478 1089 | 188,82 1090 | 598,245 1091 | 47,487 1092 | 227,683 1093 | 147,574 1094 | 47,78 1095 | 188,203 1096 | 615,270 1097 | 270,501 1098 | 490,504 1099 | 227,539 1100 | 412,621 1101 | 607,680 1102 | 163,411 1103 | 4,9 1104 | 9,353 1105 | 8,537 1106 | 497,436 1107 | 329,443 1108 | 432,455 1109 | 163,267 1110 | 679,677 1111 | 357,626 1112 | 45,167 1113 | 157,457 1114 | 330,565 1115 | 227,552 1116 | 414,235 1117 | 677,236 1118 | 645,661 1119 | 465,56 1120 | 357,639 1121 | 329,577 1122 | 68,127 1123 | 227,385 1124 | 497,17 1125 | 387,523 1126 | 85,72 1127 | 465,442 1128 | 654,494 1129 | 356,391 1130 | 45,386 1131 | 578,620 1132 | 619,621 1133 | 357,508 1134 | 227,627 1135 | 148,483 1136 | 198,600 1137 | 386,391 1138 | 124,391 1139 | 227,663 1140 | 227,254 1141 | 663,147 1142 | 45,134 1143 | 552,459 1144 | 198,47 1145 | 492,265 1146 | 497,380 1147 | 204,127 1148 | 329,387 1149 | 45,628 1150 | 85,435 1151 | 85,26 1152 | 497,236 1153 | 227,375 1154 | 658,598 1155 | 163,247 1156 | 204,248 1157 | 231,265 1158 | 40,405 1159 | 136,113 1160 | 198,204 1161 | 85,147 1162 | 357,606 1163 | 163,332 1164 | 497,357 1165 | 85,412 1166 | 439,272 1167 | 127,265 1168 | 9,575 1169 | 658,575 1170 | 598,673 1171 | 618,539 1172 | 497,478 1173 | 465,180 1174 | 329,256 1175 | 188,137 1176 | 487,284 1177 | 163,80 1178 | 227,473 1179 | 649,611 1180 | 198,37 1181 | 497,599 1182 | 508,528 1183 | 497,635 1184 | 465,157 1185 | 147,256 1186 | 42,585 1187 | 45,65 1188 | 699,589 1189 | 227,221 1190 | 188,150 1191 | 227,486 1192 | 227,77 1193 | 198,279 1194 | 658,300 1195 | 636,690 1196 | 465,134 1197 | 382,136 1198 | 163,214 1199 | 497,468 1200 | 382,401 1201 | 408,539 1202 | 497,59 1203 | 575,387 1204 | 213,300 1205 | 188,127 1206 | 163,479 1207 | 456,435 1208 | 85,379 1209 | 85,150 1210 | 490,464 1211 | 34,190 1212 | 45,199 1213 | 11,57 1214 | 227,584 1215 | 45,55 1216 | 497,72 1217 | 87,582 1218 | 497,566 1219 | 492,487 1220 | 672,683 1221 | 357,33 1222 | 45,212 1223 | 61,565 1224 | 227,597 1225 | 57,680 1226 | 188,117 1227 | 227,453 1228 | 85,369 1229 | 45,333 1230 | 227,44 1231 | 270,680 1232 | 680,435 1233 | 329,357 1234 | 136,456 1235 | 227,574 1236 | 267,238 1237 | 227,165 1238 | 213,267 1239 | 198,367 1240 | 157,515 1241 | 109,475 1242 | 497,291 1243 | 364,435 1244 | 198,259 1245 | 45,166 1246 | 357,661 1247 | 465,343 1248 | 497,412 1249 | 157,492 1250 | 357,517 1251 | 45,22 1252 | 148,443 1253 | 694,696 1254 | 381,236 1255 | 85,94 1256 | 225,254 1257 | 227,443 1258 | 45,552 1259 | 374,233 1260 | 594,236 1261 | 9,486 1262 | 575,488 1263 | 198,357 1264 | 658,607 1265 | 163,256 1266 | 45,300 1267 | -------------------------------------------------------------------------------- /examples/api_examples/output/stats.txt: -------------------------------------------------------------------------------- 1 | # Undirected graph 2 | # Num. nodes: 703 3 | # Num. edges: 2584 4 | # Num. connected components: 1 5 | # Num. nodes in largest CC: 703 (100.0 % of total) 6 | # Num. edges in largest CC: 2584 (100.0 % of total) 7 | # Avg. node degree: 7.351351351351352 8 | # Num. degree 1 nodes: 184 9 | # Num. degree 2 nodes: 102 10 | # Num. self loops: 0 11 | -------------------------------------------------------------------------------- /examples/api_examples/simple-example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This simple example is the one presented in the README.md file. 8 | # Network reconstruction and sign prediction can be computed in the same manner by simply substituting LPEvaluator and 9 | # LPEvalSplit by NREvaluator and NREvalSplit or SPEvaluator and SPEvalSplit. 10 | 11 | from evalne.evaluation.evaluator import LPEvaluator 12 | from evalne.evaluation.score import Scoresheet 13 | from evalne.evaluation.split import LPEvalSplit 14 | from evalne.utils import preprocess as pp 15 | 16 | # Load and preprocess the network 17 | G = pp.load_graph('../../evalne/tests/data/network.edgelist') 18 | G, _ = pp.prep_graph(G) 19 | 20 | # Create an evaluator and generate train/test edge split 21 | traintest_split = LPEvalSplit() 22 | traintest_split.compute_splits(G) 23 | nee = LPEvaluator(traintest_split) 24 | 25 | # Create a Scoresheet to store the results 26 | scoresheet = Scoresheet() 27 | 28 | # Set the baselines 29 | methods = ['random_prediction', 'common_neighbours', 'jaccard_coefficient'] 30 | 31 | # Evaluate baselines 32 | for method in methods: 33 | result = nee.evaluate_baseline(method=method) 34 | scoresheet.log_results(result) 35 | 36 | try: 37 | # Check if OpenNE is installed 38 | import openne 39 | 40 | # Set embedding methods from OpenNE 41 | methods = ['node2vec', 'deepwalk', 'GraRep'] 42 | commands = [ 43 | 'python -m openne --method node2vec --graph-format edgelist --p 1 --q 1', 44 | 'python -m openne --method deepWalk --graph-format edgelist --number-walks 40', 45 | 'python -m openne --method grarep --graph-format edgelist --epochs 10'] 46 | edge_emb = ['average', 'hadamard'] 47 | 48 | # Evaluate embedding methods 49 | for i in range(len(methods)): 50 | command = commands[i] + " --input {} --output {} --representation-size {}" 51 | results = nee.evaluate_cmd(method_name=methods[i], method_type='ne', command=command, 52 | edge_embedding_methods=edge_emb, input_delim=' ', output_delim=' ') 53 | scoresheet.log_results(results) 54 | 55 | except ImportError: 56 | print("The OpenNE library is not installed. Reporting results only for the baselines...") 57 | pass 58 | 59 | # Get output 60 | scoresheet.print_tabular() 61 | -------------------------------------------------------------------------------- /examples/conf.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Example conf file with descriptions of each variable (also known as option) and the corresponding values accepted. 4 | 5 | # All entries are required. 6 | # If the variable is not needed it can be left blank (the = sign is required) or set to None 7 | # Options which are lists of elements must be separated by any type of white space or new line 8 | # All options in the NETWORKS section are required to have the same number of entries and in the same order 9 | # Relative paths are always taken w.r.t. the EvalNE main folder 10 | 11 | # A different conf file is required for each type of networks: directed and undirected. 12 | # Weighted graphs are not yet supported 13 | 14 | # The METHODS and TUNE_PARAMS options must be in the same order (first line in TUNE_PARAMETERS is assumed to refer 15 | # to the first method in the METHODS field) 16 | 17 | # Separators/delimiters/comments must be delimited by single quotes. 18 | # Tabs have to be provided as '\t' newlines as '\n' and white spaces as '\s' 19 | 20 | 21 | [GENERAL] 22 | # TASK: the task to evaluate i.e. link prediction (LP), sign prediction (SP), network reconstruction (NR) or node classification (NC) 23 | # Possible values: {'lp', 'sp', 'nr', 'nc'} 24 | TASK = lp 25 | 26 | # LP_NUM_EDGE_SPLITS: the number of repetitions of the experiment with different train/test edge splits 27 | # Required if task is 'lp', 'sp' or 'nr'. For 'nc' evaluation this value must be 1. 28 | LP_NUM_EDGE_SPLITS = 2 29 | 30 | # NC_NUM_NODE_SPLITS: the number of repetitions for NC experiments with different train/test node splits 31 | # Required if task is 'nc'. 32 | NC_NUM_NODE_SPLITS = 33 | 34 | # NC_NODE_FRACS: the fractions of train labels to use when evaluating NC. 35 | # Required if task is 'nc'. 36 | NC_NODE_FRACS = 0.1 0.5 0.9 37 | 38 | # NR_EDGE_SAMP_FRAC: the fraction of all possible node pairs to sample and compute precision@k for when evaluating NR. 39 | # Required if task is 'nr'. 40 | NR_EDGE_SAMP_FRAC = 0.1 41 | 42 | # EDGE_EMBEDDING_METHODS: the node-pair operators to use. 43 | # Possible values: {'average', 'hadamard', 'weighted_l1', 'weighted_l2'} 44 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 45 | 46 | # LP_MODEL: the binary classifier used to predict links from node-pair embeddings. 47 | # Possible values: Kewords such as 'LogisticRegression' or 'LogisticRegressionCV' 48 | # Possible values: Any sklearn classifier e.g.: 'sklearn.ensemble.ExtraTreesClassifier()' or 49 | # 'sklearn.svm.LinearSVC(C=1.0, kernel=’rbf’, degree=3)' 50 | LP_MODEL = LogisticRegressionCV 51 | 52 | # EMBED_DIM: the dimensions of the embedding. 53 | EMBED_DIM = 128 54 | 55 | # TIMEOUT: the maximum execution time in seconds (or None) for each method including hyperparameter tuning. 56 | TIMEOUT = 1800 57 | 58 | # VERBOSE: verbosity level of the execution. 59 | VERBOSE = True 60 | 61 | # SEED: the random seed to use in the experiments. If None or empty, system time will be used. 62 | # Possible values: {'', 'None', any_int} 63 | SEED = 42 64 | 65 | [NETWORKS] 66 | # NAMES: the names of the networks to be evaluated. 67 | NAMES = StudentDB FB 68 | 69 | # INPATHS: the paths to the files containing the networks. The system will check if these files exist before execution. 70 | INPATHS = /home/almara/Desktop/EvalNE-dev/code/data/StudentDB/studentdb.edgelist 71 | /home/almara/Desktop/EvalNE-dev/code/data/Facebook/facebook_combined.txt 72 | 73 | # DIRECTED: indicates if the networks are all directed or not. Applies to all networks evaluated at the same time. 74 | DIRECTED = False 75 | 76 | # SEPARATORS: indicates the separators used in the provided network files. Whitespaces required as: '\s' '\t' or '\n' 77 | SEPARATORS = ',' '\s' 78 | 79 | # COMMENTS: the characters indicating that a line is a comment in the network files. 80 | COMMENTS = '#' '#' 81 | 82 | # LABELPATHS: the path where the node label files can be found. 83 | # Required if task is 'nc'. 84 | LABELPATHS = 85 | 86 | [PREPROCESSING] 87 | # RELABEL: relabel or not the network nodes to 0...N (required for methods such as PRUNE) 88 | RELABEL = True 89 | 90 | # DEL_SELFLOOPS: delete or not self loops in the network. 91 | DEL_SELFLOOPS = True 92 | 93 | # SAVE_PREP_NW: indicates if the preprocessed graph should be stored or not. 94 | SAVE_PREP_NW = False 95 | 96 | # WRITE_STATS: write or not common graph statistics as header in the preprocessed network file. 97 | WRITE_STATS = True 98 | 99 | # DELIMITER: delimiter to be used when writing the preprocessed graphs to a files. 100 | DELIMITER = ',' 101 | 102 | [EDGESPLIT] 103 | # TRAINTEST_FRAC: fraction of total edges to use for training and validation. The rest should be used for testing. 104 | TRAINTEST_FRAC = 0.8 105 | 106 | # TRAINVALID_FRAC: fraction of train-validation edges to use for training. The rest should be used for validation. 107 | TRAINVALID_FRAC = 0.9 108 | 109 | # SPLIT_ALG: the algorithm to use for splitting edges in train/test, train/validation sets (except NR task). 110 | # Possible values: {'spanning_tree', 'random', 'naive', 'fast', 'timestamp'}. 111 | SPLIT_ALG = spanning_tree 112 | 113 | # OWA: determines if the open world (True) or the closed world assumption (False) for non-edges should be used. 114 | OWA = True 115 | 116 | # FE_RATIO: ratio of non-edges to edges for tr & te. The num_fe = fe_ratio * num_edges. 117 | FE_RATIO = 1 118 | 119 | [BASELINES] 120 | # LP_BASELINES: the link prediction heuristics to evaluate 121 | # Possible values: {'', 'random_prediction', 'common_neighbours', 'jaccard_coefficient', 'adamic_adar_index', 122 | # 'preferential_attachment', 'resource_allocation_index', 'cosine_similarity', 'lhn_index', 'topological_overlap', 123 | # 'katz', 'all_baselines'} 124 | LP_BASELINES = random_prediction 125 | common_neighbours 126 | jaccard_coefficient 127 | adamic_adar_index 128 | preferential_attachment 129 | resource_allocation_index 130 | katz 131 | 132 | # NEIGHBOURHOOD: for directed graphs indicates if the in or the out neighbourhood should be used. 133 | # Possible values: {'', 'in', 'out'} 134 | NEIGHBOURHOOD = in out 135 | 136 | [OPENNE METHODS] 137 | # NAMES_OPNE: the names of methods from OpenNE to be evaluated. In the same order as METHODS_OPNE. 138 | NAMES_OPNE = node2vec lap-opne hope-opne 139 | 140 | # METHODS_OPNE: the command line call to perform in order to evaluate each method. 141 | # Possible values for the '--method' parameter: {'node2vec', 'deepWalk', 'line', 'grarep', 'sdne', 'hope', 'lap', 'gf'} 142 | # Note 1: the --directed parameter for OpenNE is not required, EvalNE will automatically send the correct input edgelist 143 | # Note 2: the --graph-format does not need to be specified, EvalNE will set it automatically 144 | METHODS_OPNE = python -m openne --method node2vec --graph-format edgelist --epochs 100 --number-walks 10 --walk-length 80 --window-size 10 145 | python -m openne --method lap --epochs 100 146 | python -m openne --method hope --epochs 100 147 | # python -m openne --method gf --epochs 100 148 | # python -m openne --method sdne --epochs 100 --encoder-list [1024,128] --beta 5 --bs 500 149 | # python -m openne --method deepWalk --graph-format edgelist --epochs 100 --number-walks 10 --walk-length 80 --window-size 10 150 | # python -m openne --method line --graph-format edgelist --epochs 10 151 | # python -m openne --method grarep --epochs 100 152 | 153 | # TUNE_PARAMS_OPNE: the parameters of methods from OpenNE to be tuned by the library. 154 | # Example: tuning the p and q parameters for node2vec: --p 0.25 0.5 1 2 --q 0.25 0.5 1 2 155 | # Example: tuning the kstep parameter for grarep: --kstep 1 5 10 156 | # Note: The first line must contain the parameters for the first method, the second line those of the second methods, ... 157 | TUNE_PARAMS_OPNE = 158 | 159 | [OTHER METHODS] 160 | # NAMES_OTHER: the names of any other methods, not form OpenNE, to be evaluated. In the same order as METHODS_OTHER. 161 | NAMES_OTHER = wys verse mnmf struc2vec 162 | 163 | # EMBTYPE_OTHER: indicates the method's output types: node embeddings (ne), edge embeddings (ee) or node similarities (e2e) 164 | # Possible values: {'ne', 'ee', 'e2e'}. 165 | EMBTYPE_OTHER = ne ne ne ne 166 | 167 | # WRITE_WEIGHTS_OTHER: indicates if training graphs should be given as input to methods weighted (True) or unweighted (False). 168 | WRITE_WEIGHTS_OTHER = False False False False 169 | 170 | # WRITE_DIR_OTHER: indicates is training graphs should be given as input to methods with both edge dir. (True) or one (False). 171 | WRITE_DIR_OTHER = True True True True 172 | 173 | # METHODS_OTHER: the command line call to perform in order to evaluate each method. 174 | # Note: The string must contain the call to the method as it would be written in the command line. 175 | # For 'ne' methods placeholders (i.e. {}) need to be provided for the parameters: input network file, 176 | # output file and embedding dimensionality, precisely IN THIS ORDER. 177 | # For 'ee' methods with parameters: input network file, input train edgelist, input test edgelist, output 178 | # train embeddings, output test embeddings and embedding dimensionality, 6 placeholders (i.e. {}) need to 179 | # be provided, precisely IN THIS ORDER. 180 | # For methods with parameters: input network file, input edgelist, output embeddings, and embedding 181 | # dimensionality, 4 placeholders (i.e. {}) need to be provided, precisely IN THIS ORDER. 182 | # For 'e2e' methods with parameters: input network file, input train edgelist, input test edgelist, output 183 | # train predictions, output test predictions and embedding dimensionality, 6 placeholders (i.e. {}) need 184 | # to be provided, precisely IN THIS ORDER. 185 | # For methods with parameters: input network file, input edgelist, output predictions, and embedding 186 | # dimensionality, 4 placeholders (i.e. {}) need to be provided, precisely IN THIS ORDER. 187 | METHODS_OTHER = /home/almara/Desktop/EvalNE-dev/code/methods/wys/venv/bin/python3 /home/almara/Desktop/EvalNE-dev/code/methods/wys/src/main.py --edge-path {} --embedding-path {} --dimensions {} --attention-path /dev/null --epochs 200 --window-size 5 --learning-rate 0.01 --beta 0.5 --gamma 0.5 --num-of-walks 80 188 | python /home/almara/Desktop/EvalNE-dev/code/methods/verse/python/main.py --input {} --output {} --dimension {} --undirected --alpha 0.85 --nsamples 3 --threads 4 189 | /home/almara/Desktop/EvalNE-dev/code/methods/M-NMF-py/venv/bin/python /home/almara/Desktop/EvalNE-dev/code/methods/M-NMF-py/src/main.py --input {} --embedding-output {} --dimensions {} --assignment-output /dev/null --log-output /dev/null --cluster-mean-output /dev/null --dump-matrices False 190 | python /home/almara/Desktop/EvalNE-dev/code/methods/struc2vec/src/main.py --input {} --output {} --dimensions {} --num-walks 20 --walk-length 80 --window-size 5 --OPT1 True --OPT2 True --OPT3 True --until-layer 6 191 | 192 | # TUNE_PARAMS_OTHER: the parameters to be tuned by the library. 193 | # Example: tuning the number of negative samples for metapath2vec: -negative 1 5 10 194 | # Note: The first line must contain the parameters for the first method, the second line those of the second methods, ... 195 | TUNE_PARAMS_OTHER = 196 | 197 | # INPUT_DELIM_OTHER: delimiter for the input network expected by each of the methods. 198 | INPUT_DELIM_OTHER = ',' ',' ',' '\s' 199 | 200 | # OUTPUT_DELIM_OTHER: the delimiter used by each method in the output file (when writing node embeddings, edge embeddings or predictions) to files. 201 | OUTPUT_DELIM_OTHER = ',' ',' ',' '\s' 202 | 203 | [REPORT] 204 | # MAXIMIZE: the score to maximize when performing model validation. 205 | # Possible values for LP, SP and NR: {'auroc', 'f_score', 'precision', 'recall', 'accuracy', 'fallout', 'miss'} 206 | # Possible values for NC: {'f1_micro', 'f1_macro', 'f1_weighted'} 207 | MAXIMIZE = auroc 208 | 209 | # SCORES: the score to be reported in the output file. If '%(maximize)s' the output will be tabular Alg.\Network 210 | # Possible values: {'', '%(maximize)s', 'all'} 211 | # Note: tabular output is not available for mixes of directed and undirected networks. 212 | SCORES = %(maximize)s 213 | 214 | # CURVES: the curves to provide as output. 215 | # Possible values: {'', 'roc', 'pr', 'all'} 216 | CURVES = roc 217 | 218 | # PRECATK_VALS: the values of k for which to provide the precision at k 219 | PRECATK_VALS = 2 10 100 200 500 800 1000 10000 100000 220 | -------------------------------------------------------------------------------- /examples/replicated_setups/cne/conf_cne.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which replicates the experimental section of the CNE paper. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 3 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = hadamard 12 | LP_MODEL = LogisticRegressionCV 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 42 17 | 18 | [NETWORKS] 19 | NAMES = Facebook PPI ArXiv BlogCatalog Wikipedia StudentDB 20 | INPATHS = ../data/Facebook/facebook_combined.txt 21 | ../data/PPI/ppi.edgelist 22 | ../data/Astro-PH/CA-AstroPh.txt 23 | ../data/BlogCatalog/blog.edgelist 24 | ../data/Wiki/wiki.edgelist 25 | ../data/StudentDB/studentdb.edgelist 26 | DIRECTED = False 27 | SEPARATORS = '\s' ',' '\t' ',' ',' ',' 28 | COMMENTS = '#' '#' '#' '#' '#' '#' 29 | LABELPATHS = 30 | 31 | [PREPROCESSING] 32 | RELABEL = True 33 | DEL_SELFLOOPS = True 34 | SAVE_PREP_NW = False 35 | WRITE_STATS = True 36 | DELIMITER = ',' 37 | 38 | [EDGESPLIT] 39 | TRAINTEST_FRAC = 0.5 40 | TRAINVALID_FRAC = 0.9 41 | SPLIT_ALG = spanning_tree 42 | OWA = False 43 | FE_RATIO = 1 44 | 45 | [BASELINES] 46 | LP_BASELINES = common_neighbours 47 | jaccard_coefficient 48 | adamic_adar_index 49 | preferential_attachment 50 | NEIGHBOURHOOD = in out 51 | 52 | [OPENNE METHODS] 53 | NAMES_OPNE = 54 | # node2vec 55 | # deepWalk 56 | # line 57 | METHODS_OPNE = 58 | # python -m openne --method node2vec --epochs 100 --number-walks 10 --walk-length 80 --window-size 10 59 | # python -m openne --method deepWalk --epochs 100 --number-walks 10 --walk-length 80 --window-size 10 60 | # python -m openne --method line --epochs 100 61 | TUNE_PARAMS_OPNE = 62 | #--p 0.25 0.5 1 2 4 --q 0.25 0.5 1 2 4 63 | 64 | [OTHER METHODS] 65 | NAMES_OTHER = node2vec deepWalk line metapath2vec CNE_d8_uniform CNE_d8_degree CNE_d16_uniform CNE_d16_degree 66 | EMBTYPE_OTHER = ne ne ne ne e2e e2e e2e e2e 67 | WRITE_WEIGHTS_OTHER = False False True False False False False False 68 | WRITE_DIR_OTHER = False False True False True True True True 69 | # By default EvalNE autofills the dimension parameter. If a different values is needed, the same parameter can be passed 70 | # to the method again. The later value is the one used by the method. This hack only works for e2e methods 71 | METHODS_OTHER = python ../methods/node2vec/main.py --input {} --output {} --dimensions {} --walk-length 80 --num-walks 10 --window-size 10 --workers 4 72 | deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --walk-length 40 --window-size 5 --workers 1 73 | ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 1 74 | ../methods/metapath2vec/metapath2vec -min-count 1 -iter 500 -negative 10 -threads 1 -train {} -output {} -size {} 75 | python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 100 --dimension 8 --prior 'uniform' 76 | python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 100 --dimension 8 --prior 'degree' 77 | python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 100 --dimension 16 --prior 'uniform' 78 | python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 100 --dimension 16 --prior 'degree' 79 | TUNE_PARAMS_OTHER = --p 0.25 0.5 1 2 4 --q 0.25 0.5 1 2 4 80 | INPUT_DELIM_OTHER = '\s' '\s' '\s' '\s' ',' ',' ',' ',' 81 | OUTPUT_DELIM_OTHER = '\s' '\s' '\s' '\s' ',' ',' ',' ',' 82 | 83 | [REPORT] 84 | MAXIMIZE = auroc 85 | SCORES = all 86 | CURVES = 87 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/maxent/conf_maxent.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Configuration file for running the Maxent model and baselines on all small datasets 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 3 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegressionCV 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 42 17 | 18 | [NETWORKS] 19 | NAMES = StudentDB Facebook PPI Wikipedia GR-QC 20 | INPATHS = ../data/StudentDB/studentdb.edgelist 21 | ../data/Facebook/facebook_combined.txt 22 | ../data/PPI/ppi.edgelist 23 | ../data/Wiki/wiki.edgelist 24 | ../data/GR-QC/CA-GrQc.txt 25 | DIRECTED = False 26 | SEPARATORS = ',' '\s' ',' ',' '\t' 27 | COMMENTS = '#' '#' '#' '#' '#' 28 | LABELPATHS = 29 | 30 | [PREPROCESSING] 31 | RELABEL = True 32 | DEL_SELFLOOPS = True 33 | SAVE_PREP_NW = False 34 | WRITE_STATS = True 35 | DELIMITER = ',' 36 | 37 | [EDGESPLIT] 38 | TRAINTEST_FRAC = 0.5 39 | TRAINVALID_FRAC = 0.9 40 | SPLIT_ALG = spanning_tree 41 | OWA = False 42 | FE_RATIO = 1 43 | 44 | [BASELINES] 45 | LP_BASELINES = 46 | ;common_neighbours 47 | ; jaccard_coefficient 48 | ; adamic_adar_index 49 | ; preferential_attachment 50 | ; resource_allocation_index 51 | ; katz 0.01 52 | NEIGHBOURHOOD = 53 | 54 | [OPENNE METHODS] 55 | NAMES_OPNE = 56 | METHODS_OPNE = 57 | TUNE_PARAMS_OPNE = 58 | 59 | [OTHER METHODS] 60 | NAMES_OTHER = maxent_bin5 61 | #role2vec maxent_cn_pa_ra maxent_cn_pa_jc_ra 62 | #role2vec node2vec line deepWalk struc2vec SDNE AROPE maxent_bin5 maxent_bin100 CNE maxent_cn_pa maxent_cn_pa_jc_ra 63 | EMBTYPE_OTHER = e2e 64 | #ne e2e e2e 65 | #ne ne ne ne ne ne e2e e2e e2e e2e e2e e2e 66 | WRITE_WEIGHTS_OTHER = False 67 | #False False False 68 | #False False True False False True False False False False False False 69 | WRITE_DIR_OTHER = False 70 | #True False False 71 | #True True True True True True False False False False False False 72 | METHODS_OTHER = /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 5 [1,0] 73 | #../methods/role2vec-master/venv/bin/python3 ../methods/role2vec-master/src/main.py --graph-input {} --output {} --dimensions {} --workers 8 --walk-length 20 --walk-number 20 74 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 200 --prior 'CN' 'PA' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 75 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 200 --prior 'CN' 'PA' 'JC' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 76 | ; ../methods/node2vec/venv/bin/python ../methods/node2vec/main.py --input {} --output {} --dimensions {} --workers 8 --walk-length 20 --num-walks 20 77 | ; ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -threads 8 78 | ; ../methods/deepwalk/venv/bin/deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --workers 8 --walk-length 20 --number-walks 20 79 | ; ../methods/struc2vec/venv/bin/python ../methods/struc2vec/src/main.py --input {} --output {} --dimensions {} --OPT1 True --OPT2 True --OPT3 True --until-layer 6 --workers 8 --num-walks 20 --walk-length 20 80 | ; ../methods/GEM-master/venv/bin/python ../methods/GEM-master/main.py --input {} --output {} --dimension {} --method sdne --max_iter 5 --bs 500 81 | ; ../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 82 | ; /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 5 [1,0] 83 | ; /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 100 [1,0] 84 | ; ../methods/CNE/venv/bin/python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 500 --prior 'degree' --learning_rate 0.05 85 | ; python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 86 | ; python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'JC' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 87 | TUNE_PARAMS_OTHER = 88 | #--P 0.5 1 2 --Q 0.5 1 2 89 | ; --p 0.5 1 2 --q 0.5 1 2 90 | ; -negative 5 10 91 | ; --window-size 5 10 20 92 | ; --window-size 5 10 20 93 | ; --beta 2 5 10 --encoder-list [1024,512,] 94 | ; --weights [1,0,0,0] [0,1,0,0] [0,0,1,0] [0,0,0,1] [1,0.1,0.01,0.001] [1,0.5,0.05,0.005] 95 | INPUT_DELIM_OTHER = ',' 96 | #',' ',' ',' 97 | #',' '\s' '\s' '\s' '\s' '\s' ',' ',' ',' ',' ',' ',' 98 | OUTPUT_DELIM_OTHER = ',' 99 | #',' ',' ',' 100 | #',' '\s' '\s' '\s' '\s' ',' ',' ',' ',' ',' ',' ',' 101 | 102 | [REPORT] 103 | MAXIMIZE = auroc 104 | SCORES = %(maximize)s 105 | CURVES = all 106 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/maxent/conf_maxent2.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which showcases the difference in accuracy for metapath2vec when using different levels of parallelism. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 1 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegressionCV 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 111 17 | 18 | [NETWORKS] 19 | NAMES = Wikipedia 20 | #StudentDB Facebook PPI Wikipedia GR-QC 21 | INPATHS = ../data/Wiki/wiki.edgelist 22 | ;../data/StudentDB/studentdb.edgelist 23 | ; ../data/Facebook/facebook_combined.txt 24 | ; ../data/PPI/ppi.edgelist 25 | ; ../data/Wiki/wiki.edgelist 26 | ; ../data/GR-QC/CA-GrQc.txt 27 | DIRECTED = False 28 | SEPARATORS = ',' 29 | #',' '\s' ',' ',' '\t' 30 | COMMENTS = '#' 31 | #'#' '#' '#' '#' '#' 32 | LABELPATHS = 33 | 34 | [PREPROCESSING] 35 | RELABEL = True 36 | DEL_SELFLOOPS = True 37 | SAVE_PREP_NW = False 38 | WRITE_STATS = True 39 | DELIMITER = ',' 40 | 41 | [EDGESPLIT] 42 | TRAINTEST_FRAC = 0.8 43 | TRAINVALID_FRAC = 0.9 44 | SPLIT_ALG = spanning_tree 45 | OWA = False 46 | FE_RATIO = 1 47 | 48 | [BASELINES] 49 | LP_BASELINES = 50 | ;common_neighbours 51 | ; jaccard_coefficient 52 | ; adamic_adar_index 53 | ; preferential_attachment 54 | ; resource_allocation_index 55 | ; katz 0.01 56 | ; all_baselines 57 | NEIGHBOURHOOD = 58 | 59 | [OPENNE METHODS] 60 | NAMES_OPNE = 61 | METHODS_OPNE = 62 | TUNE_PARAMS_OPNE = 63 | 64 | [OTHER METHODS] 65 | NAMES_OTHER = maxent_cn_pa_ra 66 | #maxent_bin100a2 67 | #maxent_cn_pa_ra maxent_cn_pa_a3_ra 68 | #maxent_bin5 maxent_bin100 69 | #maxent_cn_pa_jc_ra 70 | #AROPE maxent_bin200 maxent_bin400 maxent_bin200a3 maxent_bin400a3 71 | EMBTYPE_OTHER = e2e 72 | #e2e 73 | #e2e e2e 74 | #e2e e2e 75 | #e2e e2e e2e e2e 76 | #e2e e2e 77 | WRITE_WEIGHTS_OTHER = False 78 | #False 79 | #False False 80 | #False False 81 | #False False False False 82 | #False False 83 | WRITE_DIR_OTHER = False 84 | #False 85 | #False False 86 | #False False 87 | #False False False False 88 | #False False 89 | METHODS_OTHER = python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 10000 --prior 'CN' 'PA' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 90 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 100 [1,0] 91 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 92 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'A3' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 93 | #/home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 5 [1,0.1] 94 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 100 [1,0.1] 95 | #../methods/CNE/venv/bin/python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 500 --prior 'degree' --learning_rate 0.05 96 | #python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'JC' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 97 | #../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 98 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 200 [1,0.1] 99 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 800 [1,0.1] 100 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 200 [1,0] 101 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 800 [1,0] 102 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 500 --prior 'CN' 'PA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 103 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 500 --prior 'CN' 'PA' 'JC' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 104 | TUNE_PARAMS_OTHER = 105 | #--weights [1,0,0,0] [0,1,0,0] [0,0,1,0] [0,0,0,1] [1,0.1,0.01,0.001] [1,0.5,0.05,0.005] 106 | INPUT_DELIM_OTHER = ',' 107 | #',' 108 | #',' ',' 109 | #',' ',' ',' ',' ',' 110 | #',' ',' 111 | OUTPUT_DELIM_OTHER = ',' 112 | #',' 113 | #',' ',' 114 | #',' ',' ',' ',' ',' 115 | #',' ',' 116 | 117 | [REPORT] 118 | MAXIMIZE = auroc 119 | SCORES = %(maximize)s 120 | CURVES = all 121 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/maxent/conf_maxentNR.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Maxent conf all methods and small datasets 4 | 5 | [GENERAL] 6 | TASK = nr 7 | LP_NUM_EDGE_SPLITS = 1 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 0.1 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegressionCV 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = False 16 | SEED = 42 17 | 18 | [NETWORKS] 19 | NAMES = PPI GR-QC StudentDB Facebook Wikipedia 20 | INPATHS = ../data/PPI/ppi.edgelist 21 | ../data/GR-QC/CA-GrQc.txt 22 | ../data/StudentDB/studentdb.edgelist 23 | ../data/Facebook/facebook_combined.txt 24 | ../data/Wiki/wiki.edgelist 25 | DIRECTED = False 26 | SEPARATORS = ',' '\t' ',' '\s' ',' 27 | COMMENTS = '#' '#' '#' '#' '#' 28 | LABELPATHS = 29 | 30 | [PREPROCESSING] 31 | RELABEL = True 32 | DEL_SELFLOOPS = True 33 | SAVE_PREP_NW = False 34 | WRITE_STATS = True 35 | DELIMITER = ',' 36 | 37 | [EDGESPLIT] 38 | TRAINTEST_FRAC = 0.8 39 | TRAINVALID_FRAC = 0.9 40 | SPLIT_ALG = spanning_tree 41 | OWA = False 42 | FE_RATIO = 1 43 | 44 | [BASELINES] 45 | LP_BASELINES = 46 | #common_neighbours 47 | # jaccard_coefficient 48 | # adamic_adar_index 49 | # preferential_attachment 50 | # resource_allocation_index 51 | # katz 0.01 52 | NEIGHBOURHOOD = 53 | 54 | [OPENNE METHODS] 55 | NAMES_OPNE = 56 | METHODS_OPNE = 57 | TUNE_PARAMS_OPNE = 58 | 59 | [OTHER METHODS] 60 | NAMES_OTHER = AROPE AROPEkatz maxent_cn_pa_jc_ra 61 | #node2vec line deepWalk struc2vec SDNE AROPE maxent_bin200a2a3 maxent_bin400a2a3 maxent_bin200a2 maxent_bin400a2 CNE maxent_cn_pa maxent_cn_pa_a3_ra 62 | EMBTYPE_OTHER = e2e e2e e2e 63 | #ne ne ne ne ne e2e e2e e2e e2e e2e e2e e2e e2e 64 | WRITE_WEIGHTS_OTHER = False False False 65 | #False True False False True False False False False False False False False 66 | WRITE_DIR_OTHER = False False False 67 | #True True True True True False False False False False False False False 68 | METHODS_OTHER = ../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 --weights [0,0.1,0,0.001] 69 | ../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 --weights [1,0.1,0.01,0.001] 70 | python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'JC' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 71 | #../methods/node2vec/venv/bin/python ../methods/node2vec/main.py --input {} --output {} --dimensions {} --workers 8 --walk-length 20 --num-walks 20 72 | # ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -threads 8 73 | # ../methods/deepwalk/venv/bin/deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --workers 8 --walk-length 20 --number-walks 20 74 | # ../methods/struc2vec/venv/bin/python ../methods/struc2vec/src/main.py --input {} --output {} --dimensions {} --OPT1 True --OPT2 True --OPT3 True --until-layer 6 --workers 8 --num-walks 20 --walk-length 20 75 | # ../methods/GEM-master/venv/bin/python ../methods/GEM-master/main.py --input {} --output {} --dimension {} --method sdne --max_iter 5 --bs 500 76 | # ../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 77 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 200 [1,0.1] 78 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 400 [1,0.1] 79 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 200 [1,0] 80 | # /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 400 [1,0] 81 | # ../methods/CNE/venv/bin/python ../methods/CNE/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --dimension 8 --epochs 500 --prior 'degree' --learning_rate 0.05 82 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 83 | # python /home/alexandru/Desktop/maxentcombined/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 1000 --prior 'CN' 'PA' 'A3' 'RA' --learning_rate 1.0 --optimizer 'newton' --memory 'quadratic' 84 | # matlab -nodisplay -nojvm -nosplash -nodesktop -r 'tg="{}"; tr_e="{}"; te_e="{}"; tr_pred="{}" ; te_pred="{}"; dim={}; epochs=5000; f="cn"; addpath("/home/alexandru/Desktop/maxent/"); try, run test(tg,tr_e, te_e, tr_pred, te_pred, dim, epochs, f), catch e, disp(getReport(e)), exit(1), end, exit(0);' 85 | TUNE_PARAMS_OTHER = 86 | ;--p 0.5 1 2 --q 0.5 1 2 87 | ; -negative 5 10 88 | ; --window-size 5 10 20 89 | ; --window-size 5 10 20 90 | ; --beta 2 5 10 --encoder-list [1024,512,] 91 | ; --weights [1,0,0,0] [0,1,0,0] [0,0,1,0] [0,0,0,1] [1,0.1,0.01,0.001] [1,0.5,0.05,0.005] 92 | INPUT_DELIM_OTHER = ',' ',' ',' 93 | #'\s' '\s' '\s' '\s' '\s' ',' ',' ',' ',' ',' ',' ',' ',' 94 | OUTPUT_DELIM_OTHER = ',' ',' ',' 95 | #'\s' '\s' '\s' '\s' ',' ',' ',' ',' ',' ',' ',' ',' ',' 96 | 97 | [REPORT] 98 | MAXIMIZE = auroc 99 | SCORES = %(maximize)s 100 | CURVES = all 101 | PRECATK_VALS = 1 2 10 100 200 500 800 1000 10000 100000 -------------------------------------------------------------------------------- /examples/replicated_setups/maxent/conf_maxentbig.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Configuration file for running the Maxent model and baselines on all small datasets 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 5 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegressionCV 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 111 17 | 18 | [NETWORKS] 19 | NAMES = YouTube DBLP BlogCatalog Flickr 20 | INPATHS = ../data/YouTube/data/edges.csv 21 | ../data/DBLP/com-dblp.ungraph.txt 22 | ../data/BlogCatalog/blog.edgelist 23 | ../data/Flickr/data/edges.csv 24 | DIRECTED = False 25 | SEPARATORS = ',' '\t' ',' ',' 26 | COMMENTS = '#' '#' '#' '#' 27 | LABELPATHS = 28 | 29 | [PREPROCESSING] 30 | RELABEL = True 31 | DEL_SELFLOOPS = True 32 | SAVE_PREP_NW = True 33 | WRITE_STATS = True 34 | DELIMITER = ',' 35 | 36 | [EDGESPLIT] 37 | TRAINTEST_FRAC = 0.8 38 | TRAINVALID_FRAC = 0.9 39 | SPLIT_ALG = fast 40 | OWA = False 41 | FE_RATIO = 1 42 | 43 | [BASELINES] 44 | LP_BASELINES = katz 0.01 45 | NEIGHBOURHOOD = 46 | 47 | [OPENNE METHODS] 48 | NAMES_OPNE = 49 | METHODS_OPNE = 50 | TUNE_PARAMS_OPNE = 51 | 52 | [OTHER METHODS] 53 | NAMES_OTHER = AROPE maxent_bin5 maxent_bin100 LINE 54 | EMBTYPE_OTHER = e2e e2e e2e ne 55 | WRITE_WEIGHTS_OTHER = False False False True 56 | WRITE_DIR_OTHER = False False False True 57 | METHODS_OTHER = ../methods/AROPE/venv/bin/python ../methods/AROPE/python/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --order 4 --weights [1.0,0.1,0.01,0.001] 58 | /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 5 [1,0] 59 | /home/alexandru/Desktop/maxentcombined/matlab/run_test.sh /usr/local/MATLAB/MATLAB_Runtime/v95 {} {} {} {} {} {} 100 [1,0] 60 | ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -threads 8 -negative 5 61 | TUNE_PARAMS_OTHER = 62 | INPUT_DELIM_OTHER = ',' ',' ',' '\s' 63 | OUTPUT_DELIM_OTHER = ',' ',' ',' '\s' 64 | 65 | [REPORT] 66 | MAXIMIZE = auroc 67 | SCORES = %(maximize)s 68 | CURVES = all 69 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/node2vec/conf_node2vec.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which replicates the experimental section of the node2vec paper. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 3 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | # Run once for each ee method: average hadamard weighted_l1 weighted_l2 12 | EDGE_EMBEDDING_METHODS = average 13 | LP_MODEL = LogisticRegression 14 | EMBED_DIM = 128 15 | TIMEOUT = None 16 | VERBOSE = True 17 | SEED = 42 18 | 19 | [NETWORKS] 20 | NAMES = Facebook PPI ArXiv 21 | INPATHS = ../../../data/Facebook/facebook_combined.txt 22 | ../../../data/PPI/ppi.edgelist 23 | ../../../data/Astro-PH/CA-AstroPh.txt 24 | DIRECTED = False 25 | SEPARATORS = '\s' ',' '\t' 26 | COMMENTS = '#' '#' '#' 27 | LABELPATHS = 28 | 29 | [PREPROCESSING] 30 | RELABEL = True 31 | DEL_SELFLOOPS = True 32 | SAVE_PREP_NW = False 33 | WRITE_STATS = True 34 | DELIMITER = ',' 35 | 36 | [EDGESPLIT] 37 | TRAINTEST_FRAC = 0.5 38 | TRAINVALID_FRAC = 0.9 39 | SPLIT_ALG = spanning_tree 40 | OWA = False 41 | FE_RATIO = 1 42 | 43 | [BASELINES] 44 | LP_BASELINES = common_neighbours 45 | jaccard_coefficient 46 | adamic_adar_index 47 | preferential_attachment 48 | NEIGHBOURHOOD = in out 49 | 50 | [OPENNE METHODS] 51 | NAMES_OPNE = 52 | METHODS_OPNE = 53 | TUNE_PARAMS_OPNE = 54 | 55 | [OTHER METHODS] 56 | NAMES_OTHER = node2vec deepWalk line 57 | EMBTYPE_OTHER = ne ne ne 58 | WRITE_WEIGHTS_OTHER = False False True 59 | WRITE_DIR_OTHER = True True True 60 | METHODS_OTHER = python ../../../methods/node2vec/main.py --input {} --output {} --dimensions {} --walk-length 80 --num-walks 10 --window-size 10 --workers 8 61 | python ../../../methods/node2vec/main.py --input {} --output {} --dimensions {} --walk-length 80 --num-walks 10 --window-size 10 --workers 8 --p 1 --q 1 62 | ../../../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 1 63 | TUNE_PARAMS_OTHER = --p 0.25 0.5 1 2 4 --q 0.25 0.5 1 2 4 64 | INPUT_DELIM_OTHER = '\s' '\s' '\s' 65 | OUTPUT_DELIM_OTHER = '\s' '\s' '\s' 66 | 67 | [REPORT] 68 | MAXIMIZE = auroc 69 | SCORES = all 70 | CURVES = 71 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/others/conf_cnev2.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file used to evaluate the CNEv2 implementation 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 2 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 0.1 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 12 | LP_MODEL = LogisticRegression 13 | EMBED_DIM = 128 14 | VERBOSE = False 15 | SEED = None 16 | 17 | [NETWORKS] 18 | NAMES = Facebook StudentDB 19 | INPATHS = ../data/Facebook/facebook_combined.txt 20 | ../data/StudentDB/studentdb.edgelist 21 | DIRECTED = False 22 | SEPARATORS = '\s' ',' 23 | COMMENTS = '#' '#' 24 | LABELPATHS = 25 | 26 | [PREPROCESSING] 27 | RELABEL = True 28 | DEL_SELFLOOPS = True 29 | SAVE_PREP_NW = False 30 | WRITE_STATS = True 31 | DELIMITER = ',' 32 | 33 | [EDGESPLIT] 34 | TRAINTEST_FRAC = 0.8 35 | TRAINVALID_FRAC = 0.9 36 | SPLIT_ALG = spanning_tree 37 | OWA = False 38 | FE_RATIO = 1 39 | 40 | [BASELINES] 41 | LP_BASELINES = common_neighbours 42 | jaccard_coefficient 43 | # adamic_adar_index 44 | # preferential_attachment 45 | NEIGHBOURHOOD = in out 46 | 47 | [OPENNE METHODS] 48 | NAMES_OPNE = deepWalk_OPNE line_OPNE 49 | METHODS_OPNE = python -m openne --method node2vec --number-walks 10 --walk-length 80 --window-size 10 --p 1.0 --q 1.0 --workers 8 50 | python -m openne --method deepWalk --number-walks 10 --walk-length 80 --window-size 10 --workers 8 51 | TUNE_PARAMS_OPNE = 52 | 53 | [OTHER METHODS] 54 | NAMES_OTHER = CNEv2_d8_degree CNE_d8_degree Prune 55 | #CNE_d8_uniform 56 | EMBTYPE_OTHER = e2e e2e ne 57 | WRITE_WEIGHTS_OTHER = False False False 58 | WRITE_DIR_OTHER = True True True 59 | # By default EvalNE autofills the dimension parameter. If a different values is needed, the same parameter can be passed 60 | # to the method again. The later value is the one used by the method. This hack only works for e2e methods 61 | METHODS_OTHER = python /home/alexandru/Desktop/CNE2/CNEv2/mainv2.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 10 --dimension 8 --prior 'degree' --optimizer adam --learning_rate 0.01 --s1 1 62 | python /home/alexandru/Desktop/CNE2/CNEv2/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 10 --dimension 8 --prior 'degree' --optimizer adam --learning_rate 0.01 --s1 1 63 | python ../methods/PRUNE/src/main.py --inputgraph {} --output {} --dimension {} 64 | # python ../methods/CNEv2/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 100 --dimension 8 --prior 'uniform' --optimizer adam 65 | # python ../methods/CNEv2/main.py --inputgraph {} --tr_e {} --te_e {} --tr_pred {} --te_pred {} --dimension {} --epochs 500 --dimension 8 --prior 'degree' --optimizer adam 66 | 67 | TUNE_PARAMS_OTHER = 68 | INPUT_DELIM_OTHER = ',' ',' '\s' 69 | OUTPUT_DELIM_OTHER = ',' ',' ',' 70 | 71 | [REPORT] 72 | MAXIMIZE = auroc 73 | SCORES = all 74 | CURVES = 75 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/others/conf_opennetest.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which compares the original implementations of DeepWalk, Node2vec and LINE with those available in OpenNE. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 5 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegression 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 0 17 | 18 | [NETWORKS] 19 | NAMES = Facebook PPI ArXiv BlogCatalog Wikipedia StudentDB 20 | #GR-QC 21 | INPATHS = ../data/Facebook/facebook_combined.txt 22 | ../data/PPI/ppi.edgelist 23 | ../data/Astro-PH/CA-AstroPh.txt 24 | ../data/BlogCatalog/blog.edgelist 25 | ../data/Wiki/wiki.edgelist 26 | ../data/StudentDB/studentdb.edgelist 27 | # ../data/GR-QC/CA-GrQc.txt 28 | DIRECTED = False 29 | #False 30 | SEPARATORS = '\s' ',' '\t' ',' ',' ',' 31 | #'\t' 32 | COMMENTS = '#' '#' '#' '#' '#' '#' 33 | #'#' 34 | LABELPATHS = 35 | 36 | [PREPROCESSING] 37 | RELABEL = True 38 | DEL_SELFLOOPS = True 39 | SAVE_PREP_NW = False 40 | WRITE_STATS = True 41 | DELIMITER = ',' 42 | 43 | [EDGESPLIT] 44 | TRAINTEST_FRAC = 0.5 45 | TRAINVALID_FRAC = 0.9 46 | SPLIT_ALG = spanning_tree 47 | OWA = True 48 | FE_RATIO = 1 49 | 50 | [BASELINES] 51 | LP_BASELINES = 52 | NEIGHBOURHOOD = 53 | 54 | [OPENNE METHODS] 55 | NAMES_OPNE = node2vec_OPNE deepWalk_OPNE line_OPNE 56 | METHODS_OPNE = python -m openne --method line --epochs 1 --negative-ratio 5 --order 2 --workers 8 57 | python -m openne --method node2vec --number-walks 10 --walk-length 80 --window-size 10 --p 1.0 --q 1.0 --workers 8 58 | python -m openne --method deepWalk --number-walks 10 --walk-length 40 --window-size 5 --workers 8 59 | TUNE_PARAMS_OPNE = 60 | 61 | [OTHER METHODS] 62 | NAMES_OTHER = line node2vec deepWalk 63 | EMBTYPE_OTHER = ne ne ne 64 | WRITE_WEIGHTS_OTHER = True False False 65 | WRITE_DIR_OTHER = True True True 66 | METHODS_OTHER = ../methods/LINE-master/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 8 67 | python ../methods/node2vec/main.py --input {} --output {} --dimensions {} --p 1.0 --q 1.0 --num-walks 10 --walk-length 80 --window-size 10 --workers 8 68 | deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --walk-length 40 --window-size 5 --workers 8 69 | TUNE_PARAMS_OTHER = 70 | INPUT_DELIM_OTHER = '\s' '\s' '\s' 71 | OUTPUT_DELIM_OTHER = '\s' '\s' '\s' 72 | 73 | [REPORT] 74 | MAXIMIZE = auroc 75 | SCORES = all 76 | CURVES = 77 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/others/conf_parTest.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which showcases the difference in accuracy for metapath2vec when using different levels of parallelism. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 5 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = average hadamard weighted_l1 weighted_l2 12 | LP_MODEL = LogisticRegression 13 | EMBED_DIM = 128 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = None 17 | 18 | [NETWORKS] 19 | NAMES = Facebook PPI ArXiv 20 | INPATHS = ../data/Facebook/facebook_combined.txt 21 | ../data/PPI/ppi.edgelist 22 | ../data/Astro-PH/CA-AstroPh.txt 23 | DIRECTED = False 24 | SEPARATORS = '\s' ',' '\t' 25 | COMMENTS = '#' '#' '#' 26 | LABELPATHS = 27 | 28 | [PREPROCESSING] 29 | RELABEL = True 30 | DEL_SELFLOOPS = True 31 | SAVE_PREP_NW = False 32 | WRITE_STATS = True 33 | DELIMITER = ',' 34 | 35 | [EDGESPLIT] 36 | TRAINTEST_FRAC = 0.5 37 | TRAINVALID_FRAC = 0.9 38 | SPLIT_ALG = spanning_tree 39 | OWA = True 40 | FE_RATIO = 1 41 | 42 | [BASELINES] 43 | LP_BASELINES = 44 | NEIGHBOURHOOD = 45 | 46 | [OPENNE METHODS] 47 | NAMES_OPNE = 48 | METHODS_OPNE = 49 | TUNE_PARAMS_OPNE = 50 | 51 | [OTHER METHODS] 52 | NAMES_OTHER = line_4t metapath2vec_4t line_8t metapath2vec_8t 53 | EMBTYPE_OTHER = ne ne ne ne 54 | WRITE_WEIGHTS_OTHER = True False True False 55 | WRITE_DIR_OTHER = True True True True 56 | METHODS_OTHER = ../methods/LINE-master/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 4 57 | ../methods/metapath2vec/metapath2vec -min-count 1 -iter 500 -negative 10 -threads 4 -train {} -output {} -size {} 58 | ../methods/LINE-master/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 8 59 | ../methods/metapath2vec/metapath2vec -min-count 1 -iter 500 -negative 10 -threads 8 -train {} -output {} -size {} 60 | TUNE_PARAMS_OTHER = 61 | INPUT_DELIM_OTHER = '\s' '\s' '\s' '\s' 62 | OUTPUT_DELIM_OTHER = '\s' '\s' '\s' '\s' 63 | 64 | [REPORT] 65 | MAXIMIZE = auroc 66 | SCORES = auroc 67 | CURVES = 68 | PRECATK_VALS = -------------------------------------------------------------------------------- /examples/replicated_setups/prune/conf_prune.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which replicates the experimental section of the PRUNE paper. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 3 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | # The EE method used in the paper is not mentioned, we will use: 12 | EDGE_EMBEDDING_METHODS = average hadamard 13 | LP_MODEL = LogisticRegression 14 | # The experiment repeats are not mentioned. We will use 3: 15 | EMBED_DIM = 128 16 | TIMEOUT = None 17 | VERBOSE = True 18 | SEED = 42 19 | 20 | [NETWORKS] 21 | NAMES = HepPh 22 | FbWallPost 23 | Webspam 24 | INPATHS = ../data/cit-HepPh/Cit-HepPh.txt 25 | ../data/Facebook-wallposts/prep_graph.edgelist 26 | ../data/Webspam/prep_graph.edgelist 27 | DIRECTED = True 28 | SEPARATORS = '\t' 29 | ',' 30 | ',' 31 | COMMENTS = '#' 32 | '#' 33 | '#' 34 | LABELPATHS = 35 | 36 | [PREPROCESSING] 37 | RELABEL = True 38 | DEL_SELFLOOPS = True 39 | SAVE_PREP_NW = False 40 | WRITE_STATS = True 41 | DELIMITER = ',' 42 | 43 | [EDGESPLIT] 44 | TRAINTEST_FRAC = 0.8 45 | TRAINVALID_FRAC = 0.9 46 | SPLIT_ALG = spanning_tree 47 | OWA = False 48 | FE_RATIO = 1 49 | 50 | [BASELINES] 51 | LP_BASELINES = 52 | NEIGHBOURHOOD = 53 | 54 | [OPENNE METHODS] 55 | NAMES_OPNE = sdne 56 | METHODS_OPNE = python -m openne --method sdne 57 | TUNE_PARAMS_OPNE = 58 | 59 | [OTHER METHODS] 60 | NAMES_OTHER = node2vec deepWalk line prune 61 | EMBTYPE_OTHER = ne ne ne ne 62 | WRITE_WEIGHTS_OTHER = False False True False 63 | WRITE_DIR_OTHER = False False True True 64 | METHODS_OTHER = python ../methods/node2vec/main.py --input {} --output {} --dimensions {} --workers 8 65 | deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --workers 1 66 | ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 100 -negative 5 -rho 0.025 -threads 8 67 | python ../methods/PRUNE/src/main.py --lamb 0.01 --epoch 50 --inputgraph {} --output {} --dimension {} 68 | TUNE_PARAMS_OTHER = --p 0.25 0.5 1 2 4 --q 0.25 0.5 1 2 4 69 | INPUT_DELIM_OTHER = '\s' '\s' '\s' '\s' 70 | OUTPUT_DELIM_OTHER = '\s' '\s' '\s' ',' 71 | 72 | [REPORT] 73 | MAXIMIZE = auroc 74 | SCORES = all 75 | CURVES = 76 | PRECATK_VALS = 77 | -------------------------------------------------------------------------------- /examples/replicated_setups/prune/prep_data_prune.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 18/12/2018 6 | 7 | # This code preprocessed the Facebook wall post and the Webspam datasets in order to produce edgelists 8 | # which can be then used to replicate the paper experiments using EvalNE. 9 | 10 | from __future__ import division 11 | 12 | import os 13 | from sys import argv 14 | 15 | import networkx as nx 16 | 17 | from evalne.utils import preprocess as pp 18 | 19 | 20 | def main(): 21 | # Check cmd args 22 | if len(argv) != 3: 23 | print("ERROR: wrong number of parameters") 24 | print("Usage: prep_data_prune.py ") 25 | exit(-1) 26 | 27 | # Extract the dataset names and paths 28 | fb_path, fb_name = os.path.split(argv[1]) 29 | ws_path, ws_name = os.path.split(argv[2]) 30 | 31 | # Preprocess FB graph 32 | G1 = prep_fb(argv[1]) 33 | 34 | # Store FB graph to a file 35 | pp.save_graph(G1, output_path=fb_path + "/prep_graph_slfloops.edgelist", delimiter=',', write_stats=True) 36 | 37 | # Preprocess WS graph 38 | G2 = prep_ws(argv[2]) 39 | 40 | # Store preprocessed graph to a file 41 | pp.save_graph(G2, output_path=ws_path + "/prep_graph_slfloops.edgelist", delimiter=',', write_stats=True) 42 | 43 | print("Preprocessing finished.") 44 | 45 | 46 | def prep_fb(inpath): 47 | """ 48 | Preprocess facebook wall post graph. 49 | """ 50 | # Load a graph 51 | G = pp.load_graph(inpath, delimiter='\t', comments='#', directed=True) 52 | 53 | # The FB graph is stores as destination, origin so needs to be reversed 54 | G = G.reverse() 55 | 56 | # Preprocess the graph 57 | G, ids = pp.prep_graph(G, relabel=True, del_self_loops=False) 58 | 59 | # Return the preprocessed graph 60 | return G 61 | 62 | 63 | def prep_ws(inpath): 64 | """ 65 | Preprocess web spam graph. 66 | """ 67 | # Create an empty digraph 68 | G = nx.DiGraph() 69 | 70 | # Read the file and create the graph 71 | src = 0 72 | f = open(inpath, 'r') 73 | for line in f: 74 | if src != 0: 75 | arr = line.split() 76 | for dst in arr: 77 | dst_id = int(dst.split(':')[0]) 78 | # We consider the graph unweighted 79 | G.add_edge(src, dst_id) 80 | src += 1 81 | # G.add_node(src-2) 82 | 83 | # Preprocess the graph 84 | G, ids = pp.prep_graph(G, relabel=True, del_self_loops=False) 85 | 86 | # Return the preprocessed graph 87 | return G 88 | 89 | 90 | if __name__ == "__main__": 91 | main() 92 | -------------------------------------------------------------------------------- /examples/replicated_setups/sdne/conf_sdne.ini: -------------------------------------------------------------------------------- 1 | ; conf.ini 2 | 3 | # Conf file which replicates the experimental section of the SDNE paper. 4 | 5 | [GENERAL] 6 | TASK = lp 7 | LP_NUM_EDGE_SPLITS = 3 8 | NC_NUM_NODE_SPLITS = 9 | NC_NODE_FRACS = 10 | NR_EDGE_SAMP_FRAC = 11 | EDGE_EMBEDDING_METHODS = hadamard 12 | LP_MODEL = LogisticRegression 13 | EMBED_DIM = 100 14 | TIMEOUT = None 15 | VERBOSE = True 16 | SEED = 42 17 | 18 | [NETWORKS] 19 | NAMES = GR-QC 20 | INPATHS = ../data/GR-QC/CA-GrQc.txt 21 | DIRECTED = True 22 | SEPARATORS = '\t' 23 | COMMENTS = '#' 24 | LABELPATHS = 25 | 26 | [PREPROCESSING] 27 | RELABEL = True 28 | DEL_SELFLOOPS = True 29 | SAVE_PREP_NW = False 30 | WRITE_STATS = True 31 | DELIMITER = ',' 32 | 33 | [EDGESPLIT] 34 | TRAINTEST_FRAC = 0.85 35 | TRAINVALID_FRAC = 0.9 36 | SPLIT_ALG = spanning_tree 37 | OWA = False 38 | FE_RATIO = 166.6 39 | 40 | [BASELINES] 41 | LP_BASELINES = common_neighbours 42 | NEIGHBOURHOOD = in out 43 | 44 | [OPENNE METHODS] 45 | NAMES_OPNE = SDNE 46 | GraRep 47 | LapEig 48 | METHODS_OPNE = python -m openne --method sdne --epochs 1 --encoder-list [5242,500,100] --directed 49 | python -m openne --method grarep --epochs 1 --kstep 5 --directed 50 | python -m openne --method lap --epochs 1 --directed 51 | TUNE_PARAMS_OPNE = 52 | 53 | [OTHER METHODS] 54 | NAMES_OTHER = deepWalk line 55 | EMBTYPE_OTHER = ne ne 56 | WRITE_WEIGHTS_OTHER = False True 57 | WRITE_DIR_OTHER = True True 58 | METHODS_OTHER = deepwalk --input {} --output {} --representation-size {} --format 'edgelist' --walk-length 40 --number-walks 40 --window-size 10 --workers 1 --undirected False 59 | ../methods/LINE/linux/line -train {} -output {} -size {} -order 2 -samples 1000 -negative 5 -rho 0.025 -threads 1 60 | TUNE_PARAMS_OTHER = 61 | INPUT_DELIM_OTHER = '\s' '\s' 62 | OUTPUT_DELIM_OTHER = '\s' '\s' 63 | 64 | [REPORT] 65 | MAXIMIZE = precision 66 | SCORES = all 67 | CURVES = 68 | PRECATK_VALS = 2 10 100 200 300 500 800 1000 10000 69 | 70 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | scikit-learn 3 | networkx 4 | scipy 5 | matplotlib 6 | pandas 7 | pyparsing 8 | tqdm 9 | kiwisolver 10 | joblib 11 | -------------------------------------------------------------------------------- /scripts/set_numpy_threads.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- coding: utf-8 -*- 3 | # Author: Mara Alexandru Cristian 4 | # Contact: alexandru.mara@ugent.be 5 | # Date: 03/01/2022 6 | # Howto: 7 | # 1) give exec permisions with: chmod +x ./set_numpy_threads.sh 8 | # 2) run script with: source ./set_numpy_threads.sh 9 | 10 | read -p 'Input maximum number of threads: ' threads 11 | export OMP_NUM_THREADS=$threads 12 | export OPENBLAS_NUM_THREADS=$threads 13 | export MKL_NUM_THREADS=$threads 14 | export VECLIB_MAXIMUM_THREADS=$threads 15 | export NUMEXPR_NUM_THREADS=$threads 16 | echo 'Maximum number of threads set!' 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Setup script for EvalNE. You can install the library globally using: 3 | 4 | python setup.py install 5 | 6 | Or for a single user with: 7 | 8 | python setup.py install --user 9 | """ 10 | 11 | from setuptools import setup, find_packages 12 | 13 | setup( 14 | name="evalne", 15 | version='0.4.0', 16 | url="https://github.com/Dru-Mara/EvalNE", 17 | license="MIT License", 18 | author="Alexandru Mara", 19 | author_email='alexandru.mara@ugent.be', 20 | description="Open Source Network Embedding Evaluation toolkit", 21 | long_description=open("./README.md").read(), 22 | long_description_content_type="text/markdown", 23 | keywords='evaluation embedding link-prediction sign-prediction node-classification network-reconstruction ' 24 | 'networks graphs visualization', 25 | packages=find_packages(), 26 | python_requires='>3.7', 27 | zip_safe=False, 28 | tests_require=["pytest", "pytest-cov"], 29 | install_requires=[ 30 | 'numpy', 31 | 'scikit-learn', 32 | 'networkx', 33 | 'scipy', 34 | 'matplotlib', 35 | 'pandas', 36 | 'pyparsing', 37 | 'tqdm', 38 | 'kiwisolver', 39 | 'joblib' 40 | ], 41 | classifiers=[ 42 | "Intended Audience :: Science/Research", 43 | "Intended Audience :: Developers", 44 | "Topic :: Scientific/Engineering :: Information Analysis", 45 | "Topic :: Software Development", 46 | "Programming Language :: Python :: 3", 47 | "License :: OSI Approved :: MIT License", 48 | "Natural Language :: English", 49 | "Operating System :: OS Independent", 50 | ] 51 | ) 52 | --------------------------------------------------------------------------------