├── .flake8
├── .gitignore
├── LICENSE
├── README.md
├── __init__.py
├── links_cluster.py
├── pylintrc
├── requirements.txt
└── tests
    └── test_links_cluster.py


/.flake8:
--------------------------------------------------------------------------------
1 | 
2 | [flake8]
3 | max_line_length = 99
4 | exclude=venv,.git


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | pip-wheel-metadata/
 24 | share/python-wheels/
 25 | *.egg-info/
 26 | .installed.cfg
 27 | *.egg
 28 | MANIFEST
 29 | 
 30 | # PyInstaller
 31 | #  Usually these files are written by a python script from a template
 32 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 33 | *.manifest
 34 | *.spec
 35 | 
 36 | # Installer logs
 37 | pip-log.txt
 38 | pip-delete-this-directory.txt
 39 | 
 40 | # Unit test / coverage reports
 41 | htmlcov/
 42 | .tox/
 43 | .nox/
 44 | .coverage
 45 | .coverage.*
 46 | .cache
 47 | nosetests.xml
 48 | coverage.xml
 49 | *.cover
 50 | *.py,cover
 51 | .hypothesis/
 52 | .pytest_cache/
 53 | cover/
 54 | 
 55 | # Translations
 56 | *.mo
 57 | *.pot
 58 | 
 59 | # Django stuff:
 60 | *.log
 61 | local_settings.py
 62 | db.sqlite3
 63 | db.sqlite3-journal
 64 | 
 65 | # Flask stuff:
 66 | instance/
 67 | .webassets-cache
 68 | 
 69 | # Scrapy stuff:
 70 | .scrapy
 71 | 
 72 | # Sphinx documentation
 73 | docs/_build/
 74 | 
 75 | # PyBuilder
 76 | target/
 77 | 
 78 | # Jupyter Notebook
 79 | .ipynb_checkpoints
 80 | 
 81 | # IPython
 82 | profile_default/
 83 | ipython_config.py
 84 | 
 85 | # pyenv
 86 | #   For a library or package, you might want to ignore these files since the code is
 87 | #   intended to run in multiple environments; otherwise, check them in:
 88 | # .python-version
 89 | 
 90 | # pipenv
 91 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 92 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 93 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 94 | #   install all needed dependencies.
 95 | #Pipfile.lock
 96 | 
 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 98 | __pypackages__/
 99 | 
100 | # Celery stuff
101 | celerybeat-schedule
102 | celerybeat.pid
103 | 
104 | # SageMath parsed files
105 | *.sage.py
106 | 
107 | # Environments
108 | .env
109 | .venv
110 | env/
111 | venv/
112 | ENV/
113 | env.bak/
114 | venv.bak/
115 | 
116 | # Spyder project settings
117 | .spyderproject
118 | .spyproject
119 | 
120 | # Rope project settings
121 | .ropeproject
122 | 
123 | # mkdocs documentation
124 | /site
125 | 
126 | # mypy
127 | .mypy_cache/
128 | .dmypy.json
129 | dmypy.json
130 | 
131 | # Pyre type checker
132 | .pyre/
133 | 
134 | # pytype static type analyzer
135 | .pytype/
136 | 
137 | .idea/


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 |                             Preamble
  9 | 
 10 |   The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 |   The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 |   When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 |   To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 |   For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 |   Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 |   For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 |   Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 |   Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 |   The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |                        TERMS AND CONDITIONS
 72 | 
 73 |   0. Definitions.
 74 | 
 75 |   "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 |   "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 |   "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 |   To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 |   A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 |   To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 |   To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 |   An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 |   1. Source Code.
113 | 
114 |   The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 |   A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 |   The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 |   The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 |   The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 |   The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 |   2. Basic Permissions.
155 | 
156 |   All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 |   You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 |   Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 |   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 |   No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 |   When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 |   4. Conveying Verbatim Copies.
196 | 
197 |   You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 |   You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 |   5. Conveying Modified Source Versions.
209 | 
210 |   You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 |     a) The work must carry prominent notices stating that you modified
215 |     it, and giving a relevant date.
216 | 
217 |     b) The work must carry prominent notices stating that it is
218 |     released under this License and any conditions added under section
219 |     7.  This requirement modifies the requirement in section 4 to
220 |     "keep intact all notices".
221 | 
222 |     c) You must license the entire work, as a whole, under this
223 |     License to anyone who comes into possession of a copy.  This
224 |     License will therefore apply, along with any applicable section 7
225 |     additional terms, to the whole of the work, and all its parts,
226 |     regardless of how they are packaged.  This License gives no
227 |     permission to license the work in any other way, but it does not
228 |     invalidate such permission if you have separately received it.
229 | 
230 |     d) If the work has interactive user interfaces, each must display
231 |     Appropriate Legal Notices; however, if the Program has interactive
232 |     interfaces that do not display Appropriate Legal Notices, your
233 |     work need not make them do so.
234 | 
235 |   A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 |   6. Conveying Non-Source Forms.
246 | 
247 |   You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 |     a) Convey the object code in, or embodied in, a physical product
253 |     (including a physical distribution medium), accompanied by the
254 |     Corresponding Source fixed on a durable physical medium
255 |     customarily used for software interchange.
256 | 
257 |     b) Convey the object code in, or embodied in, a physical product
258 |     (including a physical distribution medium), accompanied by a
259 |     written offer, valid for at least three years and valid for as
260 |     long as you offer spare parts or customer support for that product
261 |     model, to give anyone who possesses the object code either (1) a
262 |     copy of the Corresponding Source for all the software in the
263 |     product that is covered by this License, on a durable physical
264 |     medium customarily used for software interchange, for a price no
265 |     more than your reasonable cost of physically performing this
266 |     conveying of source, or (2) access to copy the
267 |     Corresponding Source from a network server at no charge.
268 | 
269 |     c) Convey individual copies of the object code with a copy of the
270 |     written offer to provide the Corresponding Source.  This
271 |     alternative is allowed only occasionally and noncommercially, and
272 |     only if you received the object code with such an offer, in accord
273 |     with subsection 6b.
274 | 
275 |     d) Convey the object code by offering access from a designated
276 |     place (gratis or for a charge), and offer equivalent access to the
277 |     Corresponding Source in the same way through the same place at no
278 |     further charge.  You need not require recipients to copy the
279 |     Corresponding Source along with the object code.  If the place to
280 |     copy the object code is a network server, the Corresponding Source
281 |     may be on a different server (operated by you or a third party)
282 |     that supports equivalent copying facilities, provided you maintain
283 |     clear directions next to the object code saying where to find the
284 |     Corresponding Source.  Regardless of what server hosts the
285 |     Corresponding Source, you remain obligated to ensure that it is
286 |     available for as long as needed to satisfy these requirements.
287 | 
288 |     e) Convey the object code using peer-to-peer transmission, provided
289 |     you inform other peers where the object code and Corresponding
290 |     Source of the work are being offered to the general public at no
291 |     charge under subsection 6d.
292 | 
293 |   A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 |   A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 |   "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 |   If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 |   The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 |   Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 |   7. Additional Terms.
344 | 
345 |   "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 |   When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 |   Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 |     a) Disclaiming warranty or limiting liability differently from the
366 |     terms of sections 15 and 16 of this License; or
367 | 
368 |     b) Requiring preservation of specified reasonable legal notices or
369 |     author attributions in that material or in the Appropriate Legal
370 |     Notices displayed by works containing it; or
371 | 
372 |     c) Prohibiting misrepresentation of the origin of that material, or
373 |     requiring that modified versions of such material be marked in
374 |     reasonable ways as different from the original version; or
375 | 
376 |     d) Limiting the use for publicity purposes of names of licensors or
377 |     authors of the material; or
378 | 
379 |     e) Declining to grant rights under trademark law for use of some
380 |     trade names, trademarks, or service marks; or
381 | 
382 |     f) Requiring indemnification of licensors and authors of that
383 |     material by anyone who conveys the material (or modified versions of
384 |     it) with contractual assumptions of liability to the recipient, for
385 |     any liability that these contractual assumptions directly impose on
386 |     those licensors and authors.
387 | 
388 |   All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 |   If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 |   Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 |   8. Termination.
408 | 
409 |   You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 |   However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 |   Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 |   Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 |   9. Acceptance Not Required for Having Copies.
436 | 
437 |   You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 |   10. Automatic Licensing of Downstream Recipients.
447 | 
448 |   Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 |   An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 |   You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 |   11. Patents.
472 | 
473 |   A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 |   A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 |   Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 |   In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 |   If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 |   If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 |   A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 |   Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 |   12. No Surrender of Others' Freedom.
541 | 
542 |   If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 |   13. Use with the GNU Affero General Public License.
553 | 
554 |   Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 |   14. Revised Versions of this License.
564 | 
565 |   The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 |   Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 |   If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 |   Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 |   15. Disclaimer of Warranty.
590 | 
591 |   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 |   16. Limitation of Liability.
601 | 
602 |   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 |   17. Interpretation of Sections 15 and 16.
613 | 
614 |   If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |                      END OF TERMS AND CONDITIONS
622 | 
623 |             How to Apply These Terms to Your New Programs
624 | 
625 |   If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 |   To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 |     <one line to give the program's name and a brief idea of what it does.>
635 |     Copyright (C) <year>  <name of author>
636 | 
637 |     This program is free software: you can redistribute it and/or modify
638 |     it under the terms of the GNU General Public License as published by
639 |     the Free Software Foundation, either version 3 of the License, or
640 |     (at your option) any later version.
641 | 
642 |     This program is distributed in the hope that it will be useful,
643 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
644 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 |     GNU General Public License for more details.
646 | 
647 |     You should have received a copy of the GNU General Public License
648 |     along with this program.  If not, see <https://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 |   If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 |     <program>  Copyright (C) <year>  <name of author>
656 |     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 |     This is free software, and you are welcome to redistribute it
658 |     under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 |   You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <https://www.gnu.org/licenses/>.
668 | 
669 |   The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <https://www.gnu.org/licenses/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Links Online Clustering
 2 | 
 3 | Python implementation of the Links Online Clustering algorithm: 
 4 | https://arxiv.org/abs/1801.10123
 5 | 
 6 | __Title__: Links: A High-Dimensional Online Clustering Method
 7 | 
 8 | __Authors__: Philip Andrew Mansfield, Quan Wang, Carlton Downey, Li Wan, Ignacio Lopez Moreno
 9 | 
10 | ## Overview
11 | This is a clustering algorithm for online data. That is, it will predict 
12 | cluster membership for vectors that it is shown one-by-one. It does not 
13 | require examining the entire dataset to predict cluster membership.
14 | 
15 | It works by maintaining a two-level hierarchy of clusters and subclusters.
16 | Each subcluster has a centroid that is compared with new vector for prediction
17 | using cosine similarity. Depending on the previous data that has been seen, 
18 | the new data point can be assigned to an existing cluster/subcluster, 
19 | assigned to a new subcluster within an existing cluster, or 
20 | assigned to a new subcluster and cluster. 
21 | 
22 | Instantiating the class requires 3 hyperparameters:
23 | * cluster_similarity_threshold
24 | * subcluster_similarity_threshold
25 | * pair_similarity_maximum
26 | 
27 | These details are best understood by reading the paper.
28 | 
29 | ## Installation
30 | 
31 | `pip install -r requirements.txt`
32 | 
33 | ## Usage example
34 | 
35 | ```python
36 | from links_cluster import LinksCluster
37 | 
38 | ...
39 | links_cluster = LinksCluster(cluster_similarity_threshold, subcluster_similarity_threshold, pair_similarity_maximum)
40 | for vector in data:
41 |     predicted_cluster = links_cluster.predict(vector)
42 | 
43 | ```
44 | 
45 | For more usage examples, see the `tests`.
46 | 
47 | 
48 | 


--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
1 | from .links_cluster import LinksCluster
2 | 
3 | __all__ = ['LinksCluster']
4 | 


--------------------------------------------------------------------------------
/links_cluster.py:
--------------------------------------------------------------------------------
  1 | """Links online clustering algorithm.
  2 | 
  3 | Reference: https://arxiv.org/abs/1801.10123
  4 | """
  5 | import logging
  6 | 
  7 | import numpy as np
  8 | from scipy.spatial.distance import cosine
  9 | 
 10 | 
 11 | class Subcluster:
 12 |     """Class for subclusters and edges between subclusters."""
 13 |     def __init__(self, initial_vector: np.ndarray, store_vectors: bool = False):
 14 |         self.input_vectors = [initial_vector]
 15 |         self.centroid = initial_vector
 16 |         self.n_vectors = 1
 17 |         self.store_vectors = store_vectors
 18 |         self.connected_subclusters = set()
 19 | 
 20 |     def add(self, vector: np.ndarray):
 21 |         """Add a new vector to the subcluster, update the centroid."""
 22 |         if self.store_vectors:
 23 |             self.input_vectors.append(vector)
 24 |         self.n_vectors += 1
 25 |         if self.centroid is None:
 26 |             self.centroid = vector
 27 |         else:
 28 |             self.centroid = (self.n_vectors - 1) / \
 29 |                             self.n_vectors * self.centroid \
 30 |                             + vector / self.n_vectors
 31 | 
 32 |     def merge(self,
 33 |               subcluster_merge: 'Subcluster',
 34 |               delete_merged: bool = True):
 35 |         """Merge subcluster_merge into self. Update centroids."""
 36 |         if self.store_vectors:
 37 |             self.input_vectors += subcluster_merge.input_vectors
 38 | 
 39 |         # Update centroid and n_vectors
 40 |         self.centroid = self.n_vectors * self.centroid \
 41 |             + subcluster_merge.n_vectors \
 42 |             * subcluster_merge.centroid
 43 |         self.centroid /= self.n_vectors + subcluster_merge.n_vectors
 44 |         self.n_vectors += subcluster_merge.n_vectors
 45 |         try:
 46 |             subcluster_merge.connected_subclusters.remove(self)
 47 |             self.connected_subclusters.remove(subcluster_merge)
 48 |         except KeyError:
 49 |             logging.warning("Attempted to merge unconnected subclusters. "
 50 |                             "Merging anyway.")
 51 |         for sc in subcluster_merge.connected_subclusters:
 52 |             sc.connected_subclusters.remove(subcluster_merge)
 53 |             if self not in sc.connected_subclusters and sc != self:
 54 |                 sc.connected_subclusters.update({self})
 55 |         self.connected_subclusters.update(subcluster_merge.connected_subclusters)
 56 |         if delete_merged:
 57 |             del subcluster_merge
 58 | 
 59 | 
 60 | class LinksCluster:
 61 |     """An online clustering algorithm."""
 62 |     def __init__(self,
 63 |                  cluster_similarity_threshold: float,
 64 |                  subcluster_similarity_threshold: float,
 65 |                  pair_similarity_maximum: float,
 66 |                  store_vectors=False
 67 |                  ):
 68 |         self.clusters = []
 69 |         self.cluster_similarity_threshold = cluster_similarity_threshold
 70 |         self.subcluster_similarity_threshold = subcluster_similarity_threshold
 71 |         self.pair_similarity_maximum = pair_similarity_maximum
 72 |         self.store_vectors = store_vectors
 73 | 
 74 |     def predict(self, new_vector: np.ndarray) -> int:
 75 |         """Predict a cluster id for new_vector."""
 76 |         if len(self.clusters) == 0:
 77 |             # Handle first vector
 78 |             self.clusters.append([Subcluster(new_vector, store_vectors=self.store_vectors)])
 79 |             return 0
 80 | 
 81 |         best_subcluster = None
 82 |         best_similarity = -np.inf
 83 |         best_subcluster_cluster_id = None
 84 |         best_subcluster_id = None
 85 |         for cl_idx, cl in enumerate(self.clusters):
 86 |             for sc_idx, sc in enumerate(cl):
 87 |                 cossim = 1.0 - cosine(new_vector, sc.centroid)
 88 |                 if cossim > best_similarity:
 89 |                     best_subcluster = sc
 90 |                     best_similarity = cossim
 91 |                     best_subcluster_cluster_id = cl_idx
 92 |                     best_subcluster_id = sc_idx
 93 |         if best_similarity >= self.subcluster_similarity_threshold:  # eq. (20)
 94 |             # Add to existing subcluster
 95 |             best_subcluster.add(new_vector)
 96 |             self.update_cluster(best_subcluster_cluster_id, best_subcluster_id)
 97 |             assigned_cluster = best_subcluster_cluster_id
 98 |         else:
 99 |             # Create new subcluster
100 |             new_subcluster = Subcluster(new_vector, store_vectors=self.store_vectors)
101 |             cossim = 1.0 - cosine(new_subcluster.centroid, best_subcluster.centroid)
102 |             if cossim >= self.sim_threshold(best_subcluster.n_vectors, 1):  # eq. (21)
103 |                 # New subcluster is part of existing cluster
104 |                 self.add_edge(best_subcluster, new_subcluster)
105 |                 self.clusters[best_subcluster_cluster_id].append(new_subcluster)
106 |                 assigned_cluster = best_subcluster_cluster_id
107 |             else:
108 |                 # New subcluster is a new cluster
109 |                 self.clusters.append([new_subcluster])
110 |                 assigned_cluster = len(self.clusters) - 1
111 |         return assigned_cluster
112 | 
113 |     @staticmethod
114 |     def add_edge(sc1: Subcluster, sc2: Subcluster):
115 |         """Add an edge between subclusters sc1, and sc2."""
116 |         sc1.connected_subclusters.add(sc2)
117 |         sc2.connected_subclusters.add(sc1)
118 | 
119 |     def update_edge(self, sc1: Subcluster, sc2: Subcluster):
120 |         """Compare subclusters sc1 and sc2, remove or add an edge depending on cosine similarity.
121 | 
122 |         Args:
123 |             sc1: Subcluster
124 |                 First subcluster to compare
125 |             sc2: Subcluster
126 |                 Second subcluster to compare
127 | 
128 |         Returns:
129 |             bool
130 |                 True if the edge is valid
131 |                 False if the edge is not valid
132 |         """
133 |         cossim = 1.0 - cosine(sc1.centroid, sc2.centroid)
134 |         threshold = self.sim_threshold(sc1.n_vectors, sc2.n_vectors)
135 |         if cossim < threshold:
136 |             try:
137 |                 sc1.connected_subclusters.remove(sc2)
138 |                 sc2.connected_subclusters.remove(sc1)
139 |             except KeyError:
140 |                 logging.warning("Attempted to update an invalid edge that didn't exist. "
141 |                                 "Edge remains nonexistant.")
142 |             return False
143 |         else:
144 |             sc1.connected_subclusters.add(sc2)
145 |             sc2.connected_subclusters.add(sc1)
146 |             return True
147 | 
148 |     def merge_subclusters(self, cl_idx, sc_idx1, sc_idx2):
149 |         """Merge subclusters with id's sc_idx1 and sc_idx2 of cluster with id cl_idx."""
150 |         sc2 = self.clusters[cl_idx][sc_idx2]
151 |         self.clusters[cl_idx][sc_idx1].merge(sc2)
152 |         self.update_cluster(cl_idx, sc_idx1)
153 |         self.clusters[cl_idx] = self.clusters[cl_idx][:sc_idx2] \
154 |             + self.clusters[cl_idx][sc_idx2 + 1:]
155 |         for sc in self.clusters[cl_idx]:
156 |             if sc2 in sc.connected_subclusters:
157 |                 sc.connected_subclusters.remove(sc2)
158 | 
159 |     def update_cluster(self, cl_idx: int, sc_idx: int):
160 |         """Update cluster
161 | 
162 |         Subcluster with id sc_idx has been changed, and we want to
163 |         update the parent cluster according to the discussion in
164 |         section 3.4 of the paper.
165 | 
166 |         Args:
167 |             cl_idx: int
168 |                 The index of the cluster to update
169 |             sc_idx: int
170 |                 The index of the subcluster that has been changed
171 | 
172 |         Returns:
173 |             None
174 | 
175 |         """
176 |         updated_sc = self.clusters[cl_idx][sc_idx]
177 |         severed_subclusters = []
178 |         connected_scs = set(updated_sc.connected_subclusters)
179 |         for connected_sc in connected_scs:
180 |             connected_sc_idx = None
181 |             for c_sc_idx, sc in enumerate(self.clusters[cl_idx]):
182 |                 if sc == connected_sc:
183 |                     connected_sc_idx = c_sc_idx
184 |             if connected_sc_idx is None:
185 |                 raise ValueError(f"Connected subcluster of {sc_idx} "
186 |                                  f"was not found in cluster list of {cl_idx}.")
187 |             cossim = 1.0 - cosine(updated_sc.centroid, connected_sc.centroid)
188 |             if cossim >= self.subcluster_similarity_threshold:
189 |                 self.merge_subclusters(cl_idx, sc_idx, connected_sc_idx)
190 |             else:
191 |                 are_connected = self.update_edge(updated_sc, connected_sc)
192 |                 if not are_connected:
193 |                     severed_subclusters.append(connected_sc_idx)
194 |         for severed_sc_id in severed_subclusters:
195 |             severed_sc = self.clusters[cl_idx][severed_sc_id]
196 |             if len(severed_sc.connected_subclusters) == 0:
197 |                 for cluster_sc in self.clusters[cl_idx]:
198 |                     if cluster_sc != severed_sc:
199 |                         cossim = 1.0 - cosine(cluster_sc.centroid,
200 |                                               severed_sc.centroid)
201 |                         if cossim >= self.sim_threshold(cluster_sc.n_vectors,
202 |                                                         severed_sc.n_vectors):
203 |                             self.add_edge(cluster_sc, severed_sc)
204 |             if len(severed_sc.connected_subclusters) == 0:
205 |                 self.clusters[cl_idx] = self.clusters[cl_idx][:severed_sc_id] \
206 |                     + self.clusters[cl_idx][severed_sc_id + 1:]
207 |                 self.clusters.append([severed_sc])
208 | 
209 |     def get_all_vectors(self):
210 |         """Return all stored vectors from entire history.
211 | 
212 |         Returns:
213 |             list
214 |                 list of vectors
215 | 
216 |         Raises:
217 |             RuntimeError
218 |                 if self.store_vectors is False (i.e. there are no stored vectors)
219 |         """
220 |         if not self.store_vectors:
221 |             raise RuntimeError("Vectors were not stored, so can't be collected")
222 |         all_vectors = []
223 |         for cl in self.clusters:
224 |             for scl in cl:
225 |                 all_vectors += scl.input_vectors
226 |         return all_vectors
227 | 
228 |     def sim_threshold(self, k: int, kp: int) -> float:
229 |         """Compute the similarity threshold.
230 | 
231 |         This is based on equations (16) and (24) of the paper.
232 | 
233 |         Args:
234 |             k: int
235 |                 The number of vectors in a cluster or subcluster
236 |             kp: int
237 |                 k-prime in the paper, the number of vectors in another
238 |                 cluster or subcluster
239 | 
240 |         Returns:
241 |             float
242 |                 The similarity threshold for inclusion in a cluster or subcluster.
243 |         """
244 |         s = (1.0 + 1.0 / k * (1.0 / self.cluster_similarity_threshold ** 2 - 1.0))
245 |         s *= (1.0 + 1.0 / kp * (1.0 / self.cluster_similarity_threshold ** 2 - 1.0))
246 |         s = 1.0 / np.sqrt(s)  # eq. (16)
247 |         s = self.cluster_similarity_threshold ** 2 \
248 |             + (self.pair_similarity_maximum - self.cluster_similarity_threshold ** 2) \
249 |             / (1.0 - self.cluster_similarity_threshold ** 2) \
250 |             * (s - self.cluster_similarity_threshold ** 2)  # eq. (24)
251 |         return s
252 | 


--------------------------------------------------------------------------------
/pylintrc:
--------------------------------------------------------------------------------
  1 | [MASTER]
  2 | 
  3 | # A comma-separated list of package or module names from where C extensions may
  4 | # be loaded. Extensions are loading into the active Python interpreter and may
  5 | # run arbitrary code.
  6 | extension-pkg-whitelist=
  7 | 
  8 | # Add files or directories to the blacklist. They should be base names, not
  9 | # paths.
 10 | ignore=venv,.git
 11 | 
 12 | # Add files or directories matching the regex patterns to the blacklist. The
 13 | # regex matches against base names, not paths.
 14 | ignore-patterns=
 15 | 
 16 | # Python code to execute, usually for sys.path manipulation such as
 17 | # pygtk.require().
 18 | #init-hook=
 19 | 
 20 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 21 | # number of processors available to use.
 22 | jobs=1
 23 | 
 24 | # Control the amount of potential inferred values when inferring a single
 25 | # object. This can help the performance when dealing with large functions or
 26 | # complex, nested conditions.
 27 | limit-inference-results=100
 28 | 
 29 | # List of plugins (as comma separated values of python modules names) to load,
 30 | # usually to register additional checkers.
 31 | load-plugins=
 32 | 
 33 | # Pickle collected data for later comparisons.
 34 | persistent=yes
 35 | 
 36 | # Specify a configuration file.
 37 | #rcfile=
 38 | 
 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit
 40 | # user-friendly hints instead of false-positive error messages.
 41 | suggestion-mode=yes
 42 | 
 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the
 44 | # active Python interpreter and may run arbitrary code.
 45 | unsafe-load-any-extension=no
 46 | 
 47 | 
 48 | [MESSAGES CONTROL]
 49 | 
 50 | # Only show warnings with the listed confidence levels. Leave empty to show
 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
 52 | confidence=
 53 | 
 54 | # Disable the message, report, category or checker with the given id(s). You
 55 | # can either give multiple identifiers separated by comma (,) or put this
 56 | # option multiple times (only on the command line, not in the configuration
 57 | # file where it should appear only once). You can also use "--disable=all" to
 58 | # disable everything first and then reenable specific checks. For example, if
 59 | # you want to run only the similarities checker, you can use "--disable=all
 60 | # --enable=similarities". If you want to run only the classes checker, but have
 61 | # no Warning level messages displayed, use "--disable=all --enable=classes
 62 | # --disable=W".
 63 | disable=invalid-name,
 64 |         no-name-in-module,
 65 |         bad-continuation,
 66 |         import-error,
 67 |         print-statement,
 68 |         parameter-unpacking,
 69 |         unpacking-in-except,
 70 |         old-raise-syntax,
 71 |         backtick,
 72 |         long-suffix,
 73 |         old-ne-operator,
 74 |         old-octal-literal,
 75 |         import-star-module-level,
 76 |         non-ascii-bytes-literal,
 77 |         raw-checker-failed,
 78 |         bad-inline-option,
 79 |         locally-disabled,
 80 |         file-ignored,
 81 |         suppressed-message,
 82 |         useless-suppression,
 83 |         deprecated-pragma,
 84 |         use-symbolic-message-instead,
 85 |         duplicate-code,
 86 |         too-many-instance-attributes,
 87 |         too-few-public-methods,
 88 |         too-many-branches,
 89 |         too-many-arguments,
 90 |         too-many-locals,
 91 |         too-many-statements,
 92 |         unnecessary-pass,
 93 |         fixme,
 94 |         apply-builtin,
 95 |         basestring-builtin,
 96 |         buffer-builtin,
 97 |         cmp-builtin,
 98 |         coerce-builtin,
 99 |         execfile-builtin,
100 |         file-builtin,
101 |         long-builtin,
102 |         raw_input-builtin,
103 |         reduce-builtin,
104 |         standarderror-builtin,
105 |         unicode-builtin,
106 |         xrange-builtin,
107 |         coerce-method,
108 |         delslice-method,
109 |         getslice-method,
110 |         setslice-method,
111 |         no-absolute-import,
112 |         old-division,
113 |         dict-iter-method,
114 |         dict-view-method,
115 |         next-method-called,
116 |         metaclass-assignment,
117 |         indexing-exception,
118 |         raising-string,
119 |         reload-builtin,
120 |         oct-method,
121 |         hex-method,
122 |         nonzero-method,
123 |         cmp-method,
124 |         input-builtin,
125 |         round-builtin,
126 |         intern-builtin,
127 |         unichr-builtin,
128 |         map-builtin-not-iterating,
129 |         zip-builtin-not-iterating,
130 |         range-builtin-not-iterating,
131 |         filter-builtin-not-iterating,
132 |         using-cmp-argument,
133 |         eq-without-hash,
134 |         div-method,
135 |         idiv-method,
136 |         rdiv-method,
137 |         exception-message-attribute,
138 |         invalid-str-codec,
139 |         sys-max-int,
140 |         bad-python3-import,
141 |         deprecated-string-function,
142 |         deprecated-str-translate-call,
143 |         deprecated-itertools-function,
144 |         deprecated-types-field,
145 |         next-method-defined,
146 |         dict-items-not-iterating,
147 |         dict-keys-not-iterating,
148 |         dict-values-not-iterating,
149 |         deprecated-operator-function,
150 |         deprecated-urllib-function,
151 |         xreadlines-attribute,
152 |         deprecated-sys-function,
153 |         exception-escape,
154 |         comprehension-escape
155 | 
156 | # Enable the message, report, category or checker with the given id(s). You can
157 | # either give multiple identifier separated by comma (,) or put this option
158 | # multiple time (only on the command line, not in the configuration file where
159 | # it should appear only once). See also the "--disable" option for examples.
160 | enable=c-extension-no-member
161 | 
162 | 
163 | [REPORTS]
164 | 
165 | # Python expression which should return a note less than 10 (10 is the highest
166 | # note). You have access to the variables errors warning, statement which
167 | # respectively contain the number of errors / warnings messages and the total
168 | # number of statements analyzed. This is used by the global evaluation report
169 | # (RP0004).
170 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
171 | 
172 | # Template used to display messages. This is a python new-style format string
173 | # used to format the message information. See doc for all details.
174 | #msg-template=
175 | 
176 | # Set the output format. Available formats are text, parseable, colorized, json
177 | # and msvs (visual studio). You can also give a reporter class, e.g.
178 | # mypackage.mymodule.MyReporterClass.
179 | output-format=text
180 | 
181 | # Tells whether to display a full report or only the messages.
182 | reports=no
183 | 
184 | # Activate the evaluation score.
185 | score=yes
186 | 
187 | 
188 | [REFACTORING]
189 | 
190 | # Maximum number of nested blocks for function / method body
191 | max-nested-blocks=5
192 | 
193 | # Complete name of functions that never returns. When checking for
194 | # inconsistent-return-statements if a never returning function is called then
195 | # it will be considered as an explicit return statement and no message will be
196 | # printed.
197 | never-returning-functions=sys.exit
198 | 
199 | 
200 | [VARIABLES]
201 | 
202 | # List of additional names supposed to be defined in builtins. Remember that
203 | # you should avoid defining new builtins when possible.
204 | additional-builtins=
205 | 
206 | # Tells whether unused global variables should be treated as a violation.
207 | allow-global-unused-variables=yes
208 | 
209 | # List of strings which can identify a callback function by name. A callback
210 | # name must start or end with one of those strings.
211 | callbacks=cb_,
212 |           _cb
213 | 
214 | # A regular expression matching the name of dummy variables (i.e. expected to
215 | # not be used).
216 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
217 | 
218 | # Argument names that match this expression will be ignored. Default to name
219 | # with leading underscore.
220 | ignored-argument-names=_.*|^ignored_|^unused_
221 | 
222 | # Tells whether we should check for unused import in __init__ files.
223 | init-import=no
224 | 
225 | # List of qualified module names which can have objects that can redefine
226 | # builtins.
227 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
228 | 
229 | 
230 | [MISCELLANEOUS]
231 | 
232 | # List of note tags to take in consideration, separated by a comma.
233 | notes=FIXME,
234 |       XXX,
235 |       TODO
236 | 
237 | 
238 | [SIMILARITIES]
239 | 
240 | # Ignore comments when computing similarities.
241 | ignore-comments=yes
242 | 
243 | # Ignore docstrings when computing similarities.
244 | ignore-docstrings=yes
245 | 
246 | # Ignore imports when computing similarities.
247 | ignore-imports=no
248 | 
249 | # Minimum lines number of a similarity.
250 | min-similarity-lines=4
251 | 
252 | 
253 | [BASIC]
254 | 
255 | # Naming style matching correct argument names.
256 | argument-naming-style=snake_case
257 | 
258 | # Regular expression matching correct argument names. Overrides argument-
259 | # naming-style.
260 | #argument-rgx=
261 | 
262 | # Naming style matching correct attribute names.
263 | attr-naming-style=snake_case
264 | 
265 | # Regular expression matching correct attribute names. Overrides attr-naming-
266 | # style.
267 | #attr-rgx=
268 | 
269 | # Bad variable names which should always be refused, separated by a comma.
270 | bad-names=foo,
271 |           bar,
272 |           baz,
273 |           toto,
274 |           tutu,
275 |           tata
276 | 
277 | # Naming style matching correct class attribute names.
278 | class-attribute-naming-style=any
279 | 
280 | # Regular expression matching correct class attribute names. Overrides class-
281 | # attribute-naming-style.
282 | #class-attribute-rgx=
283 | 
284 | # Naming style matching correct class names.
285 | class-naming-style=PascalCase
286 | 
287 | # Regular expression matching correct class names. Overrides class-naming-
288 | # style.
289 | #class-rgx=
290 | 
291 | # Naming style matching correct constant names.
292 | const-naming-style=UPPER_CASE
293 | 
294 | # Regular expression matching correct constant names. Overrides const-naming-
295 | # style.
296 | #const-rgx=
297 | 
298 | # Minimum line length for functions/classes that require docstrings, shorter
299 | # ones are exempt.
300 | docstring-min-length=-1
301 | 
302 | # Naming style matching correct function names.
303 | function-naming-style=snake_case
304 | 
305 | # Regular expression matching correct function names. Overrides function-
306 | # naming-style.
307 | #function-rgx=
308 | 
309 | # Good variable names which should always be accepted, separated by a comma.
310 | good-names=i,
311 |            j,
312 |            k,
313 |            ex,
314 |            Run,
315 |            _
316 | 
317 | # Include a hint for the correct naming format with invalid-name.
318 | include-naming-hint=no
319 | 
320 | # Naming style matching correct inline iteration names.
321 | inlinevar-naming-style=any
322 | 
323 | # Regular expression matching correct inline iteration names. Overrides
324 | # inlinevar-naming-style.
325 | #inlinevar-rgx=
326 | 
327 | # Naming style matching correct method names.
328 | method-naming-style=snake_case
329 | 
330 | # Regular expression matching correct method names. Overrides method-naming-
331 | # style.
332 | #method-rgx=
333 | 
334 | # Naming style matching correct module names.
335 | module-naming-style=snake_case
336 | 
337 | # Regular expression matching correct module names. Overrides module-naming-
338 | # style.
339 | #module-rgx=
340 | 
341 | # Colon-delimited sets of names that determine each other's naming style when
342 | # the name regexes allow several styles.
343 | name-group=
344 | 
345 | # Regular expression which should only match function or class names that do
346 | # not require a docstring.
347 | no-docstring-rgx=^_
348 | 
349 | # List of decorators that produce properties, such as abc.abstractproperty. Add
350 | # to this list to register other decorators that produce valid properties.
351 | # These decorators are taken in consideration only for invalid-name.
352 | property-classes=abc.abstractproperty
353 | 
354 | # Naming style matching correct variable names.
355 | variable-naming-style=snake_case
356 | 
357 | # Regular expression matching correct variable names. Overrides variable-
358 | # naming-style.
359 | #variable-rgx=
360 | 
361 | 
362 | [STRING]
363 | 
364 | # This flag controls whether the implicit-str-concat-in-sequence should
365 | # generate a warning on implicit string concatenation in sequences defined over
366 | # several lines.
367 | check-str-concat-over-line-jumps=no
368 | 
369 | 
370 | [SPELLING]
371 | 
372 | # Limits count of emitted suggestions for spelling mistakes.
373 | max-spelling-suggestions=4
374 | 
375 | # Spelling dictionary name. Available dictionaries: none. To make it working
376 | # install python-enchant package..
377 | spelling-dict=
378 | 
379 | # List of comma separated words that should not be checked.
380 | spelling-ignore-words=
381 | 
382 | # A path to a file that contains private dictionary; one word per line.
383 | spelling-private-dict-file=
384 | 
385 | # Tells whether to store unknown words to indicated private dictionary in
386 | # --spelling-private-dict-file option instead of raising a message.
387 | spelling-store-unknown-words=no
388 | 
389 | 
390 | [FORMAT]
391 | 
392 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
393 | expected-line-ending-format=
394 | 
395 | # Regexp for a line that is allowed to be longer than the limit.
396 | ignore-long-lines=^\s*(# )?<?https?://\S+>?$
397 | 
398 | # Number of spaces of indent required inside a hanging or continued line.
399 | indent-after-paren=4
400 | 
401 | # String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
402 | # tab).
403 | indent-string='    '
404 | 
405 | # Maximum number of characters on a single line.
406 | max-line-length=99
407 | 
408 | # Maximum number of lines in a module.
409 | max-module-lines=1000
410 | 
411 | # List of optional constructs for which whitespace checking is disabled. `dict-
412 | # separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
413 | # `trailing-comma` allows a space between comma and closing bracket: (a, ).
414 | # `empty-line` allows space-only lines.
415 | no-space-check=trailing-comma,
416 |                dict-separator
417 | 
418 | # Allow the body of a class to be on the same line as the declaration if body
419 | # contains single statement.
420 | single-line-class-stmt=no
421 | 
422 | # Allow the body of an if to be on the same line as the test if there is no
423 | # else.
424 | single-line-if-stmt=no
425 | 
426 | 
427 | [LOGGING]
428 | 
429 | # Format style used to check logging format string. `old` means using %
430 | # formatting, while `new` is for `{}` formatting.
431 | logging-format-style=old
432 | 
433 | # Logging modules to check that the string format arguments are in logging
434 | # function parameter format.
435 | logging-modules=logging
436 | 
437 | 
438 | [TYPECHECK]
439 | 
440 | # List of decorators that produce context managers, such as
441 | # contextlib.contextmanager. Add to this list to register other decorators that
442 | # produce valid context managers.
443 | contextmanager-decorators=contextlib.contextmanager
444 | 
445 | # List of members which are set dynamically and missed by pylint inference
446 | # system, and so shouldn't trigger E1101 when accessed. Python regular
447 | # expressions are accepted.
448 | generated-members=
449 | 
450 | # Tells whether missing members accessed in mixin class should be ignored. A
451 | # mixin class is detected if its name ends with "mixin" (case insensitive).
452 | ignore-mixin-members=yes
453 | 
454 | # Tells whether to warn about missing members when the owner of the attribute
455 | # is inferred to be None.
456 | ignore-none=yes
457 | 
458 | # This flag controls whether pylint should warn about no-member and similar
459 | # checks whenever an opaque object is returned when inferring. The inference
460 | # can return multiple potential results while evaluating a Python object, but
461 | # some branches might not be evaluated, which results in partial inference. In
462 | # that case, it might be useful to still emit no-member and other checks for
463 | # the rest of the inferred objects.
464 | ignore-on-opaque-inference=yes
465 | 
466 | # List of class names for which member attributes should not be checked (useful
467 | # for classes with dynamically set attributes). This supports the use of
468 | # qualified names.
469 | ignored-classes=optparse.Values,thread._local,_thread._local
470 | 
471 | # List of module names for which member attributes should not be checked
472 | # (useful for modules/projects where namespaces are manipulated during runtime
473 | # and thus existing member attributes cannot be deduced by static analysis. It
474 | # supports qualified module names, as well as Unix pattern matching.
475 | ignored-modules=
476 | 
477 | # Show a hint with possible names when a member name was not found. The aspect
478 | # of finding the hint is based on edit distance.
479 | missing-member-hint=yes
480 | 
481 | # The minimum edit distance a name should have in order to be considered a
482 | # similar match for a missing member name.
483 | missing-member-hint-distance=1
484 | 
485 | # The total number of similar names that should be taken in consideration when
486 | # showing a hint for a missing member.
487 | missing-member-max-choices=1
488 | 
489 | 
490 | [IMPORTS]
491 | 
492 | # Allow wildcard imports from modules that define __all__.
493 | allow-wildcard-with-all=no
494 | 
495 | # Analyse import fallback blocks. This can be used to support both Python 2 and
496 | # 3 compatible code, which means that the block might have code that exists
497 | # only in one or another interpreter, leading to false positives when analysed.
498 | analyse-fallback-blocks=no
499 | 
500 | # Deprecated modules which should not be used, separated by a comma.
501 | deprecated-modules=optparse,tkinter.tix
502 | 
503 | # Create a graph of external dependencies in the given file (report RP0402 must
504 | # not be disabled).
505 | ext-import-graph=
506 | 
507 | # Create a graph of every (i.e. internal and external) dependencies in the
508 | # given file (report RP0402 must not be disabled).
509 | import-graph=
510 | 
511 | # Create a graph of internal dependencies in the given file (report RP0402 must
512 | # not be disabled).
513 | int-import-graph=
514 | 
515 | # Force import order to recognize a module as part of the standard
516 | # compatibility libraries.
517 | known-standard-library=
518 | 
519 | # Force import order to recognize a module as part of a third party library.
520 | known-third-party=enchant
521 | 
522 | 
523 | [DESIGN]
524 | 
525 | # Maximum number of arguments for function / method.
526 | max-args=5
527 | 
528 | # Maximum number of attributes for a class (see R0902).
529 | max-attributes=7
530 | 
531 | # Maximum number of boolean expressions in an if statement.
532 | max-bool-expr=5
533 | 
534 | # Maximum number of branch for function / method body.
535 | max-branches=12
536 | 
537 | # Maximum number of locals for function / method body.
538 | max-locals=15
539 | 
540 | # Maximum number of parents for a class (see R0901).
541 | max-parents=7
542 | 
543 | # Maximum number of public methods for a class (see R0904).
544 | max-public-methods=20
545 | 
546 | # Maximum number of return / yield for function / method body.
547 | max-returns=6
548 | 
549 | # Maximum number of statements in function / method body.
550 | max-statements=50
551 | 
552 | # Minimum number of public methods for a class (see R0903).
553 | min-public-methods=2
554 | 
555 | 
556 | [CLASSES]
557 | 
558 | # List of method names used to declare (i.e. assign) instance attributes.
559 | defining-attr-methods=__init__,
560 |                       __new__,
561 |                       setUp
562 | 
563 | # List of member names, which should be excluded from the protected access
564 | # warning.
565 | exclude-protected=_asdict,
566 |                   _fields,
567 |                   _replace,
568 |                   _source,
569 |                   _make
570 | 
571 | # List of valid names for the first argument in a class method.
572 | valid-classmethod-first-arg=cls
573 | 
574 | # List of valid names for the first argument in a metaclass class method.
575 | valid-metaclass-classmethod-first-arg=cls
576 | 
577 | 
578 | [EXCEPTIONS]
579 | 
580 | # Exceptions that will emit a warning when being caught. Defaults to
581 | # "BaseException, Exception".
582 | overgeneral-exceptions=BaseException,
583 |                        Exception


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | scipy


--------------------------------------------------------------------------------
/tests/test_links_cluster.py:
--------------------------------------------------------------------------------
  1 | """Tests for LinksCluster and LinksSubcluster classes."""
  2 | # pylint: disable=W0201, E1101
  3 | 
  4 | import numpy as np
  5 | 
  6 | from links_cluster import LinksCluster, Subcluster
  7 | 
  8 | 
  9 | class TestLinksCluster:
 10 |     """Tests for LinksCluster class."""
 11 |     def setup_method(self):
 12 |         """Setup for tests."""
 13 |         self.cluster_similarity_threshold = 0.3
 14 |         self.subcluster_similarity_threshold = 0.2
 15 |         self.pair_similarity_maximum = 1.0
 16 |         self.cluster = LinksCluster(self.cluster_similarity_threshold,
 17 |                                     self.subcluster_similarity_threshold,
 18 |                                     self.pair_similarity_maximum,
 19 |                                     store_vectors=True)
 20 |         self.vector_dim = 256
 21 | 
 22 |     def random_vec(self):
 23 |         """Generate random vector of the correct shape."""
 24 |         return np.random.random((self.vector_dim))
 25 | 
 26 |     def rotate_vec(self, vector, angle):
 27 |         """Rotate a vector in the x-y plane by angle (radians).
 28 | 
 29 |         Note that this only has the expected impact on the cosine
 30 |         similarity if the vector is mostly in the x-y plane.
 31 |         """
 32 |         rotation_matrix = np.identity(self.vector_dim)
 33 |         rotation_matrix[0:2, 0:2] = [
 34 |             [np.cos(angle), -1 * np.sin(angle)],
 35 |             [np.sin(angle), np.cos(angle)]]
 36 |         vector = rotation_matrix.dot(vector)
 37 |         return vector
 38 | 
 39 |     def test_init(self):
 40 |         """Test that cluster has the expected type."""
 41 |         assert isinstance(self.cluster, LinksCluster)
 42 | 
 43 |     def test_predict_once(self):
 44 |         """Test that first prediction returns 0."""
 45 |         vector = np.random.random((self.vector_dim, ))
 46 |         prediction = self.cluster.predict(vector)
 47 |         assert prediction == 0
 48 |         assert vector in self.cluster.get_all_vectors()
 49 | 
 50 |     def test_predict_many(self):
 51 |         """Test that we can make many predictions, multiple classes are predicted."""
 52 |         how_many = 100
 53 |         predictions = []
 54 |         vectors = []
 55 |         for _ in range(how_many):
 56 |             vector = np.random.uniform(0, 1, self.vector_dim)
 57 |             vector[0] += 1000.0
 58 |             if np.random.uniform(0, 1) < 0.5:
 59 |                 vector = self.rotate_vec(vector, 3.14)
 60 |             prediction = self.cluster.predict(vector)
 61 |             vectors.append(vector)
 62 |             predictions.append(prediction)
 63 |         assert len(predictions) == how_many
 64 |         assert not all(p == 0 for p in predictions)
 65 |         assert len(vectors) == len(self.cluster.get_all_vectors())
 66 | 
 67 |     def test_add_same_subcluster(self):
 68 |         """Test clustering after adding within same subcluster."""
 69 |         vector = self.random_vec()
 70 |         vector[0] += 1000.0
 71 |         first_prediction = self.cluster.predict(vector)
 72 |         vector2 = self.rotate_vec(
 73 |             vector,
 74 |             0.1 * np.arccos(self.subcluster_similarity_threshold))
 75 |         second_prediction = self.cluster.predict(vector2)
 76 |         assert first_prediction == second_prediction
 77 |         assert len(self.cluster.clusters) == 1
 78 |         assert len(self.cluster.clusters[0]) == 1  # Should be one subcluster
 79 | 
 80 |     def test_add_new_subcluster(self):
 81 |         """Test clustering after adding to new subcluster."""
 82 |         vector = self.random_vec()
 83 |         vector[0] += 1000.0
 84 |         first_prediction = self.cluster.predict(vector)
 85 |         vector2 = self.rotate_vec(
 86 |             vector,
 87 |             1.01 * np.arccos(self.subcluster_similarity_threshold))
 88 |         second_prediction = self.cluster.predict(vector2)
 89 |         assert first_prediction == second_prediction
 90 |         assert len(self.cluster.clusters) == 1
 91 |         assert len(self.cluster.clusters[0]) == 2  # New subcluster created.
 92 | 
 93 |     def test_add_new_cluster(self):
 94 |         """Test clustering after adding to new subcluster."""
 95 |         vector = self.random_vec()
 96 |         vector[0] += 1000.0
 97 |         first_prediction = self.cluster.predict(vector)
 98 |         vector2 = self.rotate_vec(
 99 |             vector,
100 |             2 * np.arccos(self.cluster_similarity_threshold))
101 |         second_prediction = self.cluster.predict(vector2)
102 |         assert first_prediction == 0
103 |         assert second_prediction == 1
104 |         assert len(self.cluster.clusters) == 2
105 |         assert len(self.cluster.clusters[0]) == 1
106 |         assert len(self.cluster.clusters[1]) == 1
107 | 
108 |     def test_add_edge(self):
109 |         """Test adding an edge between subclusters."""
110 |         sc1 = Subcluster(self.random_vec())
111 |         sc2 = Subcluster(self.random_vec())
112 |         self.cluster.add_edge(sc1, sc2)
113 |         assert sc2 in sc1.connected_subclusters
114 |         assert sc1 in sc2.connected_subclusters
115 | 
116 |     def test_update_edge_valid(self):
117 |         """Test update_edge with a valid edge."""
118 |         vector = self.random_vec()
119 |         similar_vector = vector + 1.0e-6 * self.random_vec()
120 |         sc1 = Subcluster(vector)
121 |         sc2 = Subcluster(similar_vector)
122 |         edge_is_valid = self.cluster.update_edge(sc1, sc2)
123 |         assert edge_is_valid
124 |         assert sc2 in sc1.connected_subclusters
125 |         assert sc1 in sc2.connected_subclusters
126 | 
127 |     def test_update_edge_invalid(self):
128 |         """Test update_edge with an invalid edge."""
129 |         vector = self.random_vec()
130 |         vector[0] += 1000.0
131 |         different_vector = self.rotate_vec(vector, 3.14)
132 |         sc1 = Subcluster(vector)
133 |         sc2 = Subcluster(different_vector)
134 |         self.cluster.add_edge(sc1, sc2)
135 |         edge_is_valid = self.cluster.update_edge(sc1, sc2)
136 |         assert not edge_is_valid
137 |         assert sc2 not in sc1.connected_subclusters
138 |         assert sc1 not in sc2.connected_subclusters
139 | 
140 |     def test_merge_subclusters(self):
141 |         """Test that merging subclusters works as expected."""
142 |         vector = self.random_vec()
143 |         vector[0] += 1000.0
144 |         first_prediction = self.cluster.predict(vector)
145 |         vector2 = self.rotate_vec(
146 |             vector,
147 |             1.01 * np.arccos(self.subcluster_similarity_threshold))
148 |         second_prediction = self.cluster.predict(vector2)
149 |         # These asserts test that we have created 2 subclusters in the same cluster
150 |         assert first_prediction == second_prediction
151 |         assert len(self.cluster.clusters) == 1
152 |         assert len(self.cluster.clusters[0]) == 2
153 | 
154 |         # Merge subclusters and test for correctness
155 |         self.cluster.merge_subclusters(0, 0, 1)
156 |         assert len(self.cluster.clusters) == 1
157 |         assert len(self.cluster.clusters[0]) == 1
158 |         assert self.cluster.clusters[0][0].n_vectors == 2
159 |         assert len(self.cluster.clusters[0][0].input_vectors) == 2
160 |         np.testing.assert_array_almost_equal(
161 |             self.cluster.clusters[0][0].centroid,
162 |             np.mean([vector, vector2], axis=0))
163 | 
164 |     def test_get_all_vectors(self):
165 |         """Test that get_all_vectors returns all vectors."""
166 |         how_many = 100
167 |         for _ in range(how_many):
168 |             vector = self.random_vec()
169 |             self.cluster.predict(vector)
170 |         assert how_many == len(self.cluster.get_all_vectors())
171 | 
172 |     def test_sim_threshold_limit(self):
173 |         """Test that the limit for large k is near 1.0."""
174 |         large_k = 2 ** 25
175 |         thresh = self.cluster.sim_threshold(large_k, large_k)
176 |         assert np.abs(thresh - 1.0) < 1.0e-6
177 | 
178 | 
179 | class TestSubcluster:
180 |     """Tests for Subcluster class."""
181 |     def setup_method(self):
182 |         """Setup for tests."""
183 |         self.vector_dim = 256
184 |         self.initial_vector = self.random_vec()
185 |         self.subcluster = Subcluster(self.initial_vector, store_vectors=True)
186 | 
187 |     def random_vec(self):
188 |         """Generate random vector of the correct shape."""
189 |         return np.random.random((self.vector_dim))
190 | 
191 |     def test_init(self):
192 |         """Test basic class initialization properties."""
193 |         assert isinstance(self.subcluster, Subcluster)
194 |         assert self.subcluster.n_vectors == 1
195 |         assert len(self.subcluster.input_vectors) == 1
196 | 
197 |     def test_store_vector(self):
198 |         """Test that the input vectors are stored."""
199 |         assert np.array_equal(self.initial_vector, self.subcluster.input_vectors[0])
200 | 
201 |     def test_add_vector(self):
202 |         """Test that we can a new vector."""
203 |         new_vector = self.random_vec()
204 |         self.subcluster.add(new_vector)
205 |         assert self.subcluster.n_vectors == 2
206 |         assert len(self.subcluster.input_vectors) == 2
207 |         assert np.array_equal(self.subcluster.centroid,
208 |                               np.mean([self.initial_vector, new_vector],
209 |                                       axis=0))
210 | 
211 |     def test_add_multiple_vectors(self):
212 |         """Test that we can add multiple vectors."""
213 |         how_many = 10
214 |         new_vectors = np.random.random((how_many, self.vector_dim))
215 |         for i in range(how_many):
216 |             new_vector = new_vectors[i]
217 |             self.subcluster.add(new_vector)
218 |         expected_centroid = np.mean(
219 |             np.concatenate(
220 |                 [np.expand_dims(self.initial_vector, axis=0),
221 |                  new_vectors],
222 |                 axis=0
223 |             ),
224 |             axis=0
225 |         )
226 |         assert self.subcluster.n_vectors == how_many + 1
227 |         assert len(self.subcluster.input_vectors) == how_many + 1
228 |         np.testing.assert_array_almost_equal(
229 |             self.subcluster.centroid,
230 |             expected_centroid,
231 |             decimal=12)
232 | 
233 |     def test_merge(self):
234 |         """Test that subclusters can be merged."""
235 |         new_vector = self.random_vec()
236 |         new_subcluster = Subcluster(new_vector, store_vectors=True)
237 |         self.subcluster.connected_subclusters.add(new_subcluster)
238 |         new_subcluster.connected_subclusters.add(self.subcluster)
239 |         self.subcluster.merge(new_subcluster)
240 |         assert self.subcluster.n_vectors == 2
241 |         assert len(self.subcluster.input_vectors) == 2
242 | 
243 |     def test_merge_connections(self):
244 |         """Test that we can merge subclusters that have external edges."""
245 |         new_vector_1 = self.random_vec()
246 |         new_subcluster_1 = Subcluster(new_vector_1, store_vectors=True)
247 |         new_vector_2 = self.random_vec()
248 |         new_subcluster_2 = Subcluster(new_vector_2, store_vectors=True)
249 |         new_subcluster_1.connected_subclusters.update(
250 |             {new_subcluster_2, self.subcluster})
251 |         new_subcluster_2.connected_subclusters.update(
252 |             {new_subcluster_1, self.subcluster})
253 |         self.subcluster.connected_subclusters.update(
254 |             {new_subcluster_1, new_subcluster_2})
255 |         self.subcluster.merge(new_subcluster_2)
256 |         assert self.subcluster.n_vectors == 2
257 |         assert len(self.subcluster.input_vectors) == 2
258 |         assert len(self.subcluster.connected_subclusters) == 1
259 |         assert self.subcluster.connected_subclusters == {new_subcluster_1}
260 | 


--------------------------------------------------------------------------------