├── .vars.sh ├── DEPENDENCIES.md ├── LICENSE.md ├── README.md ├── cuda ├── defines.cu ├── fullbatch_loader.cu ├── highlight.cu ├── join.jcu ├── matrix_reduce.cu ├── mean_disp_normalizer.cu └── random.cu ├── deploy ├── deploy.sh ├── docker │ ├── Dockerfile │ ├── README │ └── setup-nvidia-devices.sh ├── init-pyenv ├── libsodium.patch ├── openpgm.patch ├── systemd │ ├── veles.forge_server.service │ └── veles.web_status.service ├── ubuntu.sh └── upstart │ ├── veles.forge_server.conf │ └── veles.web_status.conf ├── devices └── device_infos.json ├── docs ├── __init__.py ├── generate_units_args.py └── site_config.py ├── ocl ├── benchmark.cl ├── benchmark.store_output.cl ├── defines.cl ├── fullbatch_loader.cl ├── gemm.cl ├── gemm.store_output.cl ├── highlight.cl ├── join.jcl ├── matrix_multiplication.cl ├── matrix_multiplication_begin.cl ├── matrix_multiplication_end.cl ├── matrix_multiplication_precise.cl ├── matrix_multiplication_subsum.cl ├── matrix_reduce.cl ├── mean_disp_normalizer.cl └── random.cl ├── requirements-dev.2.txt ├── requirements-dev.3.txt ├── requirements.txt └── veles ├── __init__.py ├── __main__.py ├── accelerated_units.py ├── avatar.py ├── backends.py ├── client.py ├── cmdline.py ├── compat.py ├── config.py ├── distributable.py ├── dot_pip.py ├── downloader.py ├── dummy.py ├── ensemble ├── __init__.py ├── base_workflow.py ├── model_workflow.py └── test_workflow.py ├── error.py ├── external ├── __init__.py ├── daemon │ ├── LICENSE.PSF-2 │ ├── __init__.py │ ├── daemon.py │ └── version │ │ ├── __init__.py │ │ └── version_info.py ├── dot_parser.py ├── freetype │ ├── __init__.py │ ├── ft_enums │ │ ├── __init__.py │ │ ├── ft_encodings.py │ │ ├── ft_face_flags.py │ │ ├── ft_fstypes.py │ │ ├── ft_glyph_bbox_modes.py │ │ ├── ft_glyph_formats.py │ │ ├── ft_kerning_modes.py │ │ ├── ft_lcd_filters.py │ │ ├── ft_load_flags.py │ │ ├── ft_load_targets.py │ │ ├── ft_open_modes.py │ │ ├── ft_outline_flags.py │ │ ├── ft_pixel_modes.py │ │ ├── ft_render_modes.py │ │ ├── ft_stroker_borders.py │ │ ├── ft_stroker_linecaps.py │ │ ├── ft_stroker_linejoins.py │ │ ├── ft_style_flags.py │ │ ├── tt_adobe_ids.py │ │ ├── tt_apple_ids.py │ │ ├── tt_mac_ids.py │ │ ├── tt_mac_langids.py │ │ ├── tt_ms_ids.py │ │ ├── tt_ms_langids.py │ │ ├── tt_name_ids.py │ │ └── tt_platforms.py │ ├── ft_errors.py │ ├── ft_structs.py │ └── ft_types.py ├── fysom │ └── __init__.py ├── hog.py ├── kitchen │ ├── __init__.py │ └── text │ │ ├── __init__.py │ │ └── converters.py ├── manhole.py ├── prettytable.py ├── progressbar │ ├── __init__.py │ ├── compat.py │ ├── progressbar.py │ ├── spinner.py │ └── widgets.py ├── pydev │ ├── __init__.py │ ├── constants.py │ ├── dont_trace.py │ ├── exec1.py │ ├── exec2.py │ └── reload.py ├── pydot.py └── pytrie.py ├── forge ├── __init__.py ├── forge_client.py ├── forge_server.py └── templates │ ├── confirmation.html │ ├── email.html │ ├── email.txt │ ├── email_un.html │ ├── email_un.txt │ ├── not_registered.html │ ├── successful_registration.html │ ├── successful_unregistration.html │ └── unconfirmation.html ├── forge_common.py ├── genetics ├── __init__.py ├── config.py ├── core.py └── optimization_workflow.py ├── genre_recognition.xml ├── graphics_client.py ├── graphics_server.py ├── import_file.py ├── input_joiner.py ├── interaction.py ├── iplotter.py ├── json_encoders.py ├── launcher.py ├── loader ├── __init__.py ├── base.py ├── ensemble.py ├── file_image.py ├── file_loader.py ├── fullbatch.py ├── fullbatch_image.py ├── hdfs_loader.py ├── image.py ├── image_mse.py ├── interactive.py ├── libsndfile.py ├── libsndfile_loader.py ├── loader_hdf5.py ├── pickles.py ├── restful.py └── saver.py ├── logger.py ├── mapped_object_registry.py ├── mean_disp_normalizer.py ├── memory.py ├── mutable.py ├── network_common.py ├── normalization.py ├── numpy_ext.py ├── ocl_blas.py ├── opencl_types.py ├── paths.py ├── pickle2.py ├── plotter.py ├── plotting_units.py ├── plumbing.py ├── portable.py ├── prng ├── __init__.py ├── random_generator.py └── uniform.py ├── publishing ├── __init__.py ├── backend.py ├── confluence.py ├── confluence_backend.py ├── confluence_template.xml ├── ipynb_template.ipynb ├── jinja2_template_backend.py ├── latex_template.tex ├── markdown_backend.py ├── markdown_template.html ├── markdown_template.md ├── pdf_backend.py ├── publisher.py └── registry.py ├── restful_api.py ├── result_provider.py ├── samples ├── GeneticExample │ ├── __init__.py │ ├── genetics.py │ ├── genetics_config.py │ └── manifest.json └── __init__.py ├── scripts ├── __init__.py ├── bboxer.py ├── compare_snapshots.py ├── generate_frontend.py ├── update_forge.py └── valgrind-python.supp ├── server.py ├── snapshotter.py ├── thread_pool.py ├── timeit2.py ├── txzmq ├── __init__.py ├── connection.py ├── manager.py └── sharedio.py ├── unit_registry.py ├── units.py ├── verified.py ├── web_status.py ├── workflow.py ├── zmq_loader.py └── zope_verify_fix.py /.vars.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if test "$1" = "smudge"; then 4 | commit=$(git rev-parse HEAD) 5 | date=$(git log --pretty=format:"%ad" --date=rfc2822 -1) 6 | sed -e "s/\\\$Commit\\\$/$commit/g" -e "s/\\\$Date\\\$/$date/g" <&0 7 | cat <&0 8 | elif test "$1" = "clean"; then 9 | sed -e "s/\(__git__ = \"\).*/\\1\\\$Commit\\\$\"/g" -e "s/\\(__date__ = mktime_tz(parsedate_tz(\"\\).*/\\1\\\$Date\\\$\"))/g" <&0 10 | fi 11 | -------------------------------------------------------------------------------- /DEPENDENCIES.md: -------------------------------------------------------------------------------- 1 | Licenses of packages which Veles depends on 2 | =========================================== 3 | 4 | The table below was filled using the data obtained via the following script: 5 | 6 | ```sh 7 | pip install git+https://github.com/vmarkovtsev/yolk.git 8 | for d in $(cat requirements.txt); do yolk -f license,name -M $(echo "$d" | sed 's/[<>=]/ /g' | cut -d ' ' -f 1) && echo ""; done 9 | ``` 10 | 11 | | name | license | 12 | |-----------------------|--------------------------------------------| 13 | | matplotlib | BSD | 14 | | numpy | BSD | 15 | | scipy | BSD | 16 | | Pillow | Standard PIL License | 17 | | six | MIT | 18 | | tornado | Apache License, Version 2.0 | 19 | | motor | Apache License, Version 2.0 | 20 | | pymongo | Apache License, Version 2.0 | 21 | | Twisted | MIT | 22 | | ply | BSD | 23 | | paramiko | LGPL | 24 | | opencl4py | Simplified BSD | 25 | | argcomplete | Apache Software License | 26 | | ipython | BSD | 27 | | jpeg4py | Simplified BSD | 28 | | cffi | MIT | 29 | | Glymur | MIT | 30 | | lockfile | MIT | 31 | | python-snappy | BSD | 32 | | pycrypto | Public domain | 33 | | ecdsa | MIT | 34 | | pyzmq | LGPL, BSD -> BSD | 35 | | wget | Public Domain | 36 | | service_identity | MIT | 37 | | pygit2 | GPLv2 with linking exception | 38 | | pyinotify | MIT License | 39 | | cuda4py | Simplified BSD | 40 | | psutil | BSD | 41 | | pyxDamerauLevenshtein | BSD 3-Clause License | 42 | | h5py | BSD | 43 | | Jinja2 | BSD | 44 | | pycparser | BSD | 45 | | pyodbc | MIT | 46 | | progressbar (bundled) | LGPL, BSD -> BSD | 47 | | fysom (bundled) | BSD | 48 | | freetype (bundled) | BSD | 49 | | daemon (bundled) | PSF | 50 | | freetype (bundled) | BSD | 51 | | pydot (bundled) | MIT | 52 | | manhole (bundled) | BSD | 53 | | prettytable (bundled) | BSD | 54 | | pytrie (bundled) | BSD | 55 | | txzmq (rewritten) | Apache License, Version 2.0 (was GPL) | 56 | | xmltodict (bundled) | BSD | 57 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2013-2015 Samsung Electronics Co., Ltd. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Veles 2 | ===== 3 | Distributed platform for rapid Deep learning application development 4 | -------------------------------------------------------------------- 5 | Consists of: 6 | 7 | * Platform - https://github.com/Samsung/veles 8 | 9 | * Znicz Plugin - [Neural Network engine](https://github.com/Samsung/veles.znicz) 10 | 11 | * Mastodon - [Veles <-> Java bridge for Hadoop etc.](https://github.com/Samsung/veles.mastodon) 12 | 13 | * SoundFeatureExtraction - [audio feature extraction library](https://github.com/Samsung/veles.sound_feature_extraction) 14 | 15 | Home page: https://velesnet.ml 16 | 17 | Named after https://en.wikipedia.org/wiki/Veles_(god) 18 | 19 | Released under Apache 2.0 license. Copyright © Samsung Electronics Co., Ltd., 2013-2015. 20 | -------------------------------------------------------------------------------- /cuda/defines.cu: -------------------------------------------------------------------------------- 1 | #ifndef _DEFINES_ 2 | #define _DEFINES_ 3 | 4 | #include "highlight.cu" 5 | 6 | #ifndef dtype 7 | #define dtype int 8 | #endif 9 | 10 | typedef unsigned char uchar; 11 | 12 | #ifndef FLT_MAX 13 | #define FLT_MAX 1E+37 14 | #endif 15 | 16 | #define MIN(x, y) (((x) < (y)) ? (x) : (y)) 17 | 18 | #define SIGN(x) ((x) ? ((x) > 0 ? 1 : -1) : 0) 19 | 20 | /** 21 | * @brief CUDA implementation of atomicAdd for double. 22 | * 23 | * CUDA already has following implementations: 24 | * - int atomicAdd(int* address, int val); 25 | * - unsigned int atomicAdd(unsigned int* address, unsigned int val); 26 | * - unsigned long long int atomicAdd(unsigned long long int* address, 27 | * unsigned long long int val); 28 | * - float atomicAdd(float* address, float val); 29 | */ 30 | __device__ __forceinline__ double atomicAdd(double *address, double val) { 31 | unsigned long long int *address_as_ull = (unsigned long long int*)address; 32 | unsigned long long int old = *address_as_ull, assumed; 33 | do { 34 | assumed = old; 35 | old = atomicCAS(address_as_ull, assumed, 36 | __double_as_longlong(val + 37 | __longlong_as_double(assumed))); 38 | } while (assumed != old); 39 | return __longlong_as_double(old); 40 | } 41 | 42 | #endif // _DEFINES_ 43 | -------------------------------------------------------------------------------- /cuda/fullbatch_loader.cu: -------------------------------------------------------------------------------- 1 | #ifndef SAMPLE_SIZE 2 | #error "SAMPLE_SIZE should be defined" 3 | #endif 4 | #ifndef MAX_MINIBATCH_SIZE 5 | #error "MAX_MINIBATCH_SIZE should be defined" 6 | #endif 7 | 8 | 9 | extern "C" 10 | __global__ void fill_minibatch_data_labels( 11 | const original_data_dtype *original_data, 12 | minibatch_data_dtype *minibatch_data, 13 | const int start_offset, 14 | const int count, 15 | #if LABELS > 0 16 | const int *original_labels, 17 | int *minibatch_labels, 18 | #endif 19 | const int *shuffled_indices, 20 | int *minibatch_indices) { 21 | 22 | int idx = blockIdx.x * blockDim.x + threadIdx.x; 23 | int sample_number = idx / SAMPLE_SIZE; 24 | int real_sample_number = sample_number < count ? shuffled_indices[start_offset + sample_number] : -1; 25 | 26 | int offs_in_sample = idx % SAMPLE_SIZE; 27 | int offs_in_data = real_sample_number * SAMPLE_SIZE + offs_in_sample; 28 | int offs_in_minibatch = sample_number * SAMPLE_SIZE + offs_in_sample; 29 | 30 | if (idx < (MAX_MINIBATCH_SIZE * SAMPLE_SIZE)) { 31 | minibatch_data[offs_in_minibatch] = sample_number < count ? (minibatch_data_dtype)original_data[offs_in_data] : 0; 32 | #if LABELS > 0 33 | minibatch_labels[sample_number] = sample_number < count ? original_labels[real_sample_number] : -1; 34 | #endif 35 | minibatch_indices[sample_number] = real_sample_number; 36 | } 37 | } 38 | 39 | 40 | #if TARGET > 0 41 | extern "C" 42 | __global__ void fill_minibatch_target( 43 | const original_target_dtype *original_target, 44 | minibatch_target_dtype *minibatch_target, 45 | const int start_offset, 46 | const int count, 47 | int *shuffled_indices) { 48 | 49 | int idx = blockIdx.x * blockDim.x + threadIdx.x; 50 | int sample_number = idx / TARGET_SIZE; 51 | int real_sample_number = sample_number < count ? shuffled_indices[start_offset + sample_number] : -1; 52 | 53 | int offs_in_sample = idx % TARGET_SIZE; 54 | int offs_in_target = real_sample_number * TARGET_SIZE + offs_in_sample; 55 | int offs_in_minibatch = sample_number * TARGET_SIZE + offs_in_sample; 56 | 57 | if (idx < (MAX_MINIBATCH_SIZE * TARGET_SIZE)) { 58 | minibatch_target[offs_in_minibatch] = sample_number < count ? (minibatch_target_dtype)original_target[offs_in_target] : 0; 59 | } 60 | } 61 | #endif 62 | -------------------------------------------------------------------------------- /cuda/highlight.cu: -------------------------------------------------------------------------------- 1 | #ifndef HIGHTLIGHTING_CUH_ 2 | #define HIGHTLIGHTING_CUH_ 3 | 4 | #ifdef __CDT_PARSER__ 5 | 6 | #define __launch_bounds__(x) 7 | #define __restrict__ 8 | #define __device__ 9 | #define __global__ 10 | #define __shared__ 11 | #define __constant__ 12 | 13 | struct int2 { int x; int y; }; 14 | struct uint2 { unsigned int x; unsigned int y; }; 15 | struct long2 { long x; long y; }; 16 | struct ulong2 { long x; long y; }; 17 | struct float2 { float x; float y; }; 18 | 19 | struct int3 { int x; int y; int z; }; 20 | struct uint3 { unsigned int x; unsigned int y; unsigned int z; }; 21 | struct long3 { long x; long y; long z; }; 22 | struct ulong3 { long x; long y; long z; }; 23 | struct float3 { float x; float y; float z; }; 24 | 25 | struct int4 { int x; int y; int z; int w; }; 26 | struct uint4 { unsigned int x; unsigned int y; unsigned int z; unsigned int w; }; 27 | struct float4 { float x; float y; float z; float w; }; 28 | 29 | struct dim3 { unsigned int x; unsigned int y; unsigned int z; }; 30 | typedef dim3 gridDim; 31 | typedef dim3 blockDim; 32 | typedef uint3 blockIdx; 33 | typedef uint3 threadIdx; 34 | 35 | typedef int warpSize; 36 | 37 | void __syncthreads(); 38 | 39 | #endif // __CDT_PARSER__ 40 | 41 | #endif // HIGHTLIGHTING_CUH_ 42 | -------------------------------------------------------------------------------- /cuda/join.jcu: -------------------------------------------------------------------------------- 1 | #include "defines.cu" 2 | 3 | __device__ inline void copy(const etype *input, const int size, etype *output) { 4 | for (int i = 0; i < size; i++) { 5 | output[i] = input[i]; 6 | } 7 | } 8 | 9 | /// @brief Copies several buffers into one. 10 | extern "C" __global__ 11 | void join(etype *output 12 | {% for i in range(inputs|length) %} 13 | , const etype *input{{ i }} 14 | {% endfor %} 15 | ) { 16 | 17 | int index = blockIdx.x * blockDim.x + threadIdx.x; 18 | const int sizes[] = { 19 | {% for input in inputs %} 20 | {{ input.size // input.shape[0] }}, 21 | {% endfor %} 22 | }; 23 | const int inputs_number = {{ inputs|length }}; 24 | int output_size = 0; 25 | for (int i = 0; i < inputs_number; i++) { 26 | output_size += sizes[i]; 27 | } 28 | int output_offset = index * output_size; 29 | int input_size = 0; 30 | 31 | {% for i in range(inputs|length) %} 32 | input_size = sizes[{{ i }}]; 33 | copy(input{{ i }} + index * input_size, input_size, output + output_offset); 34 | output_offset += input_size; 35 | {% endfor %} 36 | } -------------------------------------------------------------------------------- /cuda/matrix_reduce.cu: -------------------------------------------------------------------------------- 1 | /* TODO(a.kazantsev): rewrite reduction as follows: 2 | * 3 | * Read more at: http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#ixzz3Pdbxsl5K 4 | __global__ void warpReduce() { 5 | int laneId = threadIdx.x & 0x1f; 6 | // Seed starting value as inverse lane ID 7 | int value = 31 - laneId; 8 | 9 | // Use XOR mode to perform butterfly reduction 10 | for (int i=16; i>=1; i/=2) 11 | value += __shfl_xor(value, i, 32); 12 | 13 | // "value" now contains the sum across all threads 14 | printf("Thread %d final value = %d\n", threadIdx.x, value); 15 | } 16 | */ 17 | 18 | /// @brief Define for reduce operation on matrix rows or columns. 19 | /// @author Kazantsev Alexey 20 | /// @details Sizes should be declared externally (values are given for example): 21 | /// #define REDUCE_SIZE 64 22 | /// #define A_WIDTH 10 23 | /// #define A_HEIGHT 100500 24 | /// 25 | /// As well as Matricies: 26 | /// #define A err_y 27 | /// 28 | /// And summation by columns if neccessary (otherwise summation by rows is assumed): 29 | /// #define A_COL 30 | /// 31 | /// size_t WorkSize[2] = {A_WIDTH * REDUCE_SIZE} or {A_HEIGHT * REDUCE_SIZE} #ifdef A_COL 32 | /// size_t LocalSize[2] = {REDUCE_SIZE} 33 | /// 34 | /// The result will be in (sum + AS[0]), output offset will be in bx, write it in if (tx == 0) { ... } 35 | __shared__ dtype AS[REDUCE_SIZE]; 36 | 37 | int bx = blockIdx.x; // from 0 to number of resulting output elements - 1 38 | int tx = threadIdx.x; // from 0 to REDUCE_SIZE - 1 39 | 40 | dtype sum = 0; 41 | 42 | #ifdef A_COL 43 | int offs = bx + tx * A_WIDTH; 44 | #define ARRAY_SIZE A_HEIGHT 45 | #define OFFS (REDUCE_SIZE * A_WIDTH) 46 | #else 47 | int offs = bx * A_WIDTH + tx; 48 | #define ARRAY_SIZE A_WIDTH 49 | #define OFFS REDUCE_SIZE 50 | #endif 51 | for (int i = 0; i < ARRAY_SIZE / REDUCE_SIZE; i++, offs += OFFS) { 52 | sum += A[offs]; 53 | } 54 | // Sum the remaining part 55 | #if (ARRAY_SIZE % REDUCE_SIZE) != 0 56 | if (tx < ARRAY_SIZE % REDUCE_SIZE) { 57 | sum += A[offs]; 58 | } 59 | #endif 60 | 61 | AS[tx] = sum; 62 | // ensure all shared loaded 63 | __syncthreads(); 64 | 65 | // Final summation 66 | sum = 0; 67 | int n = MIN(ARRAY_SIZE, REDUCE_SIZE); 68 | while (n > 1) { 69 | sum += (n & 1) ? AS[n - 1] : 0; 70 | n >>= 1; 71 | if (tx < n) { 72 | AS[tx] += AS[n + tx]; 73 | } 74 | // ensure all shared summed 75 | __syncthreads(); 76 | } 77 | 78 | #undef OFFS 79 | #undef ARRAY_SIZE 80 | 81 | // The result will be in (sum + AS[0]), output offset will be in bx, write it in if (tx == 0) { ... } 82 | 83 | /// Define for reduce operation on matrix rows or columns ends here. 84 | -------------------------------------------------------------------------------- /cuda/mean_disp_normalizer.cu: -------------------------------------------------------------------------------- 1 | #include "defines.cu" 2 | 3 | /* 4 | @brief Normalizes array of images according to mean and dispersion. 5 | @param input Array of the images. 6 | @param mean Image with the mean over the dataset. 7 | @param rdisp Image with the 1.0 / dispersion over the dataset. 8 | @param output Array of the output images. 9 | @details output[:] = ((dtype)input[:] - (dtype)mean) * rdisp. 10 | */ 11 | extern "C" 12 | __global__ void normalize_mean_disp(const input_type /* IN */ *input, 13 | const mean_type /* IN */ *mean, 14 | const dtype /* IN */ *rdisp, 15 | dtype /* OUT */ *output) { 16 | int offs_in_sample = blockIdx.x * blockDim.x + threadIdx.x; 17 | int offs = (blockIdx.y * blockDim.y + threadIdx.y) * SAMPLE_SIZE + offs_in_sample; 18 | output[offs] = ((dtype)input[offs] - (dtype)mean[offs_in_sample]) * rdisp[offs_in_sample]; 19 | } 20 | -------------------------------------------------------------------------------- /deploy/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Vadim Markovtsev 3 | 4 | # Install Veles 5 | RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ 6 | DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ 7 | DEBIAN_FRONTEND=noninteractive apt-get install -y wget 8 | RUN echo "deb https://velesnet.ml/apt trusty main" >> /etc/apt/sources.list && \ 9 | wget -O - http://velesnet.ml/apt/velesnet.ml.gpg.key 2> /dev/null | apt-key add - && \ 10 | DEBIAN_FRONTEND=noninteractive apt-get update && \ 11 | DEBIAN_FRONTEND=noninteractive apt-get install -y python3-veles 12 | 13 | RUN apt-get clean 14 | 15 | ENTRYPOINT ["bash"] 16 | -------------------------------------------------------------------------------- /deploy/docker/README: -------------------------------------------------------------------------------- 1 | --------------------------------------------------------- 2 | NB! 3 | If you want to build veles container with GPU usage, 4 | you should copy cuda_X.X.X_linux_XX.run to this directory 5 | (docker can copy files only from directory with Dockerfile). 6 | --------------------------------------------------------- 7 | 8 | To build Veles docker image use command: 9 | $ docker build -t samsung/veles 10 | 11 | To run docker container to test image use command: 12 | $ docker run --rm --name veles samsung/veles 13 | ^ ^ ^ 14 | Delete Container Container Image 15 | after Usage Name Name 16 | OR 17 | 18 | $ ./setup-nvidia-devices.sh 19 | $ export DEVICE_OPTIONS="--device=/dev/nvidia0:/dev/nvidia0 --device=/dev/nvidiactl:/dev/nvidiactl --device=/dev/nvidia-uvm:/dev/nvidia-uvm" 20 | $ docker run --rm --name veles ${DEVICE_OPTIONS} samsung/veles 21 | 22 | You can change the number of devices in "/dev/nvidia0". 23 | If these /dev/* files aren't in your system, you should run script "setup-nvidia-devices.sh" before running containers. 24 | 25 | To run docker container with specified options, workflow and config use command: 26 | $ ./setup-nvidia-devices.sh 27 | $ export DEVICE_OPTIONS="--device=/dev/nvidia0:/dev/nvidia0 --device=/dev/nvidiactl:/dev/nvidiactl --device=/dev/nvidia-uvm:/dev/nvidia-uvm" 28 | $ docker run --rm --name veles ${DEVICE_OPTIONS} samsung/veles [OPTIONS] [path-to-workflow] [path-to-config] 29 | -------------------------------------------------------------------------------- /deploy/docker/setup-nvidia-devices.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Setup CUDA devices. 3 | /sbin/modprobe nvidia 4 | 5 | if [ "$?" -eq 0 ]; then 6 | # Count the number of NVIDIA controllers found. 7 | NVDEVS=`lspci | grep -i NVIDIA` 8 | N3D=`echo "$NVDEVS" | grep "3D controller" | wc -l` 9 | NVGA=`echo "$NVDEVS" | grep "VGA compatible controller" | wc -l` 10 | N=`expr $N3D + $NVGA - 1` 11 | 12 | for i in `seq 0 $N`; do 13 | mknod -m 666 /dev/nvidia$i c 195 $i 14 | done 15 | 16 | mknod -m 666 /dev/nvidiactl c 195 255 17 | else 18 | exit 1 19 | fi 20 | 21 | /sbin/modprobe nvidia-uvm 22 | 23 | if [ "$?" -eq 0 ]; then 24 | # Find out the major device number used by the nvidia-uvm driver 25 | D=`grep nvidia-uvm /proc/devices | awk '{print $1}'` 26 | mknod -m 666 /dev/nvidia-uvm c $D 0 27 | else 28 | exit 1 29 | fi 30 | -------------------------------------------------------------------------------- /deploy/init-pyenv: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -z "$PYENV_ROOT" ]; then 4 | if [ -n "$BASH" ]; then 5 | # $0 = "bash", try to use pwd 6 | if [ -e $(pwd)/pyenv ]; then 7 | export PYENV_ROOT=$(pwd)/pyenv 8 | else 9 | export PYENV_ROOT="" 10 | echo "Bash users must export PYENV_ROOT manually." 1>&2 11 | fi 12 | else 13 | export PYENV_ROOT=$(dirname $(readlink -f $0))/pyenv 14 | fi 15 | fi 16 | if [ -n "$PYENV_ROOT" ]; then 17 | 18 | export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" 19 | export PYENV_SHELL=bash 20 | if [ -n "$BASH" ]; then 21 | . "$PYENV_ROOT/completions/pyenv.bash" 22 | fi 23 | pyenv rehash 2>/dev/null 24 | 25 | pyenv() { 26 | local command 27 | command="$1" 28 | if [ "$#" -gt 0 ]; then 29 | shift 30 | fi 31 | 32 | case "$command" in 33 | rehash|shell) 34 | eval "`pyenv "sh-$command" "$@"`";; 35 | *) 36 | command pyenv "$command" "$@";; 37 | esac 38 | } 39 | 40 | fi # -n "$PYENV_ROOT" -------------------------------------------------------------------------------- /deploy/libsodium.patch: -------------------------------------------------------------------------------- 1 | --- configure.ac 2014-10-08 17:48:22.654942849 +0400 2 | +++ configure.ac.patched 2014-10-08 17:48:44.218943175 +0400 3 | @@ -1,4 +1,4 @@ 4 | -AC_PREREQ([2.65]) 5 | +AC_PREREQ([2.63]) 6 | AC_INIT([libsodium],[1.0.0], 7 | [https://github.com/jedisct1/libsodium/issues], 8 | [libsodium], 9 | @@ -110,23 +110,23 @@ 10 | ]) 11 | ]) 12 | 13 | -AC_ARG_ENABLE(debug, 14 | -[AS_HELP_STRING(--enable-debug,For maintainers only - please do not use)], 15 | -[ 16 | - AS_IF([test "x$enableval" = "xyes"], [ 17 | - AS_IF([test "x$LX_CFLAGS" = "xNONE"], [ 18 | - nxflags="" 19 | - for flag in `echo $CFLAGS`; do 20 | - AS_CASE([$flag], 21 | - [-O*], [ ], 22 | - [-g*], [ ], 23 | - [*], [AS_VAR_APPEND([nxflags], [" $flag"])]) 24 | - done 25 | - CFLAGS="$nxflags -O0 -g3" 26 | - ]) 27 | - CPPFLAGS="$CPPFLAGS -DDEBUG=1" 28 | - ]) 29 | -]) 30 | +#AC_ARG_ENABLE(debug, 31 | +#[AS_HELP_STRING(--enable-debug,For maintainers only - please do not use)], 32 | +#[ 33 | +# AS_IF([test "x$enableval" = "xyes"], [ 34 | +# AS_IF([test "x$LX_CFLAGS" = "xNONE"], [ 35 | +# nxflags="" 36 | +# for flag in `echo $CFLAGS`; do 37 | +# AS_CASE([$flag], 38 | +# [-O*], [ ], 39 | +# [-g*], [ ], 40 | +# [*], [AS_VAR_APPEND([nxflags], [" $flag"])]) 41 | +# done 42 | +# CFLAGS="$nxflags -O0 -g3" 43 | +# ]) 44 | +# CPPFLAGS="$CPPFLAGS -DDEBUG=1" 45 | +# ]) 46 | +#]) 47 | 48 | AC_SUBST([MAINT]) 49 | 50 | -------------------------------------------------------------------------------- /deploy/openpgm.patch: -------------------------------------------------------------------------------- 1 | --- if.c 2014-09-22 18:54:48.287237648 +0400 2 | +++ if.patched.c 2014-09-22 18:56:29.999241071 +0400 3 | @@ -413,6 +413,7 @@ 4 | #if defined(EAI_NODATA) && EAI_NODATA != EAI_NONAME 5 | case EAI_NODATA: 6 | #endif 7 | + case EAI_AGAIN: 8 | case EAI_NONAME: 9 | break; 10 | 11 | -------------------------------------------------------------------------------- /deploy/systemd/veles.forge_server.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Veles Forge Server 3 | 4 | [Service] 5 | Type=simple 6 | Environment="FORGE_SERVER_ARGS=" 7 | EnvironmentFile=-/etc/default/veles.forge_server 8 | 9 | ExecStartPre=-/bin/mkdir -p -m0755 /var/run/veles /var/log/veles 10 | ConditionFileIsExecutable=/usr/lib/python3/dist-packages/veles/forge/forge_server.py 11 | 12 | ExecStart=/usr/lib/python3/dist-packages/veles/forge_server.py $FORGE_SERVER_ARGS 13 | PIDFile=/var/run/veles/forge_server.lock 14 | 15 | Restart=on-abort 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /deploy/systemd/veles.web_status.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Veles Web Status Server 3 | 4 | [Service] 5 | Type=simple 6 | Environment="WEB_STATUS_ARGS=" 7 | EnvironmentFile=-/etc/default/veles.web_status 8 | 9 | ExecStartPre=-/bin/mkdir -p -m0755 /var/run/veles /var/log/veles 10 | ConditionFileIsExecutable=/usr/lib/python3/dist-packages/veles/web_status.py 11 | 12 | ExecStart=/usr/lib/python3/dist-packages/veles/web_status.py $WEB_STATUS_ARGS 13 | PIDFile=/var/run/veles/web_status.lock 14 | 15 | Restart=on-abort 16 | 17 | [Install] 18 | WantedBy=multi-user.target 19 | -------------------------------------------------------------------------------- /deploy/ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ ! -e /etc/debian_version ]; then 4 | echo "Only Debian-based distros are supported by this script." 5 | fi 6 | 7 | uver=$(lsb_release -cs) 8 | 9 | if [ $(id -u) != 0 ]; then 10 | echo "Requesting sudo..." 11 | sudo echo "Thanks" 12 | fi 13 | 14 | wget -O - https://velesnet.ml/apt/velesnet.ml.gpg.key | sudo apt-key add - 15 | if ! grep -q "deb [arch=amd64] https://velesnet.ml/apt $uver main" /etc/apt/sources.list; then 16 | echo "deb [arch=amd64] https://velesnet.ml/apt $uver main" | sudo tee -a /etc/apt/sources.list 17 | fi 18 | sudo apt-get update 19 | sudo apt-get install python3-veles -------------------------------------------------------------------------------- /deploy/upstart/veles.forge_server.conf: -------------------------------------------------------------------------------- 1 | description "Script to run Veles Forge server" 2 | author "Vadim Markovtsev, Dmitry Senin" 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | respawn limit 10 5 9 | 10 | env FORGE_SERVER=/usr/lib/python3/dist-packages/veles/forge/forge_server.py 11 | env FORGE_SERVER_ARGS= 12 | env DEFAULT_FORGE_SERVER=veles.forge_server 13 | env PYTHON=/usr/bin/python3 14 | env HOME=/tmp/veles_forge_cache 15 | 16 | setuid www-data 17 | setgid www-data 18 | 19 | pre-start script 20 | test -e $FORGE_SERVER || { echo "$FORGE_SERVER does not exist"; stop; exit 0; } 21 | end script 22 | 23 | script 24 | if [ -f /etc/default/$DEFAULT_FORGE_SERVER ]; then 25 | . /etc/default/$DEFAULT_FORGE_SERVER 26 | fi 27 | exec "$PYTHON" "$FORGE_SERVER" $FORGE_SERVER_ARGS 28 | end script 29 | -------------------------------------------------------------------------------- /deploy/upstart/veles.web_status.conf: -------------------------------------------------------------------------------- 1 | description "Script to run Veles Web status server" 2 | author "Vadim Markovtsev, Dmitry Senin" 3 | 4 | start on runlevel [2345] 5 | stop on runlevel [!2345] 6 | 7 | respawn 8 | respawn limit 10 5 9 | 10 | env WEB_STATUS=/usr/lib/python3/dist-packages/veles/web_status.py 11 | env WEB_STATUS_ARGS= 12 | env PYTHON=/usr/bin/python3 13 | env USER=www-data 14 | env GROUP=www-data 15 | env DEFAULT_WEB_STATUS=veles.web_status 16 | env HOME=/tmp/veles_web_cache 17 | env VELES_ALLOW_ROOT="" 18 | 19 | pre-start script 20 | test -e $WEB_STATUS || { echo "$WEB_STATUS does not exist"; stop; exit 0; } 21 | install -o $USER -g $GROUP -m 0755 -d /var/run/veles /var/log/veles 22 | end script 23 | 24 | script 25 | if [ -e /etc/default/$DEFAULT_WEB_STATUS ]; then 26 | . /etc/default/$DEFAULT_WEB_STATUS 27 | fi 28 | exec "$PYTHON" "$WEB_STATUS" --user $USER --group $GROUP $WEB_STATUS_ARGS 29 | end script -------------------------------------------------------------------------------- /devices/device_infos.json: -------------------------------------------------------------------------------- 1 | { 2 | "NVIDIA Corporation/GeForce GTX TITAN": { 3 | "matrix_multiplication": { 4 | "double": { 5 | "0": [ 6 | 31, 7 | false, 8 | 0.3395437159924768 9 | ], 10 | "1": [ 11 | 31, 12 | false, 13 | 0.3452338406738515 14 | ], 15 | "2": [ 16 | 31, 17 | false, 18 | 0.7053155820079459 19 | ] 20 | }, 21 | "float": { 22 | "0": [ 23 | 30, 24 | false, 25 | 0.16424041400508335 26 | ], 27 | "1": [ 28 | 30, 29 | false, 30 | 0.1728648446733132 31 | ], 32 | "2": [ 33 | 30, 34 | false, 35 | 0.3108249586657621 36 | ] 37 | } 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /docs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Samsung/veles/96218b8d04ab099d65bfff350a24412a66c26178/docs/__init__.py -------------------------------------------------------------------------------- /docs/generate_units_args.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # encoding: utf-8 3 | 4 | 5 | from jinja2 import Environment, FileSystemLoader 6 | import os 7 | import sys 8 | 9 | import veles 10 | from veles.loader.base import UserLoaderRegistry 11 | from veles.logger import Logger 12 | from veles.plotter import Plotter 13 | from veles.workflow import Workflow 14 | from veles.unit_registry import UnitRegistry 15 | from veles.units import Unit 16 | from veles.znicz.activation import Activation 17 | from veles.znicz.nn_units import ForwardBase, GradientDescentBase,\ 18 | MatchingObject 19 | 20 | 21 | docs_source_dir = os.path.join(os.path.dirname(__file__), "source") 22 | result_file_name_base = "manualrst_veles_units_kwargs" 23 | 24 | 25 | class UnitsKeywordArgumentsGenerator(Logger): 26 | @staticmethod 27 | def get_condition_loader_with_labels(unit): 28 | return (isinstance(unit, UserLoaderRegistry) 29 | and str(unit).find("MSE") < 0) 30 | 31 | @staticmethod 32 | def get_condition_loader_with_targets(unit): 33 | return (isinstance(unit, UserLoaderRegistry) 34 | and str(unit).find("MSE") > 0) 35 | 36 | @staticmethod 37 | def get_condition_forwards(unit): 38 | return (isinstance(unit, MatchingObject) and 39 | issubclass(unit, ForwardBase) and 40 | not issubclass(unit, Activation)) 41 | 42 | @staticmethod 43 | def get_condition_backwards(unit): 44 | return (isinstance(unit, MatchingObject) and 45 | issubclass(unit, GradientDescentBase) and 46 | not issubclass(unit, Activation)) 47 | 48 | @staticmethod 49 | def get_condition_forward_activations(unit): 50 | return (isinstance(unit, MatchingObject) and 51 | issubclass(unit, ForwardBase) and 52 | issubclass(unit, Activation)) 53 | 54 | @staticmethod 55 | def get_condition_backward_activations(unit): 56 | return (isinstance(unit, MatchingObject) and 57 | issubclass(unit, GradientDescentBase) and 58 | issubclass(unit, Activation)) 59 | 60 | @staticmethod 61 | def get_condition_plotters(unit): 62 | return issubclass(unit, Plotter) 63 | 64 | @staticmethod 65 | def get_condition_kohonen(unit): 66 | return str(unit).find("kohonen") > 0 67 | 68 | @staticmethod 69 | def get_condition_rbm(unit): 70 | return str(unit).find("rbm") > 0 71 | 72 | @staticmethod 73 | def get_condition_working_units(unit): 74 | return (str(unit).find("rbm") < 0 and str(unit).find("kohonen") < 0 and 75 | not issubclass(unit, Plotter) and 76 | not issubclass(unit, Activation) and 77 | not issubclass(unit, GradientDescentBase) and 78 | not issubclass(unit, ForwardBase) and 79 | not isinstance(unit, UserLoaderRegistry)) 80 | 81 | @staticmethod 82 | def get_condition_base_units(unit): 83 | return not issubclass(unit, Workflow) 84 | 85 | def run(self, output_path): 86 | def usorted(units): 87 | return sorted(units, key=lambda u: str(u)) 88 | 89 | Environment(trim_blocks=True, lstrip_blocks=True, 90 | loader=FileSystemLoader(docs_source_dir)) \ 91 | .get_template(result_file_name_base + ".jrst") \ 92 | .stream(origin=self, root_kwattrs=Unit.KWATTRS, 93 | all_units=usorted(veles.__units__), 94 | hidden_units=usorted(UnitRegistry.hidden_units)) \ 95 | .dump(output_path, encoding="utf-8") 96 | 97 | 98 | if __name__ == "__main__": 99 | sys.exit(UnitsKeywordArgumentsGenerator().run(sys.argv[1])) 100 | -------------------------------------------------------------------------------- /docs/site_config.py: -------------------------------------------------------------------------------- 1 | import veles 2 | 3 | veles.allow_root = True 4 | 5 | 6 | def update(_): 7 | pass 8 | -------------------------------------------------------------------------------- /ocl/benchmark.cl: -------------------------------------------------------------------------------- 1 | #include "defines.cl" 2 | 3 | __kernel __attribute__((reqd_work_group_size(BLOCK_SIZE, BLOCK_SIZE, 1))) 4 | void benchmark(__global dtype *A, __global dtype *B, __global dtype *C) { 5 | #define A_WIDTH SIZE 6 | #define B_WIDTH SIZE 7 | #define AB_COMMON SIZE 8 | 9 | #define STORE_OUTPUT "benchmark.store_output.cl" 10 | #include "matrix_multiplication.cl" 11 | } 12 | -------------------------------------------------------------------------------- /ocl/benchmark.store_output.cl: -------------------------------------------------------------------------------- 1 | // Just store the computed sum in the target matrix 2 | C[idx] = sum; 3 | -------------------------------------------------------------------------------- /ocl/defines.cl: -------------------------------------------------------------------------------- 1 | /* 2 | * General definitions. 3 | */ 4 | 5 | #ifndef _DEFINES_ 6 | #define _DEFINES_ 7 | 8 | 9 | /// @brief Pragmas for features. 10 | #if (sizeof_dtype == 8) && (__OPENCL_VERSION__ < 120) 11 | #pragma OPENCL EXTENSION cl_khr_fp64: enable 12 | #endif 13 | 14 | 15 | /// @brief Minimum of the two values. 16 | #define MIN(x, y) (((x) < (y)) ? (x) : (y)) 17 | 18 | 19 | #ifdef USE_ATOMICS 20 | 21 | 22 | /// @brief atom_add for float. 23 | inline float atom_add_float(__global float *addr, float vle) { 24 | int sum = *(__global int*)addr; 25 | int oldsum; 26 | do { 27 | oldsum = sum; 28 | float new_sum = *(float*)&sum + vle; 29 | sum = atomic_cmpxchg((__global volatile int *)addr, oldsum, *(int*)&new_sum); 30 | } 31 | while (sum != oldsum); 32 | return *(float*)∑ 33 | } 34 | 35 | 36 | /// @brief atom_add for double. 37 | #if sizeof_dtype == 8 38 | #pragma OPENCL EXTENSION cl_khr_int64_base_atomics: enable 39 | inline double atom_add_double(__global double *addr, double vle) { 40 | long sum = *(__global long*)addr; 41 | long oldsum; 42 | do { 43 | oldsum = sum; 44 | double new_sum = *(double*)&sum + vle; 45 | sum = atom_cmpxchg((__global volatile long *)addr, oldsum, *(long*)&new_sum); 46 | } 47 | while (sum != oldsum); 48 | return *(double*)∑ 49 | } 50 | #endif 51 | 52 | 53 | #if sizeof_dtype == 4 54 | #define ATOM_ADD(addr, vle) atom_add_float(addr, vle) 55 | #elif sizeof_dtype == 8 56 | #define ATOM_ADD(addr, vle) atom_add_double(addr, vle) 57 | #else 58 | #error Unsupported number type. 59 | #endif 60 | 61 | 62 | #endif // USE_ATOMICS 63 | 64 | 65 | /// @brief Sets all elements of array to zero. 66 | #define KERNEL_CLEAR(kernel_name, data_type) __kernel void kernel_name(__global data_type *arr) { arr[get_global_id(0)] = 0; } 67 | 68 | 69 | #endif // _DEFINES_ 70 | -------------------------------------------------------------------------------- /ocl/fullbatch_loader.cl: -------------------------------------------------------------------------------- 1 | #include "defines.cl" 2 | #include "highlight.cl" 3 | 4 | 5 | __kernel void fill_minibatch_data_labels( 6 | __global const original_data_dtype /* IN */ *original_data, 7 | __global minibatch_data_dtype /* OUT */ *minibatch_data, 8 | const int /* IN */ start_offset, 9 | const int /* IN */ count, 10 | #if LABELS > 0 11 | __global const int /* IN */ *original_labels, 12 | __global int /* OUT */ *minibatch_labels, 13 | #endif 14 | __global const int /* IN */ *shuffled_indices, 15 | __global int /* OUT */ *minibatch_indices) { 16 | 17 | int sample_number = get_global_id(0); 18 | int real_sample_number = sample_number < count ? shuffled_indices[start_offset + sample_number] : -1; 19 | 20 | int offs_in_sample = get_global_id(1); 21 | int offs_in_data = real_sample_number * SAMPLE_SIZE + offs_in_sample; 22 | int offs_in_minibatch = sample_number * SAMPLE_SIZE + offs_in_sample; 23 | 24 | minibatch_data[offs_in_minibatch] = sample_number < count ? (minibatch_data_dtype)original_data[offs_in_data] : 0; 25 | #if LABELS > 0 26 | minibatch_labels[sample_number] = sample_number < count ? original_labels[real_sample_number] : -1; 27 | #endif 28 | minibatch_indices[sample_number] = real_sample_number; 29 | } 30 | 31 | 32 | #if TARGET > 0 33 | __kernel void fill_minibatch_target( 34 | __global const original_target_dtype /* IN */ *original_target, 35 | __global minibatch_target_dtype /* OUT */ *minibatch_target, 36 | const int /* IN */ start_offset, 37 | const int /* IN */ count, 38 | __global int /* IN */ *shuffled_indices) { 39 | 40 | int sample_number = get_global_id(0); 41 | int real_sample_number = sample_number < count ? shuffled_indices[start_offset + sample_number] : -1; 42 | 43 | int offs_in_sample = get_global_id(1); 44 | int offs_in_target = real_sample_number * TARGET_SIZE + offs_in_sample; 45 | int offs_in_minibatch = sample_number * TARGET_SIZE + offs_in_sample; 46 | 47 | minibatch_target[offs_in_minibatch] = sample_number < count ? (minibatch_target_dtype)original_target[offs_in_target] : 0; 48 | } 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /ocl/gemm.cl: -------------------------------------------------------------------------------- 1 | #include "defines.cl" 2 | #include "highlight.cl" 3 | 4 | /// @brief C = A * B * alpha + C * beta. 5 | __kernel __attribute__((reqd_work_group_size(BLOCK_SIZE, BLOCK_SIZE, 1))) 6 | void gemm(__global const dtype *A, __global const dtype *B, __global dtype *C, 7 | const dtype alpha, const dtype beta, 8 | const ulong offsetA, const ulong offsetB, const ulong offsetC) { 9 | A += (size_t)offsetA; 10 | B += (size_t)offsetB; 11 | C += (size_t)offsetC; 12 | #define STORE_OUTPUT "gemm.store_output.cl" 13 | #include "matrix_multiplication.cl" 14 | } 15 | -------------------------------------------------------------------------------- /ocl/gemm.store_output.cl: -------------------------------------------------------------------------------- 1 | // Store the gemm result in the output matrix 2 | sum *= alpha; 3 | if (beta) { 4 | sum += C[idx] * beta; 5 | } 6 | C[idx] = sum; 7 | -------------------------------------------------------------------------------- /ocl/highlight.cl: -------------------------------------------------------------------------------- 1 | #ifndef _HIGHLIGHT_CL_ 2 | #define _HIGHLIGHT_CL_ 3 | 4 | // This is a hack to fix highlighting of OpenCL *.cl files without errors 5 | #ifndef __OPENCL_VERSION__ 6 | #define __kernel 7 | #define __global 8 | #define __local 9 | #endif // __OPENCL_VERSION__ 10 | 11 | #endif // _HIGHLIGHT_CL_ 12 | -------------------------------------------------------------------------------- /ocl/join.jcl: -------------------------------------------------------------------------------- 1 | #include "defines.cl" 2 | #include "highlight.cl" 3 | 4 | 5 | inline void copy(const __global etype *input, const int size, 6 | __global etype *output) { 7 | for (int i = 0; i < size; i++) { 8 | output[i] = input[i]; 9 | } 10 | } 11 | 12 | /// @brief Copies several buffers into one. 13 | __kernel 14 | void join(__global etype *output 15 | {% for i in range(inputs|length) %} 16 | , __global const etype *input{{ i }} 17 | {% endfor %} 18 | ) { 19 | 20 | int index = get_global_id(0); 21 | const int sizes[] = { 22 | {% for input in inputs %} 23 | {{ input.size // input.shape[0] }}, 24 | {% endfor %} 25 | }; 26 | const int inputs_number = {{ inputs|length }}; 27 | int output_size = 0; 28 | for (int i = 0; i < inputs_number; i++) { 29 | output_size += sizes[i]; 30 | } 31 | int output_offset = index * output_size; 32 | int input_size = 0; 33 | 34 | {% for i in range(inputs|length) %} 35 | input_size = sizes[{{ i }}]; 36 | copy(input{{ i }} + index * input_size, input_size, output + output_offset); 37 | output_offset += input_size; 38 | {% endfor %} 39 | } 40 | -------------------------------------------------------------------------------- /ocl/matrix_multiplication.cl: -------------------------------------------------------------------------------- 1 | #include "matrix_multiplication_precise.cl" 2 | -------------------------------------------------------------------------------- /ocl/matrix_multiplication_begin.cl: -------------------------------------------------------------------------------- 1 | // Check for required defines 2 | #ifndef BLOCK_SIZE 3 | #error "BLOCK_SIZE MUST be defined" 4 | #endif 5 | #ifndef A_WIDTH 6 | #error "A_WIDTH MUST be defined" 7 | #endif 8 | #ifndef B_WIDTH 9 | #error "B_WIDTH MUST be defined" 10 | #endif 11 | #ifndef AB_COMMON 12 | #error "AB_COMMON MUST be defined" 13 | #endif 14 | #ifndef STORE_OUTPUT 15 | #error "STORE_OUTPUT MUST be defined" 16 | #endif 17 | 18 | // Support for multiple includes 19 | #ifdef ALIGNED 20 | #undef ALIGNED 21 | #endif 22 | 23 | // Set default precision level 24 | #ifndef PRECISION_LEVEL 25 | #define PRECISION_LEVEL 0 26 | #endif 27 | 28 | // Choose to vectorize block sum or not 29 | #ifndef VECTOR_OPT 30 | #define VECTOR_OPT 0 31 | #endif 32 | #if (VECTOR_OPT <= 0) || (BLOCK_SIZE % 4 != 0) 33 | #define BLOCK_SUM_VECTORIZED 0 34 | #else 35 | #ifndef MULTIPLY 36 | #define BLOCK_SUM_VECTORIZED 1 37 | #else 38 | #define BLOCK_SUM_VECTORIZED 0 39 | #endif 40 | #endif 41 | 42 | // Compute number of blocks at the common matrix side 43 | #if (AB_COMMON % BLOCK_SIZE) == 0 44 | #define N_BLOCKS (AB_COMMON / BLOCK_SIZE) 45 | #if ((A_WIDTH % BLOCK_SIZE) == 0) && ((B_WIDTH % BLOCK_SIZE) == 0) 46 | #define ALIGNED 47 | #endif 48 | #else 49 | #define N_BLOCKS (AB_COMMON / BLOCK_SIZE + 1) 50 | #endif 51 | 52 | // Define default offsets 53 | #ifndef A_REAL_OFFS 54 | #define A_REAL_OFFS a_offs 55 | #endif 56 | #ifndef B_REAL_OFFS 57 | #define B_REAL_OFFS b_offs 58 | #endif 59 | #ifndef A_REAL_OFFS_VALID 60 | #define A_REAL_OFFS_VALID 1 61 | #endif 62 | #ifndef B_REAL_OFFS_VALID 63 | #define B_REAL_OFFS_VALID 1 64 | #endif 65 | -------------------------------------------------------------------------------- /ocl/matrix_multiplication_end.cl: -------------------------------------------------------------------------------- 1 | // Undefine defined values 2 | #ifdef OUT_OFFS_VALID 3 | #undef OUT_OFFS_VALID 4 | #endif 5 | #ifdef OUT_OFFS_INC 6 | #undef OUT_OFFS_INC 7 | #endif 8 | #ifdef bs_idx 9 | #undef bs_idx 10 | #endif 11 | #ifdef A_LIMIT 12 | #undef A_LIMIT 13 | #endif 14 | #ifdef A_LIMIT_X 15 | #undef A_LIMIT_X 16 | #endif 17 | #ifdef A_INC_X 18 | #undef A_INC_X 19 | #endif 20 | #ifdef A_OFFS 21 | #undef A_OFFS 22 | #endif 23 | #undef A_REAL_OFFS 24 | #undef A_REAL_OFFS_VALID 25 | #ifdef B_LIMIT 26 | #undef B_LIMIT 27 | #endif 28 | #ifdef B_LIMIT_X 29 | #undef B_LIMIT_X 30 | #endif 31 | #ifdef B_INC_X 32 | #undef B_INC_X 33 | #endif 34 | #ifdef B_OFFS 35 | #undef B_OFFS 36 | #endif 37 | #undef B_REAL_OFFS 38 | #undef B_REAL_OFFS_VALID 39 | #undef N_BLOCKS 40 | -------------------------------------------------------------------------------- /ocl/matrix_multiplication_subsum.cl: -------------------------------------------------------------------------------- 1 | // This file is included from matrix_multiplication.cl 2 | 3 | #if BLOCK_SUM_VECTORIZED == 0 4 | // Plain sumation 5 | __local dtype *_AS = &AS[as_start]; 6 | __local dtype *_BS = &BS[bs_start]; 7 | dtype block_sum = 0; 8 | #ifdef MULTIPLY 9 | #pragma unroll 10 | for (int k = 0; k < BLOCK_SIZE; k++) { 11 | block_sum += MULTIPLY(_AS[k], _BS[k]); 12 | } 13 | #else 14 | #pragma unroll 15 | for (int k = 0; k < BLOCK_SIZE; k++) { 16 | block_sum += _AS[k] * _BS[k]; 17 | } 18 | #endif 19 | #else 20 | // Vector summation 21 | dtype block_sum = 0; 22 | #if BLOCK_SIZE % 4 == 0 23 | #pragma unroll 24 | for (int k = 0; k < BLOCK_SIZE / 4; k++) { 25 | block_sum += dot(vload4(as_start + k, AS), vload4(bs_start + k, BS)); 26 | } 27 | #else 28 | #error "Control should not reach this point" 29 | #endif 30 | #endif 31 | 32 | #if PRECISION_LEVEL == 0 33 | // Simple summation 34 | sum += block_sum; 35 | #elif PRECISION_LEVEL == 1 36 | // Kahan summation 37 | volatile dtype y = block_sum - partial; 38 | dtype t = sum + y; 39 | partial = (t - sum) - y; 40 | sum = t; 41 | #elif PRECISION_LEVEL >= 2 42 | // Most precise summation 43 | int i = 0; 44 | #define x block_sum 45 | for (int k = 0; k < n_partials; k++) { 46 | dtype y = partials[k]; 47 | if (fabs(x) < fabs(y)) { 48 | dtype t = x; 49 | x = y; 50 | y = t; 51 | } 52 | volatile dtype hi = x + y; 53 | dtype lo = y - (hi - x); 54 | if (lo) { 55 | partials[i++] = lo; 56 | } 57 | x = hi; 58 | } 59 | partials[i] = x; 60 | n_partials = i + 1; 61 | #undef x 62 | #endif 63 | -------------------------------------------------------------------------------- /ocl/matrix_reduce.cl: -------------------------------------------------------------------------------- 1 | /// @brief Define for reduce operation on matrix rows or columns. 2 | /// @author Kazantsev Alexey 3 | /// @details Kernel should be defined as: 4 | /// __kernel __attribute__((reqd_work_group_size(REDUCE_SIZE, 1, 1))) 5 | /// 6 | /// Sizes should be declared externally (values are given for example): 7 | /// #define REDUCE_SIZE 64 8 | /// #define A_WIDTH 10 9 | /// #define A_HEIGHT 100500 10 | /// 11 | /// As well as Matricies: 12 | /// #define A err_y 13 | /// 14 | /// And summation by columns if neccessary (otherwise summation by rows is assumed): 15 | /// #define A_COL 16 | /// 17 | /// size_t WorkSize[2] = {A_WIDTH * REDUCE_SIZE} or {A_HEIGHT * REDUCE_SIZE} #ifdef A_COL 18 | /// size_t LocalSize[2] = {REDUCE_SIZE} 19 | /// 20 | /// The result will be in (sum + AS[0]), output offset will be in bx, write it in if (tx == 0) { ... } 21 | __local dtype AS[REDUCE_SIZE]; 22 | 23 | int bx = get_group_id(0); // from 0 to number of resulting output elements 24 | int tx = get_local_id(0); // from 0 to BLOCK_SIZE - 1 25 | 26 | dtype sum = 0; 27 | 28 | #ifdef A_COL 29 | int offs = bx + tx * A_WIDTH; 30 | #define ARRAY_SIZE A_HEIGHT 31 | #define OFFS (REDUCE_SIZE * A_WIDTH) 32 | #else 33 | int offs = bx * A_WIDTH + tx; 34 | #define ARRAY_SIZE A_WIDTH 35 | #define OFFS REDUCE_SIZE 36 | #endif 37 | for (int i = 0; i < ARRAY_SIZE / REDUCE_SIZE; i++, offs += OFFS) { 38 | sum += A[offs]; 39 | } 40 | // Sum the remaining part 41 | #if (ARRAY_SIZE % REDUCE_SIZE) != 0 42 | if (tx < ARRAY_SIZE % REDUCE_SIZE) { 43 | sum += A[offs]; 44 | } 45 | #endif 46 | 47 | AS[tx] = sum; 48 | // ensure all shared loaded 49 | barrier(CLK_LOCAL_MEM_FENCE); 50 | 51 | // Final summation 52 | sum = 0; 53 | int n = MIN(ARRAY_SIZE, REDUCE_SIZE); 54 | while (n > 1) { 55 | sum += (n & 1) ? AS[n - 1] : 0; 56 | n >>= 1; 57 | if (tx < n) { 58 | AS[tx] += AS[n + tx]; 59 | } 60 | // ensure all shared summed 61 | barrier(CLK_LOCAL_MEM_FENCE); 62 | } 63 | 64 | #undef OFFS 65 | #undef ARRAY_SIZE 66 | 67 | // The result will be in (sum + AS[0]), output offset will be in bx, write it in if (tx == 0) { ... } 68 | 69 | /// Define for reduce operation on matrix rows or columns ends here. 70 | -------------------------------------------------------------------------------- /ocl/mean_disp_normalizer.cl: -------------------------------------------------------------------------------- 1 | #include "defines.cl" 2 | #include "highlight.cl" 3 | 4 | /* 5 | @brief Normalizes array of images according to mean and dispersion. 6 | @param input Array of the images. 7 | @param mean Image with the mean over the dataset. 8 | @param rdisp Image with the 1.0 / dispersion over the dataset. 9 | @param output Array of the output images. 10 | @details output[:] = ((dtype)input[:] - (dtype)mean) * rdisp. 11 | */ 12 | __kernel 13 | void normalize_mean_disp(__global const input_type /* IN */ *input, 14 | __global const mean_type /* IN */ *mean, 15 | __global const dtype /* IN */ *rdisp, 16 | __global dtype /* OUT */ *output) { 17 | int offs_in_sample = get_global_id(0); 18 | int offs = get_global_id(1) * SAMPLE_SIZE + offs_in_sample; 19 | output[offs] = ((dtype)input[offs] - (dtype)mean[offs_in_sample]) * rdisp[offs_in_sample]; 20 | } 21 | -------------------------------------------------------------------------------- /requirements-dev.2.txt: -------------------------------------------------------------------------------- 1 | snakefood>=1.4 2 | backports.lzma>=0.0.3 3 | protobuf>=2.0 4 | -------------------------------------------------------------------------------- /requirements-dev.3.txt: -------------------------------------------------------------------------------- 1 | xmltodict>=0.9.2 2 | sphinx>=1.2.2 3 | sphinx-argparse>=0.1.13 4 | sphinx-paramlinks>=0.2.2 5 | sphinxcontrib-napoleon>=0.2.8 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib>=1.4.0 2 | numpy>=1.8.2 3 | scipy>=0.13.3 4 | Pillow>=2.0.0 5 | six>=1.3.0 6 | tornado>=4.0 7 | tornado-smtpclient>=0.1.5 8 | motor<0.4,>=0.3 9 | pymongo==2.7.1 10 | twisted>=14.0.0 11 | ply>=3.4 12 | paramiko==1.13.0 13 | opencl4py>=1.0 14 | argcomplete>=0.8 15 | ipython<4.0.0 16 | jpeg4py>=0.1.0 17 | cffi>=0.8.2 18 | glymur>=0.5.10 19 | lockfile>=0.9.1 20 | python-snappy>=0.5 21 | posix-ipc>=0.9.8 22 | pycrypto>=2.6.1 23 | ecdsa>=0.11 24 | pyzmq>=14.3.1 25 | wget>=2.2 26 | service_identity>=0.2 27 | pygit2>=0.21.4 28 | pyinotify>=0.9.4 29 | cuda4py>=1.0 30 | psutil>=1.2.1 31 | pyxDamerauLevenshtein>=1.2 32 | h5py>=2.2 33 | jinja2>=2.7.2 34 | pycparser>=2.10 35 | pyodbc>=3.0.10 36 | markdown>=2.6.2 37 | py-gfm>=0.1.1 38 | weasyprint>=0.23 39 | pyasn1>=0.1.8 40 | lmdb>=0.86 41 | -------------------------------------------------------------------------------- /veles/dot_pip.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jan 14, 2014 12 | 13 | Local dependencies (pip -t .pip) support. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from os import getenv, path 39 | import sys 40 | 41 | from veles.config import root 42 | from veles.paths import __root__ 43 | 44 | 45 | def install_dot_pip(dirname=root.common.dependencies.dirname): 46 | pp = getenv("PYTHONPATH") 47 | if pp is not None: 48 | index = len(pp.split(':')) 49 | else: 50 | index = 0 51 | for try_path in (path.join(root.common.dependencies.basedir, dirname), 52 | path.join(__root__, dirname)): 53 | if path.exists(try_path): 54 | sys.path[index:index] = try_path 55 | -------------------------------------------------------------------------------- /veles/dummy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Oct 31, 2014 12 | 13 | Dummy units for tests and benchmarks. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from time import time 39 | import uuid 40 | from zope.interface import implementer 41 | 42 | from veles.units import IUnit, TrivialUnit 43 | from veles.workflow import Workflow 44 | 45 | 46 | class DummyLauncher(object): 47 | def __init__(self): 48 | self.stopped = False 49 | self.testing = False 50 | self.id = str(uuid.uuid4()) 51 | 52 | @property 53 | def interactive(self): 54 | return False 55 | 56 | @property 57 | def is_slave(self): 58 | return False 59 | 60 | @property 61 | def is_master(self): 62 | return False 63 | 64 | @property 65 | def is_standalone(self): 66 | return True 67 | 68 | @property 69 | def log_id(self): 70 | return "DUMMY" 71 | 72 | @property 73 | def start_time(self): 74 | return time() - 1000 75 | 76 | @property 77 | def workflow_file(self): 78 | return "/path/to/workflow" 79 | 80 | @property 81 | def config_file(self): 82 | return "/path/to/config" 83 | 84 | @property 85 | def seeds(self): 86 | return [] 87 | 88 | def add_ref(self, workflow): 89 | self.workflow = workflow 90 | 91 | def del_ref(self, unit): 92 | pass 93 | 94 | def on_workflow_finished(self): 95 | pass 96 | 97 | def stop(self): 98 | pass 99 | 100 | 101 | class DummyWorkflow(Workflow): 102 | """ 103 | Dummy standalone workflow for tests and benchmarks. 104 | """ 105 | def __init__(self): 106 | """ 107 | Passes DummyLauncher as workflow parameter value. 108 | """ 109 | self._launcher = DummyLauncher() 110 | super(DummyWorkflow, self).__init__(self._launcher) 111 | self.end_point.link_from(self.start_point) 112 | 113 | @property 114 | def interactive(self): 115 | return False 116 | 117 | @property 118 | def launcher(self): 119 | return self._launcher 120 | 121 | 122 | @implementer(IUnit) 123 | class DummyUnit(TrivialUnit): 124 | """ 125 | Dummy unit. 126 | """ 127 | DISABLE_KWARGS_CHECK = True 128 | 129 | def __init__(self, **kwargs): 130 | super(DummyUnit, self).__init__(DummyWorkflow()) 131 | self.__dict__.update(kwargs) 132 | -------------------------------------------------------------------------------- /veles/ensemble/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- # pylint: disable=C0302 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jub 11, 2015 12 | 13 | Ensemble of machine learning algorithms engine. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | -------------------------------------------------------------------------------- /veles/ensemble/test_workflow.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- # pylint: disable=C0302 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on June 17, 2015 12 | 13 | Ensemble of machine learning algorithms - top level test workflow. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | import json 39 | import os 40 | from tempfile import NamedTemporaryFile 41 | from six import string_types 42 | from zope.interface import implementer 43 | from veles.ensemble.base_workflow import EnsembleWorkflowBase, \ 44 | EnsembleModelManagerBase 45 | 46 | from veles.units import IUnit 47 | 48 | 49 | @implementer(IUnit) 50 | class EnsembleTestManager(EnsembleModelManagerBase): 51 | def __init__(self, workflow, **kwargs): 52 | super(EnsembleTestManager, self).__init__(workflow, **kwargs) 53 | self.input_file = kwargs["input_file"] 54 | 55 | @property 56 | def input_file(self): 57 | return self._input_file 58 | 59 | @input_file.setter 60 | def input_file(self, value): 61 | if not isinstance(value, string_types): 62 | raise TypeError( 63 | "input_file must be a string (got %s)" % type(value)) 64 | self._input_file = value 65 | with open(self.input_file, "r") as fin: 66 | self._input_data = json.load(fin) 67 | self._results[:] = [None] * self.size 68 | 69 | @property 70 | def size(self): 71 | return len(self._input_data["models"]) 72 | 73 | def initialize(self, **kwargs): 74 | super(EnsembleTestManager, self).initialize(**kwargs) 75 | if self.testing: 76 | self.warning("--test is ignored") 77 | 78 | def run(self): 79 | index = sum(1 for r in self.results if r is not None) 80 | model = self._input_data["models"][self._model_index] 81 | id_ = model["id"] 82 | log_id = model["log_id"] 83 | snapshot = model["Snapshot"] 84 | with NamedTemporaryFile( 85 | prefix="veles-ensemble-", suffix=".json", mode="r") as fin: 86 | argv = ["--test", "--result-file", fin.name, "--stealth", 87 | "--log-id", self.launcher.log_id, "--snapshot", 88 | self._to_snapshot_arg(id_, log_id, snapshot)] + \ 89 | self._filtered_argv_ + ["root.common.disable.publishing=True"] 90 | try: 91 | self.info("Evaluating model %d / %d (#%d)...\n%s", 92 | index + 1, self.size, self._model_index, "-" * 80) 93 | fin.seek(0, os.SEEK_SET) 94 | result = self._exec(argv, fin, "test") 95 | if result is None: 96 | return 97 | self.results[index] = result 98 | finally: 99 | self._model_index += 1 100 | 101 | 102 | class EnsembleTestWorkflow(EnsembleWorkflowBase): 103 | KWATTRS = set(EnsembleTestManager.KWATTRS) 104 | MANAGER_UNIT = EnsembleTestManager 105 | 106 | 107 | def run(load, main, **kwargs): 108 | load(EnsembleTestWorkflow, **kwargs) 109 | main() 110 | -------------------------------------------------------------------------------- /veles/error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Mar 18, 2013 12 | 13 | Classes for custom exceptions 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | class VelesException(Exception): 39 | """Base class for Veles exceptions. 40 | """ 41 | pass 42 | 43 | 44 | class BadFormatError(VelesException): 45 | """Raised when bad format of data occured somethere. 46 | """ 47 | pass 48 | 49 | 50 | class Bug(VelesException): 51 | """Raised when something goes wrong but it shouldn't. 52 | """ 53 | pass 54 | 55 | 56 | class MasterSlaveCommunicationError(VelesException): 57 | """Raised when master or slaves discovers data inconsistency during the 58 | communication. 59 | """ 60 | pass 61 | -------------------------------------------------------------------------------- /veles/external/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Samsung/veles/96218b8d04ab099d65bfff350a24412a66c26178/veles/external/__init__.py -------------------------------------------------------------------------------- /veles/external/daemon/LICENSE.PSF-2: -------------------------------------------------------------------------------- 1 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 2 | -------------------------------------------- 3 | 4 | 1. This LICENSE AGREEMENT is between the Python Software Foundation 5 | ("PSF"), and the Individual or Organization ("Licensee") accessing and 6 | otherwise using this software ("Python") in source or binary form and 7 | its associated documentation. 8 | 9 | 2. Subject to the terms and conditions of this License Agreement, PSF 10 | hereby grants Licensee a nonexclusive, royalty-free, world-wide 11 | license to reproduce, analyze, test, perform and/or display publicly, 12 | prepare derivative works, distribute, and otherwise use Python 13 | alone or in any derivative version, provided, however, that PSF's 14 | License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 15 | 2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software Foundation; 16 | All Rights Reserved" are retained in Python alone or in any derivative 17 | version prepared by Licensee. 18 | 19 | 3. In the event Licensee prepares a derivative work that is based on 20 | or incorporates Python or any part thereof, and wants to make 21 | the derivative work available to others as provided herein, then 22 | Licensee hereby agrees to include in any such work a brief summary of 23 | the changes made to Python. 24 | 25 | 4. PSF is making Python available to Licensee on an "AS IS" 26 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR 27 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND 28 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS 29 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT 30 | INFRINGE ANY THIRD PARTY RIGHTS. 31 | 32 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 33 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS 34 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, 35 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 36 | 37 | 6. This License Agreement will automatically terminate upon a material 38 | breach of its terms and conditions. 39 | 40 | 7. Nothing in this License Agreement shall be deemed to create any 41 | relationship of agency, partnership, or joint venture between PSF and 42 | Licensee. This License Agreement does not grant permission to use PSF 43 | trademarks or trade name in a trademark sense to endorse or promote 44 | products or services of Licensee, or any third party. 45 | 46 | 8. By copying, installing or otherwise using Python, Licensee 47 | agrees to be bound by the terms and conditions of this License 48 | Agreement. 49 | -------------------------------------------------------------------------------- /veles/external/daemon/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # daemon/__init__.py 4 | # Part of python-daemon, an implementation of PEP 3143. 5 | # 6 | # Copyright © 2009–2010 Ben Finney 7 | # Copyright © 2006 Robert Niederreiter 8 | # 9 | # This is free software: you may copy, modify, and/or distribute this work 10 | # under the terms of the Python Software Foundation License, version 2 or 11 | # later as published by the Python Software Foundation. 12 | # No warranty expressed or implied. See the file LICENSE.PSF-2 for details. 13 | 14 | """ Library to implement a well-behaved Unix daemon process. 15 | 16 | This library implements the well-behaved daemon specification of 17 | :pep:`3143`, "Standard daemon process library". 18 | 19 | A well-behaved Unix daemon process is tricky to get right, but the 20 | required steps are much the same for every daemon program. A 21 | `DaemonContext` instance holds the behaviour and configured 22 | process environment for the program; use the instance as a context 23 | manager to enter a daemon state. 24 | 25 | Simple example of usage:: 26 | 27 | import daemon 28 | 29 | from spam import do_main_program 30 | 31 | with daemon.DaemonContext(): 32 | do_main_program() 33 | 34 | Customisation of the steps to become a daemon is available by 35 | setting options on the `DaemonContext` instance; see the 36 | documentation for that class for each option. 37 | 38 | """ 39 | 40 | 41 | from . import version 42 | from .daemon import DaemonContext, daemonize, redirect_stream 43 | 44 | _version = version.version 45 | _copyright = version.copyright 46 | _license = version.license 47 | _url = "http://pypi.python.org/pypi/python-daemon/" 48 | -------------------------------------------------------------------------------- /veles/external/daemon/version/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # daemon/version/__init__.py 4 | # Part of python-daemon, an implementation of PEP 3143. 5 | # 6 | # Copyright © 2008–2010 Ben Finney 7 | # This is free software: you may copy, modify, and/or distribute this work 8 | # under the terms of the Python Software Foundation License, version 2 or 9 | # later as published by the Python Software Foundation. 10 | # No warranty expressed or implied. See the file LICENSE.PSF-2 for details. 11 | 12 | """ Version information for the python-daemon distribution. """ 13 | 14 | from .version_info import version_info 15 | 16 | version_info['version_string'] = "1.5.5" 17 | 18 | version_short = "%(version_string)s" % version_info 19 | version_full = "%(version_string)s.r%(revno)s" % version_info 20 | version = version_short 21 | 22 | author_name = "Ben Finney" 23 | author_email = "ben+python@benfinney.id.au" 24 | author = "%(author_name)s <%(author_email)s>" % vars() 25 | 26 | copyright_year_begin = "2001" 27 | date = version_info['date'].split(' ', 1)[0] 28 | copyright_year = date.split('-')[0] 29 | copyright_year_range = copyright_year_begin 30 | if copyright_year > copyright_year_begin: 31 | copyright_year_range += "–%(copyright_year)s" % vars() 32 | 33 | copyright = ( 34 | "Copyright © %(copyright_year_range)s %(author)s and others" 35 | ) % vars() 36 | license = "PSF-2+" 37 | -------------------------------------------------------------------------------- /veles/external/daemon/version/version_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """This file is automatically generated by generate_version_info 3 | It uses the current working tree to determine the revision. 4 | So don't edit it. :) 5 | """ 6 | 7 | version_info = {'branch_nick': 'python-daemon.devel', 8 | 'build_date': '2009-05-22 19:50:06 +1000', 9 | 'clean': None, 10 | 'date': '2009-05-22 19:47:30 +1000', 11 | 'revision_id': 'ben+python@benfinney.id.au-20090522094730-p4vsa0reh7ktt4e1', 12 | 'revno': 145} 13 | 14 | revisions = {} 15 | 16 | file_revisions = {} 17 | 18 | 19 | 20 | if __name__ == '__main__': 21 | print('revision: %(revno)d' % version_info) 22 | print('nick: %(branch_nick)s' % version_info) 23 | print('revision id: %(revision_id)s' % version_info) 24 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_encodings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | An enumeration used to specify character sets supported by charmaps. Used in 11 | the FT_Select_Charmap API function. 12 | 13 | FT_ENCODING_NONE 14 | 15 | The encoding value 0 is reserved. 16 | 17 | FT_ENCODING_UNICODE 18 | 19 | Corresponds to the Unicode character set. This value covers all versions of 20 | the Unicode repertoire, including ASCII and Latin-1. Most fonts include a 21 | Unicode charmap, but not all of them. 22 | 23 | For example, if you want to access Unicode value U+1F028 (and the font 24 | contains it), use value 0x1F028 as the input value for FT_Get_Char_Index. 25 | 26 | FT_ENCODING_MS_SYMBOL 27 | 28 | Corresponds to the Microsoft Symbol encoding, used to encode mathematical 29 | symbols in the 32..255 character code range. For more information, see 30 | 'http://www.ceviz.net/symbol.htm'. 31 | 32 | FT_ENCODING_SJIS 33 | 34 | Corresponds to Japanese SJIS encoding. More info at at 35 | 'http://langsupport.japanreference.com/encoding.shtml'. See note on 36 | multi-byte encodings below. 37 | 38 | FT_ENCODING_GB2312 39 | 40 | Corresponds to an encoding system for Simplified Chinese as used used in 41 | mainland China. 42 | 43 | FT_ENCODING_BIG5 44 | 45 | Corresponds to an encoding system for Traditional Chinese as used in Taiwan 46 | and Hong Kong. 47 | 48 | FT_ENCODING_WANSUNG 49 | 50 | Corresponds to the Korean encoding system known as Wansung. For more 51 | information see 'http://www.microsoft.com/typography/unicode/949.txt'. 52 | 53 | FT_ENCODING_JOHAB 54 | 55 | The Korean standard character set (KS C 5601-1992), which corresponds to MS 56 | Windows code page 1361. This character set includes all possible Hangeul 57 | character combinations. 58 | 59 | FT_ENCODING_ADOBE_LATIN_1 60 | 61 | Corresponds to a Latin-1 encoding as defined in a Type 1 PostScript font. It 62 | is limited to 256 character codes. 63 | 64 | FT_ENCODING_ADOBE_STANDARD 65 | 66 | Corresponds to the Adobe Standard encoding, as found in Type 1, CFF, and 67 | OpenType/CFF fonts. It is limited to 256 character codes. 68 | 69 | FT_ENCODING_ADOBE_EXPERT 70 | 71 | Corresponds to the Adobe Expert encoding, as found in Type 1, CFF, and 72 | OpenType/CFF fonts. It is limited to 256 character codes. 73 | 74 | FT_ENCODING_ADOBE_CUSTOM 75 | 76 | Corresponds to a custom encoding, as found in Type 1, CFF, and OpenType/CFF 77 | fonts. It is limited to 256 character codes. 78 | 79 | FT_ENCODING_APPLE_ROMAN 80 | 81 | Corresponds to the 8-bit Apple roman encoding. Many TrueType and OpenType 82 | fonts contain a charmap for this encoding, since older versions of Mac OS are 83 | able to use it. 84 | 85 | FT_ENCODING_OLD_LATIN_2 86 | 87 | This value is deprecated and was never used nor reported by FreeType. Don't 88 | use or test for it. 89 | """ 90 | 91 | 92 | def _FT_ENC_TAG(a, b, c, d): 93 | return (ord(a) << 24 | ord(b) << 16 | ord(c) << 8 | ord(d)) 94 | 95 | 96 | FT_ENCODINGS = {'FT_ENCODING_NONE': _FT_ENC_TAG('\0', '\0', '\0', '\0'), 97 | 'FT_ENCODING_MS_SYMBOL': _FT_ENC_TAG('s', 'y', 'm', 'b'), 98 | 'FT_ENCODING_UNICODE': _FT_ENC_TAG('u', 'n', 'i', 'c'), 99 | 'FT_ENCODING_SJIS': _FT_ENC_TAG('s', 'j', 'i', 's'), 100 | 'FT_ENCODING_GB2312': _FT_ENC_TAG('g', 'b', ' ', ' '), 101 | 'FT_ENCODING_BIG5': _FT_ENC_TAG('b', 'i', 'g', '5'), 102 | 'FT_ENCODING_WANSUNG': _FT_ENC_TAG('w', 'a', 'n', 's'), 103 | 'FT_ENCODING_JOHAB': _FT_ENC_TAG('j', 'o', 'h', 'a'), 104 | 'FT_ENCODING_ADOBE_STANDARD': _FT_ENC_TAG('A', 'D', 'O', 'B'), 105 | 'FT_ENCODING_ADOBE_EXPERT': _FT_ENC_TAG('A', 'D', 'B', 'E'), 106 | 'FT_ENCODING_ADOBE_CUSTOM': _FT_ENC_TAG('A', 'D', 'B', 'C'), 107 | 'FT_ENCODING_ADOBE_LATIN1': _FT_ENC_TAG('l', 'a', 't', '1'), 108 | 'FT_ENCODING_OLD_LATIN2': _FT_ENC_TAG('l', 'a', 't', '2'), 109 | 'FT_ENCODING_APPLE_ROMAN': _FT_ENC_TAG('a', 'r', 'm', 'n')} 110 | globals().update(FT_ENCODINGS) 111 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_fstypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of bit flags that inform client applications of embedding and 11 | subsetting restrictions associated with a font. 12 | 13 | FT_FSTYPE_INSTALLABLE_EMBEDDING 14 | 15 | Fonts with no fsType bit set may be embedded and permanently installed on 16 | the remote system by an application. 17 | 18 | 19 | FT_FSTYPE_RESTRICTED_LICENSE_EMBEDDING 20 | 21 | Fonts that have only this bit set must not be modified, embedded or exchanged 22 | in any manner without first obtaining permission of the font software 23 | copyright owner. 24 | 25 | 26 | FT_FSTYPE_PREVIEW_AND_PRINT_EMBEDDING 27 | 28 | If this bit is set, the font may be embedded and temporarily loaded on the 29 | remote system. Documents containing Preview & Print fonts must be opened 30 | 'read-only'; no edits can be applied to the document. 31 | 32 | 33 | FT_FSTYPE_EDITABLE_EMBEDDING 34 | 35 | If this bit is set, the font may be embedded but must only be installed 36 | temporarily on other systems. In contrast to Preview & Print fonts, 37 | documents containing editable fonts may be opened for reading, editing is 38 | permitted, and changes may be saved. 39 | 40 | 41 | FT_FSTYPE_NO_SUBSETTING 42 | 43 | If this bit is set, the font may not be subsetted prior to embedding. 44 | 45 | 46 | FT_FSTYPE_BITMAP_EMBEDDING_ONLY 47 | 48 | If this bit is set, only bitmaps contained in the font may be embedded; no 49 | outline data may be embedded. If there are no bitmaps available in the font, 50 | then the font is unembeddable. 51 | """ 52 | 53 | FT_FSTYPES = {'FT_FSTYPE_INSTALLABLE_EMBEDDING' : 0x0000, 54 | 'FT_FSTYPE_RESTRICTED_LICENSE_EMBEDDING' : 0x0002, 55 | 'FT_FSTYPE_PREVIEW_AND_PRINT_EMBEDDING' : 0x0004, 56 | 'FT_FSTYPE_EDITABLE_EMBEDDING' : 0x0008, 57 | 'FT_FSTYPE_NO_SUBSETTING' : 0x0100, 58 | 'FT_FSTYPE_BITMAP_EMBEDDING_ONLY' : 0x0200,} 59 | globals().update(FT_FSTYPES) 60 | ft_fstype_installable_embedding = FT_FSTYPE_INSTALLABLE_EMBEDDING 61 | ft_fstype_restricted_license_embedding = FT_FSTYPE_RESTRICTED_LICENSE_EMBEDDING 62 | ft_fstype_preview_and_print_embedding = FT_FSTYPE_PREVIEW_AND_PRINT_EMBEDDING 63 | ft_fstype_editable_embedding = FT_FSTYPE_EDITABLE_EMBEDDING 64 | ft_fstype_no_subsetting = FT_FSTYPE_NO_SUBSETTING 65 | ft_fstype_bitmap_embedding_only = FT_FSTYPE_BITMAP_EMBEDDING_ONLY 66 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_glyph_bbox_modes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | The mode how the values of FT_Glyph_Get_CBox are returned. 11 | 12 | FT_GLYPH_BBOX_UNSCALED 13 | 14 | Return unscaled font units. 15 | 16 | FT_GLYPH_BBOX_SUBPIXELS 17 | 18 | Return unfitted 26.6 coordinates. 19 | 20 | FT_GLYPH_BBOX_GRIDFIT 21 | 22 | Return grid-fitted 26.6 coordinates. 23 | 24 | FT_GLYPH_BBOX_TRUNCATE 25 | 26 | Return coordinates in integer pixels. 27 | 28 | FT_GLYPH_BBOX_PIXELS 29 | 30 | Return grid-fitted pixel coordinates. 31 | """ 32 | FT_GLYPH_BBOX_MODES = {'FT_GLYPH_BBOX_UNSCALED' : 0, 33 | 'FT_GLYPH_BBOX_SUBPIXELS' : 0, 34 | 'FT_GLYPH_BBOX_GRIDFIT' : 1, 35 | 'FT_GLYPH_BBOX_TRUNCATE' : 2, 36 | 'FT_GLYPH_BBOX_PIXELS' : 3} 37 | globals().update(FT_GLYPH_BBOX_MODES) 38 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_glyph_formats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | An enumeration type used to describe the format of a given glyph image. Note 11 | that this version of FreeType only supports two image formats, even though 12 | future font drivers will be able to register their own format. 13 | 14 | FT_GLYPH_FORMAT_NONE 15 | 16 | The value 0 is reserved. 17 | 18 | FT_GLYPH_FORMAT_COMPOSITE 19 | 20 | The glyph image is a composite of several other images. This format is only 21 | used with FT_LOAD_NO_RECURSE, and is used to report compound glyphs (like 22 | accented characters). 23 | 24 | FT_GLYPH_FORMAT_BITMAP 25 | 26 | The glyph image is a bitmap, and can be described as an FT_Bitmap. You 27 | generally need to access the 'bitmap' field of the FT_GlyphSlotRec structure 28 | to read it. 29 | 30 | FT_GLYPH_FORMAT_OUTLINE 31 | 32 | The glyph image is a vectorial outline made of line segments and Bezier arcs; 33 | it can be described as an FT_Outline; you generally want to access the 34 | 'outline' field of the FT_GlyphSlotRec structure to read it. 35 | 36 | FT_GLYPH_FORMAT_PLOTTER 37 | 38 | The glyph image is a vectorial path with no inside and outside contours. Some 39 | Type 1 fonts, like those in the Hershey family, contain glyphs in this 40 | format. These are described as FT_Outline, but FreeType isn't currently 41 | capable of rendering them correctly. 42 | """ 43 | 44 | def _FT_IMAGE_TAG(a,b,c,d): 45 | return ( ord(a) << 24 | ord(b) << 16 | ord(c) << 8 | ord(d) ) 46 | 47 | FT_GLYPH_FORMATS = { 48 | 'FT_GLYPH_FORMAT_NONE' : _FT_IMAGE_TAG( '\0','\0','\0','\0' ), 49 | 'FT_GLYPH_FORMAT_COMPOSITE' : _FT_IMAGE_TAG( 'c','o','m','p' ), 50 | 'FT_GLYPH_FORMAT_BITMAP' : _FT_IMAGE_TAG( 'b','i','t','s' ), 51 | 'FT_GLYPH_FORMAT_OUTLINE' : _FT_IMAGE_TAG( 'o','u','t','l' ), 52 | 'FT_GLYPH_FORMAT_PLOTTER' : _FT_IMAGE_TAG( 'p','l','o','t' )} 53 | globals().update(FT_GLYPH_FORMATS) 54 | ft_glyph_format_none = FT_GLYPH_FORMAT_NONE 55 | ft_glyph_format_composite = FT_GLYPH_FORMAT_COMPOSITE 56 | ft_glyph_format_bitmap = FT_GLYPH_FORMAT_BITMAP 57 | ft_glyph_format_outline = FT_GLYPH_FORMAT_OUTLINE 58 | ft_glyph_format_plotter = FT_GLYPH_FORMAT_PLOTTER 59 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_kerning_modes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | An enumeration used to specify which kerning values to return in 11 | FT_Get_Kerning. 12 | 13 | 14 | FT_KERNING_DEFAULT 15 | 16 | Return scaled and grid-fitted kerning distances (value is 0). 17 | 18 | 19 | FT_KERNING_UNFITTED 20 | 21 | Return scaled but un-grid-fitted kerning distances. 22 | 23 | 24 | FT_KERNING_UNSCALED 25 | 26 | Return the kerning vector in original font units. 27 | """ 28 | FT_KERNING_MODES = { 'FT_KERNING_DEFAULT' : 0, 29 | 'FT_KERNING_UNFITTED' : 1, 30 | 'FT_KERNING_UNSCALED' : 2 } 31 | globals().update(FT_KERNING_MODES) 32 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_lcd_filters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | 10 | """ 11 | A list of values to identify various types of LCD filters. 12 | 13 | 14 | FT_LCD_FILTER_NONE 15 | 16 | Do not perform filtering. When used with subpixel rendering, this results in 17 | sometimes severe color fringes. 18 | 19 | 20 | FT_LCD_FILTER_DEFAULT 21 | 22 | The default filter reduces color fringes considerably, at the cost of a 23 | slight blurriness in the output. 24 | 25 | 26 | FT_LCD_FILTER_LIGHT 27 | 28 | The light filter is a variant that produces less blurriness at the cost of 29 | slightly more color fringes than the default one. It might be better, 30 | depending on taste, your monitor, or your personal vision. 31 | 32 | 33 | FT_LCD_FILTER_LEGACY 34 | 35 | This filter corresponds to the original libXft color filter. It provides high 36 | contrast output but can exhibit really bad color fringes if glyphs are not 37 | extremely well hinted to the pixel grid. In other words, it only works well 38 | if the TrueType bytecode interpreter is enabled and high-quality hinted fonts 39 | are used. 40 | 41 | This filter is only provided for comparison purposes, and might be disabled 42 | or stay unsupported in the future. 43 | """ 44 | 45 | FT_LCD_FILTERS = {'FT_LCD_FILTER_NONE' : 0, 46 | 'FT_LCD_FILTER_DEFAULT' : 1, 47 | 'FT_LCD_FILTER_LIGHT' : 2, 48 | 'FT_LCD_FILTER_LEGACY' : 16} 49 | globals().update(FT_LCD_FILTERS) 50 | 51 | 52 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_load_targets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of values that are used to select a specific hinting algorithm to use 11 | by the hinter. You should OR one of these values to your 'load_flags' when 12 | calling FT_Load_Glyph. 13 | 14 | Note that font's native hinters may ignore the hinting algorithm you have 15 | specified (e.g., the TrueType bytecode interpreter). You can set 16 | FT_LOAD_FORCE_AUTOHINT to ensure that the auto-hinter is used. 17 | 18 | Also note that FT_LOAD_TARGET_LIGHT is an exception, in that it always 19 | implies FT_LOAD_FORCE_AUTOHINT. 20 | 21 | 22 | FT_LOAD_TARGET_NORMAL 23 | 24 | This corresponds to the default hinting algorithm, optimized for standard 25 | gray-level rendering. For monochrome output, use FT_LOAD_TARGET_MONO instead. 26 | 27 | 28 | FT_LOAD_TARGET_LIGHT 29 | 30 | A lighter hinting algorithm for non-monochrome modes. Many generated glyphs 31 | are more fuzzy but better resemble its original shape. A bit like rendering 32 | on Mac OS X. 33 | 34 | As a special exception, this target implies FT_LOAD_FORCE_AUTOHINT. 35 | 36 | 37 | FT_LOAD_TARGET_MONO 38 | 39 | Strong hinting algorithm that should only be used for monochrome output. The 40 | result is probably unpleasant if the glyph is rendered in non-monochrome 41 | modes. 42 | 43 | 44 | FT_LOAD_TARGET_LCD 45 | 46 | A variant of FT_LOAD_TARGET_NORMAL optimized for horizontally decimated LCD 47 | displays. 48 | 49 | 50 | FT_LOAD_TARGET_LCD_V 51 | 52 | A variant of FT_LOAD_TARGET_NORMAL optimized for vertically decimated LCD 53 | displays. 54 | """ 55 | 56 | from .ft_render_modes import * 57 | 58 | 59 | def _FT_LOAD_TARGET_(x): 60 | return (x & 15) << 16 61 | FT_LOAD_TARGETS = { 62 | 'FT_LOAD_TARGET_NORMAL' : _FT_LOAD_TARGET_(FT_RENDER_MODE_NORMAL), 63 | 'FT_LOAD_TARGET_LIGHT' : _FT_LOAD_TARGET_(FT_RENDER_MODE_LIGHT), 64 | 'FT_LOAD_TARGET_MONO' : _FT_LOAD_TARGET_(FT_RENDER_MODE_MONO), 65 | 'FT_LOAD_TARGET_LCD' : _FT_LOAD_TARGET_(FT_RENDER_MODE_LCD), 66 | 'FT_LOAD_TARGET_LCD_V' : _FT_LOAD_TARGET_(FT_RENDER_MODE_LCD_V) } 67 | globals().update(FT_LOAD_TARGETS) 68 | # def FT_LOAD_TARGET_MODE(x): 69 | # return (x >> 16) & 15 70 | 71 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_open_modes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of bit-field constants used within the 'flags' field of the 11 | FT_Open_Args structure. 12 | 13 | 14 | FT_OPEN_MEMORY 15 | 16 | This is a memory-based stream. 17 | 18 | 19 | FT_OPEN_STREAM 20 | 21 | Copy the stream from the 'stream' field. 22 | 23 | 24 | FT_OPEN_PATHNAME 25 | 26 | Create a new input stream from a C path name. 27 | 28 | 29 | FT_OPEN_DRIVER 30 | 31 | Use the 'driver' field. 32 | 33 | 34 | FT_OPEN_PARAMS 35 | 36 | Use the 'num_params' and 'params' fields. 37 | """ 38 | FT_OPEN_MODES = {'FT_OPEN_MEMORY': 0x1, 39 | 'FT_OPEN_STREAM': 0x2, 40 | 'FT_OPEN_PATHNAME': 0x4, 41 | 'FT_OPEN_DRIVER': 0x8, 42 | 'FT_OPEN_PARAMS': 0x10 } 43 | globals().update(FT_OPEN_MODES) 44 | 45 | 46 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_outline_flags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of bit-field constants use for the flags in an outline's 'flags' 11 | field. 12 | 13 | 14 | FT_OUTLINE_NONE 15 | 16 | Value 0 is reserved. 17 | 18 | 19 | FT_OUTLINE_OWNER 20 | 21 | If set, this flag indicates that the outline's field arrays (i.e., 'points', 22 | 'flags', and 'contours') are 'owned' by the outline object, and should thus 23 | be freed when it is destroyed. 24 | 25 | 26 | FT_OUTLINE_EVEN_ODD_FILL 27 | 28 | By default, outlines are filled using the non-zero winding rule. If set to 1, 29 | the outline will be filled using the even-odd fill rule (only works with the 30 | smooth rasterizer). 31 | 32 | 33 | FT_OUTLINE_REVERSE_FILL 34 | 35 | By default, outside contours of an outline are oriented in clock-wise 36 | direction, as defined in the TrueType specification. This flag is set if the 37 | outline uses the opposite direction (typically for Type 1 fonts). This flag 38 | is ignored by the scan converter. 39 | 40 | 41 | FT_OUTLINE_IGNORE_DROPOUTS 42 | 43 | By default, the scan converter will try to detect drop-outs in an outline and 44 | correct the glyph bitmap to ensure consistent shape continuity. If set, this 45 | flag hints the scan-line converter to ignore such cases. See below for more 46 | information. 47 | 48 | 49 | FT_OUTLINE_SMART_DROPOUTS 50 | 51 | Select smart dropout control. If unset, use simple dropout control. Ignored 52 | if FT_OUTLINE_IGNORE_DROPOUTS is set. See below for more information. 53 | 54 | 55 | FT_OUTLINE_INCLUDE_STUBS 56 | 57 | If set, turn pixels on for 'stubs', otherwise exclude them. Ignored if 58 | FT_OUTLINE_IGNORE_DROPOUTS is set. See below for more information. 59 | 60 | 61 | FT_OUTLINE_HIGH_PRECISION 62 | 63 | This flag indicates that the scan-line converter should try to convert this 64 | outline to bitmaps with the highest possible quality. It is typically set for 65 | small character sizes. Note that this is only a hint that might be completely 66 | ignored by a given scan-converter. 67 | 68 | 69 | FT_OUTLINE_SINGLE_PASS 70 | 71 | This flag is set to force a given scan-converter to only use a single pass 72 | over the outline to render a bitmap glyph image. Normally, it is set for very 73 | large character sizes. It is only a hint that might be completely ignored by 74 | a given scan-converter. 75 | """ 76 | FT_OUTLINE_FLAGS = { 'FT_OUTLINE_NONE' : 0x0, 77 | 'FT_OUTLINE_OWNER' : 0x1, 78 | 'FT_OUTLINE_EVEN_ODD_FILL' : 0x2, 79 | 'FT_OUTLINE_REVERSE_FILL' : 0x4, 80 | 'FT_OUTLINE_IGNORE_DROPOUTS' : 0x8, 81 | 'FT_OUTLINE_SMART_DROPOUTS' : 0x10, 82 | 'FT_OUTLINE_INCLUDE_STUBS' : 0x20, 83 | 'FT_OUTLINE_HIGH_PRECISION' : 0x100, 84 | 'FT_OUTLINE_SINGLE_PASS' : 0x200 } 85 | globals().update(FT_OUTLINE_FLAGS) 86 | 87 | 88 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_pixel_modes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | An enumeration type that lists the render modes supported by FreeType 2. Each 11 | mode corresponds to a specific type of scanline conversion performed on the 12 | outline. 13 | 14 | 15 | FT_PIXEL_MODE_NONE 16 | 17 | Value 0 is reserved. 18 | 19 | 20 | FT_PIXEL_MODE_MONO 21 | 22 | A monochrome bitmap, using 1 bit per pixel. Note that pixels are stored in 23 | most-significant order (MSB), which means that the left-most pixel in a byte 24 | has value 128. 25 | 26 | 27 | FT_PIXEL_MODE_GRAY 28 | 29 | An 8-bit bitmap, generally used to represent anti-aliased glyph images. Each 30 | pixel is stored in one byte. Note that the number of 'gray' levels is stored 31 | in the 'num_grays' field of the FT_Bitmap structure (it generally is 256). 32 | 33 | 34 | FT_PIXEL_MODE_GRAY2 35 | 36 | A 2-bit per pixel bitmap, used to represent embedded anti-aliased bitmaps in 37 | font files according to the OpenType specification. We haven't found a single 38 | font using this format, however. 39 | 40 | 41 | FT_PIXEL_MODE_GRAY4 42 | 43 | A 4-bit per pixel bitmap, representing embedded anti-aliased bitmaps in font 44 | files according to the OpenType specification. We haven't found a single font 45 | using this format, however. 46 | 47 | 48 | FT_PIXEL_MODE_LCD 49 | 50 | An 8-bit bitmap, representing RGB or BGR decimated glyph images used for 51 | display on LCD displays; the bitmap is three times wider than the original 52 | glyph image. See also FT_RENDER_MODE_LCD. 53 | 54 | 55 | FT_PIXEL_MODE_LCD_V 56 | 57 | An 8-bit bitmap, representing RGB or BGR decimated glyph images used for 58 | display on rotated LCD displays; the bitmap is three times taller than the 59 | original glyph image. See also FT_RENDER_MODE_LCD_V. 60 | 61 | """ 62 | 63 | FT_PIXEL_MODES = {'FT_PIXEL_MODE_NONE' : 0, 64 | 'FT_PIXEL_MODE_MONO' : 1, 65 | 'FT_PIXEL_MODE_GRAY' : 2, 66 | 'FT_PIXEL_MODE_GRAY2': 3, 67 | 'FT_PIXEL_MODE_GRAY4': 4, 68 | 'FT_PIXEL_MODE_LCD' : 5, 69 | 'FT_PIXEL_MODE_LCD_V': 6, 70 | 'FT_PIXEL_MODE_MAX' : 7} 71 | globals().update(FT_PIXEL_MODES) 72 | ft_pixel_mode_none = FT_PIXEL_MODE_NONE 73 | ft_pixel_mode_mono = FT_PIXEL_MODE_MONO 74 | ft_pixel_mode_grays = FT_PIXEL_MODE_GRAY 75 | ft_pixel_mode_pal2 = FT_PIXEL_MODE_GRAY2 76 | ft_pixel_mode_pal4 = FT_PIXEL_MODE_GRAY4 77 | 78 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_render_modes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | An enumeration type that lists the render modes supported by FreeType 2. Each 11 | mode corresponds to a specific type of scanline conversion performed on the 12 | outline. 13 | 14 | For bitmap fonts and embedded bitmaps the 'bitmap->pixel_mode' field in the 15 | FT_GlyphSlotRec structure gives the format of the returned bitmap. 16 | 17 | All modes except FT_RENDER_MODE_MONO use 256 levels of opacity. 18 | 19 | 20 | FT_RENDER_MODE_NORMAL 21 | 22 | This is the default render mode; it corresponds to 8-bit anti-aliased 23 | bitmaps. 24 | 25 | 26 | FT_RENDER_MODE_LIGHT 27 | 28 | This is equivalent to FT_RENDER_MODE_NORMAL. It is only defined as a separate 29 | value because render modes are also used indirectly to define hinting 30 | algorithm selectors. See FT_LOAD_TARGET_XXX for details. 31 | 32 | 33 | FT_RENDER_MODE_MONO 34 | 35 | This mode corresponds to 1-bit bitmaps (with 2 levels of opacity). 36 | 37 | 38 | FT_RENDER_MODE_LCD 39 | 40 | This mode corresponds to horizontal RGB and BGR sub-pixel displays like LCD 41 | screens. It produces 8-bit bitmaps that are 3 times the width of the original 42 | glyph outline in pixels, and which use the FT_PIXEL_MODE_LCD mode. 43 | 44 | 45 | FT_RENDER_MODE_LCD_V 46 | 47 | This mode corresponds to vertical RGB and BGR sub-pixel displays (like PDA 48 | screens, rotated LCD displays, etc.). It produces 8-bit bitmaps that are 3 49 | times the height of the original glyph outline in pixels and use the 50 | FT_PIXEL_MODE_LCD_V mode. 51 | """ 52 | FT_RENDER_MODES = { 'FT_RENDER_MODE_NORMAL' : 0, 53 | 'FT_RENDER_MODE_LIGHT' : 1, 54 | 'FT_RENDER_MODE_MONO' : 2, 55 | 'FT_RENDER_MODE_LCD' : 3, 56 | 'FT_RENDER_MODE_LCD_V' : 4 } 57 | globals().update(FT_RENDER_MODES) 58 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_stroker_borders.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | These values are used to select a given stroke border in 11 | FT_Stroker_GetBorderCounts and FT_Stroker_ExportBorder. 12 | 13 | 14 | FT_STROKER_BORDER_LEFT 15 | 16 | Select the left border, relative to the drawing direction. 17 | 18 | 19 | FT_STROKER_BORDER_RIGHT 20 | 21 | Select the right border, relative to the drawing direction. 22 | 23 | 24 | Note 25 | 26 | Applications are generally interested in the 'inside' and 'outside' 27 | borders. However, there is no direct mapping between these and the 'left' and 28 | 'right' ones, since this really depends on the glyph's drawing orientation, 29 | which varies between font formats. 30 | 31 | You can however use FT_Outline_GetInsideBorder and 32 | FT_Outline_GetOutsideBorder to get these. 33 | """ 34 | FT_STROKER_BORDERS = { 'FT_STROKER_BORDER_LEFT' : 0, 35 | 'FT_STROKER_BORDER_RIGHT' : 1} 36 | globals().update(FT_STROKER_BORDERS) 37 | 38 | 39 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_stroker_linecaps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | These values determine how the end of opened sub-paths are rendered in a 11 | stroke. 12 | 13 | 14 | FT_STROKER_LINECAP_BUTT 15 | 16 | The end of lines is rendered as a full stop on the last point itself. 17 | 18 | 19 | FT_STROKER_LINECAP_ROUND 20 | 21 | The end of lines is rendered as a half-circle around the last point. 22 | 23 | 24 | FT_STROKER_LINECAP_SQUARE 25 | 26 | The end of lines is rendered as a square around the last point. 27 | """ 28 | 29 | FT_STROKER_LINECAPS = { 'FT_STROKER_LINECAP_BUTT' : 0, 30 | 'FT_STROKER_LINECAP_ROUND' : 1, 31 | 'FT_STROKER_LINECAP_SQUARE' : 2} 32 | globals().update(FT_STROKER_LINECAPS) 33 | 34 | 35 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_stroker_linejoins.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | These values determine how two joining lines are rendered in a stroker. 11 | 12 | 13 | FT_STROKER_LINEJOIN_ROUND 14 | 15 | Used to render rounded line joins. Circular arcs are used to join two lines 16 | smoothly. 17 | 18 | 19 | FT_STROKER_LINEJOIN_BEVEL 20 | 21 | Used to render beveled line joins; i.e., the two joining lines are extended 22 | until they intersect. 23 | 24 | 25 | FT_STROKER_LINEJOIN_MITER 26 | 27 | Same as beveled rendering, except that an additional line break is added if 28 | the angle between the two joining lines is too closed (this is useful to 29 | avoid unpleasant spikes in beveled rendering). 30 | """ 31 | FT_STROKER_LINEJOINS = { 'FT_STROKER_LINEJOIN_ROUND' : 0, 32 | 'FT_STROKER_LINEJOIN_BEVEL' : 1, 33 | 'FT_STROKER_LINEJOIN_MITER' : 2} 34 | globals().update(FT_STROKER_LINEJOINS) 35 | 36 | 37 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/ft_style_flags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of bit-flags used to indicate the style of a given face. These are 11 | used in the 'style_flags' field of FT_FaceRec. 12 | 13 | 14 | FT_STYLE_FLAG_ITALIC 15 | 16 | Indicates that a given face style is italic or oblique. 17 | 18 | 19 | FT_STYLE_FLAG_BOLD 20 | 21 | Indicates that a given face is bold. 22 | """ 23 | FT_STYLE_FLAGS = {'FT_STYLE_FLAG_ITALIC' : 1, 24 | 'FT_STYLE_FLAG_BOLD' : 2 } 25 | globals().update(FT_STYLE_FLAGS) 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_adobe_ids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of valid values for the 'encoding_id' for TT_PLATFORM_ADOBE 11 | charmaps. This is a FreeType-specific extension! 12 | 13 | TT_ADOBE_ID_STANDARD 14 | 15 | Adobe standard encoding. 16 | 17 | 18 | TT_ADOBE_ID_EXPERT 19 | 20 | Adobe expert encoding. 21 | 22 | 23 | TT_ADOBE_ID_CUSTOM 24 | 25 | Adobe custom encoding. 26 | 27 | 28 | TT_ADOBE_ID_LATIN_1 29 | 30 | Adobe Latin 1 encoding. 31 | """ 32 | 33 | TT_ADOBE_IDS = { 34 | 'TT_ADOBE_ID_STANDARD' : 0, 35 | 'TT_ADOBE_ID_EXPERT' : 1, 36 | 'TT_ADOBE_ID_CUSTOM' : 2, 37 | 'TT_ADOBE_ID_LATIN_1' : 3 } 38 | globals().update(TT_ADOBE_IDS) 39 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_apple_ids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of valid values for the 'encoding_id' for TT_PLATFORM_APPLE_UNICODE 11 | charmaps and name entries. 12 | 13 | 14 | TT_APPLE_ID_DEFAULT 15 | 16 | Unicode version 1.0. 17 | 18 | 19 | TT_APPLE_ID_UNICODE_1_1 20 | 21 | Unicode 1.1; specifies Hangul characters starting at U+34xx. 22 | 23 | 24 | TT_APPLE_ID_ISO_10646 25 | 26 | Deprecated (identical to preceding). 27 | 28 | 29 | TT_APPLE_ID_UNICODE_2_0 30 | 31 | Unicode 2.0 and beyond (UTF-16 BMP only). 32 | 33 | 34 | TT_APPLE_ID_UNICODE_32 35 | 36 | Unicode 3.1 and beyond, using UTF-32. 37 | 38 | 39 | TT_APPLE_ID_VARIANT_SELECTOR 40 | 41 | From Adobe, not Apple. Not a normal cmap. Specifies variations on a real 42 | cmap. 43 | """ 44 | TT_APPLE_IDS = { 45 | 'TT_APPLE_ID_DEFAULT' : 0, 46 | 'TT_APPLE_ID_UNICODE_1_1' : 1, 47 | 'TT_APPLE_ID_ISO_10646' : 2, 48 | 'TT_APPLE_ID_UNICODE_2_0' : 3, 49 | 'TT_APPLE_ID_UNICODE_32' : 4, 50 | 'TT_APPLE_ID_VARIANT_SELECTOR' : 5 } 51 | globals().update(TT_APPLE_IDS) 52 | 53 | 54 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_mac_ids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of valid values for the 'encoding_id' for TT_PLATFORM_MACINTOSH 11 | charmaps and name entries. 12 | 13 | TT_MAC_ID_ROMAN 14 | 15 | TT_MAC_ID_TELUGU 16 | 17 | TT_MAC_ID_GURMUKHI 18 | 19 | TT_MAC_ID_TIBETAN 20 | 21 | TT_MAC_ID_SIMPLIFIED_CHINESE 22 | 23 | TT_MAC_ID_SINDHI 24 | 25 | TT_MAC_ID_SINHALESE 26 | 27 | TT_MAC_ID_RUSSIAN 28 | 29 | TT_MAC_ID_KANNADA 30 | 31 | TT_MAC_ID_VIETNAMESE 32 | 33 | TT_MAC_ID_MONGOLIAN 34 | 35 | TT_MAC_ID_DEVANAGARI 36 | 37 | TT_MAC_ID_HEBREW 38 | 39 | TT_MAC_ID_TAMIL 40 | 41 | TT_MAC_ID_THAI 42 | 43 | TT_MAC_ID_BURMESE 44 | 45 | TT_MAC_ID_MALDIVIAN 46 | 47 | TT_MAC_ID_TRADITIONAL_CHINESE 48 | 49 | TT_MAC_ID_JAPANESE 50 | 51 | TT_MAC_ID_GREEK 52 | 53 | TT_MAC_ID_LAOTIAN 54 | 55 | TT_MAC_ID_KHMER 56 | 57 | TT_MAC_ID_UNINTERP 58 | 59 | TT_MAC_ID_ORIYA 60 | 61 | TT_MAC_ID_RSYMBOL 62 | 63 | TT_MAC_ID_MALAYALAM 64 | 65 | TT_MAC_ID_GEEZ 66 | 67 | TT_MAC_ID_KOREAN 68 | 69 | TT_MAC_ID_GUJARATI 70 | 71 | TT_MAC_ID_BENGALI 72 | 73 | TT_MAC_ID_ARABIC 74 | 75 | TT_MAC_ID_GEORGIAN 76 | 77 | TT_MAC_ID_ARMENIAN 78 | 79 | TT_MAC_ID_SLAVIC 80 | """ 81 | 82 | TT_MAC_IDS = { 83 | 'TT_MAC_ID_ROMAN' : 0, 84 | 'TT_MAC_ID_JAPANESE' : 1, 85 | 'TT_MAC_ID_TRADITIONAL_CHINESE' : 2, 86 | 'TT_MAC_ID_KOREAN' : 3, 87 | 'TT_MAC_ID_ARABIC' : 4, 88 | 'TT_MAC_ID_HEBREW' : 5, 89 | 'TT_MAC_ID_GREEK' : 6, 90 | 'TT_MAC_ID_RUSSIAN' : 7, 91 | 'TT_MAC_ID_RSYMBOL' : 8, 92 | 'TT_MAC_ID_DEVANAGARI' : 9, 93 | 'TT_MAC_ID_GURMUKHI' : 10, 94 | 'TT_MAC_ID_GUJARATI' : 11, 95 | 'TT_MAC_ID_ORIYA' : 12, 96 | 'TT_MAC_ID_BENGALI' : 13, 97 | 'TT_MAC_ID_TAMIL' : 14, 98 | 'TT_MAC_ID_TELUGU' : 15, 99 | 'TT_MAC_ID_KANNADA' : 16, 100 | 'TT_MAC_ID_MALAYALAM' : 17, 101 | 'TT_MAC_ID_SINHALESE' : 18, 102 | 'TT_MAC_ID_BURMESE' : 19, 103 | 'TT_MAC_ID_KHMER' : 20, 104 | 'TT_MAC_ID_THAI' : 21, 105 | 'TT_MAC_ID_LAOTIAN' : 22, 106 | 'TT_MAC_ID_GEORGIAN' : 23, 107 | 'TT_MAC_ID_ARMENIAN' : 24, 108 | 'TT_MAC_ID_MALDIVIAN' : 25, 109 | 'TT_MAC_ID_SIMPLIFIED_CHINESE' : 25, 110 | 'TT_MAC_ID_TIBETAN' : 26, 111 | 'TT_MAC_ID_MONGOLIAN' : 27, 112 | 'TT_MAC_ID_GEEZ' : 28, 113 | 'TT_MAC_ID_SLAVIC' : 29, 114 | 'TT_MAC_ID_VIETNAMESE' : 30, 115 | 'TT_MAC_ID_SINDHI' : 31, 116 | 'TT_MAC_ID_UNINTERP' : 32} 117 | globals().update(TT_MAC_IDS) 118 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_ms_ids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of valid values for the 'encoding_id' for TT_PLATFORM_MICROSOFT 11 | charmaps and name entries. 12 | 13 | 14 | TT_MS_ID_SYMBOL_CS 15 | 16 | Corresponds to Microsoft symbol encoding. See FT_ENCODING_MS_SYMBOL. 17 | 18 | 19 | TT_MS_ID_UNICODE_CS 20 | 21 | Corresponds to a Microsoft WGL4 charmap, matching Unicode. See 22 | FT_ENCODING_UNICODE. 23 | 24 | 25 | TT_MS_ID_SJIS 26 | 27 | Corresponds to SJIS Japanese encoding. See FT_ENCODING_SJIS. 28 | 29 | 30 | TT_MS_ID_GB2312 31 | 32 | Corresponds to Simplified Chinese as used in Mainland China. See 33 | FT_ENCODING_GB2312. 34 | 35 | 36 | TT_MS_ID_BIG_5 37 | 38 | Corresponds to Traditional Chinese as used in Taiwan and Hong Kong. See 39 | FT_ENCODING_BIG5. 40 | 41 | 42 | TT_MS_ID_WANSUNG 43 | 44 | Corresponds to Korean Wansung encoding. See FT_ENCODING_WANSUNG. 45 | 46 | TT_MS_ID_JOHAB 47 | 48 | Corresponds to Johab encoding. See FT_ENCODING_JOHAB. 49 | 50 | 51 | TT_MS_ID_UCS_4 52 | 53 | Corresponds to UCS-4 or UTF-32 charmaps. This has been added to the OpenType 54 | specification version 1.4 (mid-2001.) 55 | """ 56 | 57 | TT_MS_IDS = { 58 | 'TT_MS_ID_SYMBOL_CS' : 0, 59 | 'TT_MS_ID_UNICODE_CS' : 1, 60 | 'TT_MS_ID_SJIS' : 2, 61 | 'TT_MS_ID_GB2312' : 3, 62 | 'TT_MS_ID_BIG_5' : 4, 63 | 'TT_MS_ID_WANSUNG' : 5, 64 | 'TT_MS_ID_JOHAB' : 6, 65 | 'TT_MS_ID_UCS_4' : 10 } 66 | globals().update(TT_MS_IDS) 67 | 68 | 69 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_name_ids.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | 10 | """ 11 | Possible values of the 'name' identifier field in the name records of the TTF 12 | 'name' table. These values are platform independent. 13 | 14 | TT_NAME_ID_COPYRIGHT 15 | 16 | TT_NAME_ID_FONT_FAMILY 17 | 18 | TT_NAME_ID_FONT_SUBFAMILY 19 | 20 | TT_NAME_ID_UNIQUE_ID 21 | 22 | TT_NAME_ID_FULL_NAME 23 | 24 | TT_NAME_ID_VERSION_STRING 25 | 26 | TT_NAME_ID_PS_NAME 27 | 28 | TT_NAME_ID_TRADEMARK 29 | 30 | TT_NAME_ID_MANUFACTURER 31 | 32 | TT_NAME_ID_DESIGNER 33 | 34 | TT_NAME_ID_DESCRIPTION 35 | 36 | TT_NAME_ID_VENDOR_URL 37 | 38 | TT_NAME_ID_DESIGNER_URL 39 | 40 | TT_NAME_ID_LICENSE 41 | 42 | TT_NAME_ID_LICENSE_URL 43 | 44 | TT_NAME_ID_PREFERRED_FAMILY 45 | 46 | TT_NAME_ID_PREFERRED_SUBFAMILY 47 | 48 | TT_NAME_ID_MAC_FULL_NAME 49 | 50 | TT_NAME_ID_SAMPLE_TEXT 51 | 52 | TT_NAME_ID_CID_FINDFONT_NAME 53 | 54 | TT_NAME_ID_WWS_FAMILY 55 | 56 | TT_NAME_ID_WWS_SUBFAMILY 57 | """ 58 | 59 | 60 | TT_NAME_IDS = { 61 | 'TT_NAME_ID_COPYRIGHT' : 0, 62 | 'TT_NAME_ID_FONT_FAMILY' : 1, 63 | 'TT_NAME_ID_FONT_SUBFAMILY' : 2, 64 | 'TT_NAME_ID_UNIQUE_ID' : 3, 65 | 'TT_NAME_ID_FULL_NAME' : 4, 66 | 'TT_NAME_ID_VERSION_STRING' : 5, 67 | 'TT_NAME_ID_PS_NAME' : 6, 68 | 'TT_NAME_ID_TRADEMARK' : 7, 69 | 70 | # the following values are from the OpenType spec 71 | 'TT_NAME_ID_MANUFACTURER' : 8, 72 | 'TT_NAME_ID_DESIGNER' : 9, 73 | 'TT_NAME_ID_DESCRIPTION' : 10, 74 | 'TT_NAME_ID_VENDOR_URL' : 11, 75 | 'TT_NAME_ID_DESIGNER_URL' : 12, 76 | 'TT_NAME_ID_LICENSE' : 13, 77 | 'TT_NAME_ID_LICENSE_URL' : 14, 78 | # number 15 is reserved 79 | 'TT_NAME_ID_PREFERRED_FAMILY' : 16, 80 | 'TT_NAME_ID_PREFERRED_SUBFAMILY' : 17, 81 | 'TT_NAME_ID_MAC_FULL_NAME' : 18, 82 | 83 | # The following code is new as of 2000-01-21 84 | 'TT_NAME_ID_SAMPLE_TEXT' : 19, 85 | 86 | # This is new in OpenType 1.3 87 | 'TT_NAME_ID_CID_FINDFONT_NAME' : 20, 88 | 89 | # This is new in OpenType 1.5 90 | 'TT_NAME_ID_WWS_FAMILY' : 21, 91 | 'TT_NAME_ID_WWS_SUBFAMILY' : 22 } 92 | globals().update(TT_NAME_IDS) 93 | 94 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_enums/tt_platforms.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011-2012 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | """ 10 | A list of valid values for the 'platform_id' identifier code in FT_CharMapRec 11 | and FT_SfntName structures. 12 | 13 | 14 | TT_PLATFORM_APPLE_UNICODE 15 | 16 | Used by Apple to indicate a Unicode character map and/or name entry. See 17 | TT_APPLE_ID_XXX for corresponding 'encoding_id' values. Note that name 18 | entries in this format are coded as big-endian UCS-2 character codes only. 19 | 20 | 21 | TT_PLATFORM_MACINTOSH 22 | 23 | Used by Apple to indicate a MacOS-specific charmap and/or name entry. See 24 | TT_MAC_ID_XXX for corresponding 'encoding_id' values. Note that most TrueType 25 | fonts contain an Apple roman charmap to be usable on MacOS systems (even if 26 | they contain a Microsoft charmap as well). 27 | 28 | 29 | TT_PLATFORM_ISO 30 | 31 | This value was used to specify ISO/IEC 10646 charmaps. It is however now 32 | deprecated. See TT_ISO_ID_XXX for a list of corresponding 'encoding_id' 33 | values. 34 | 35 | 36 | TT_PLATFORM_MICROSOFT 37 | 38 | Used by Microsoft to indicate Windows-specific charmaps. See TT_MS_ID_XXX for 39 | a list of corresponding 'encoding_id' values. Note that most fonts contain a 40 | Unicode charmap using (TT_PLATFORM_MICROSOFT, TT_MS_ID_UNICODE_CS). 41 | 42 | 43 | TT_PLATFORM_CUSTOM 44 | 45 | Used to indicate application-specific charmaps. 46 | 47 | 48 | TT_PLATFORM_ADOBE 49 | 50 | This value isn't part of any font format specification, but is used by 51 | FreeType to report Adobe-specific charmaps in an FT_CharMapRec structure. See 52 | TT_ADOBE_ID_XXX. 53 | """ 54 | 55 | TT_PLATFORMS = { 56 | 'TT_PLATFORM_APPLE_UNICODE' : 0, 57 | 'TT_PLATFORM_MACINTOSH' : 1, 58 | 'TT_PLATFORM_ISO' : 2, # deprecated 59 | 'TT_PLATFORM_MICROSOFT' : 3, 60 | 'TT_PLATFORM_CUSTOM' : 4, 61 | 'TT_PLATFORM_ADOBE' : 7} # artificial 62 | globals().update(TT_PLATFORMS) 63 | 64 | -------------------------------------------------------------------------------- /veles/external/freetype/ft_errors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # ----------------------------------------------------------------------------- 4 | # 5 | # FreeType high-level python API - Copyright 2011 Nicolas P. Rougier 6 | # Distributed under the terms of the new BSD license. 7 | # 8 | # ----------------------------------------------------------------------------- 9 | ''' 10 | Internal exception with freetype error message 11 | ''' 12 | class FT_Exception(Exception): 13 | def __init__(self, errcode, message=''): 14 | self.message = message 15 | self.errcode = errcode 16 | 17 | def __str__(self): 18 | return '%s: %s (%s)'%(self.__class__.__name__, self.message, 19 | self._errors.get(self.errcode, 'unknown error')) 20 | 21 | _errors = { 22 | 0x00: "no error" , 23 | 0x01: "cannot open resource" , 24 | 0x02: "unknown file format" , 25 | 0x03: "broken file" , 26 | 0x04: "invalid FreeType version" , 27 | 0x05: "module version is too low" , 28 | 0x06: "invalid argument" , 29 | 0x07: "unimplemented feature" , 30 | 0x08: "broken table" , 31 | 0x09: "broken offset within table" , 32 | 0x10: "invalid glyph index" , 33 | 0x11: "invalid character code" , 34 | 0x12: "unsupported glyph image format" , 35 | 0x13: "cannot render this glyph format" , 36 | 0x14: "invalid outline" , 37 | 0x15: "invalid composite glyph" , 38 | 0x16: "too many hints" , 39 | 0x17: "invalid pixel size" , 40 | 0x20: "invalid object handle" , 41 | 0x21: "invalid library handle" , 42 | 0x22: "invalid module handle" , 43 | 0x23: "invalid face handle" , 44 | 0x24: "invalid size handle" , 45 | 0x25: "invalid glyph slot handle" , 46 | 0x26: "invalid charmap handle" , 47 | 0x27: "invalid cache manager handle" , 48 | 0x28: "invalid stream handle" , 49 | 0x30: "too many modules" , 50 | 0x31: "too many extensions" , 51 | 0x40: "out of memory" , 52 | 0x41: "unlisted object" , 53 | 0x51: "cannot open stream" , 54 | 0x52: "invalid stream seek" , 55 | 0x53: "invalid stream skip" , 56 | 0x54: "invalid stream read" , 57 | 0x55: "invalid stream operation" , 58 | 0x56: "invalid frame operation" , 59 | 0x57: "nested frame access" , 60 | 0x58: "invalid frame read" , 61 | 0x60: "raster uninitialized" , 62 | 0x61: "raster corrupted" , 63 | 0x62: "raster overflow" , 64 | 0x63: "negative height while rastering" , 65 | 0x70: "too many registered caches" , 66 | 0x80: "invalid opcode" , 67 | 0x81: "too few arguments" , 68 | 0x82: "stack overflow" , 69 | 0x83: "code overflow" , 70 | 0x84: "bad argument" , 71 | 0x85: "division by zero" , 72 | 0x86: "invalid reference" , 73 | 0x87: "found debug opcode" , 74 | 0x88: "found ENDF opcode in execution stream" , 75 | 0x89: "nested DEFS" , 76 | 0x8A: "invalid code range" , 77 | 0x8B: "execution context too long" , 78 | 0x8C: "too many function definitions" , 79 | 0x8D: "too many instruction definitions" , 80 | 0x8E: "SFNT font table missing" , 81 | 0x8F: "horizontal header (hhea, table missing" , 82 | 0x90: "locations (loca, table missing" , 83 | 0x91: "name table missing" , 84 | 0x92: "character map (cmap, table missing" , 85 | 0x93: "horizontal metrics (hmtx, table missing" , 86 | 0x94: "PostScript (post, table missing" , 87 | 0x95: "invalid horizontal metrics" , 88 | 0x96: "invalid character map (cmap, format" , 89 | 0x97: "invalid ppem value" , 90 | 0x98: "invalid vertical metrics" , 91 | 0x99: "could not find context" , 92 | 0x9A: "invalid PostScript (post, table format" , 93 | 0x9B: "invalid PostScript (post, table" , 94 | 0xA0: "opcode syntax error" , 95 | 0xA1: "argument stack underflow" , 96 | 0xA2: "ignore" , 97 | 0xB0: "`STARTFONT' field missing" , 98 | 0xB1: "`FONT' field missing" , 99 | 0xB2: "`SIZE' field missing" , 100 | 0xB3: "`CHARS' field missing" , 101 | 0xB4: "`STARTCHAR' field missing" , 102 | 0xB5: "`ENCODING' field missing" , 103 | 0xB6: "`BBX' field missing" , 104 | 0xB7: "`BBX' too big" , 105 | } 106 | -------------------------------------------------------------------------------- /veles/external/kitchen/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'markhor' 2 | -------------------------------------------------------------------------------- /veles/external/kitchen/text/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'markhor' 2 | -------------------------------------------------------------------------------- /veles/external/progressbar/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # progressbar - Text progress bar library for Python. 5 | # Copyright (c) 2005 Nilton Volpato 6 | # (c) 2015 Samsung Electronics Co., Ltd. 7 | # 8 | # This library is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU Lesser General Public 10 | # License as published by the Free Software Foundation; either 11 | # version 2.1 of the License, or (at your option) any later version. 12 | # 13 | # This library is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | # Lesser General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU Lesser General Public 19 | # License along with this library; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 21 | 22 | """Text progress bar library for Python. 23 | 24 | A text progress bar is typically used to display the progress of a long 25 | running operation, providing a visual cue that processing is underway. 26 | 27 | The ProgressBar class manages the current progress, and the format of the line 28 | is given by a number of widgets. A widget is an object that may display 29 | differently depending on the state of the progress bar. There are three types 30 | of widgets: 31 | - a string, which always shows itself 32 | 33 | - a ProgressBarWidget, which may return a different value every time its 34 | update method is called 35 | 36 | - a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it 37 | expands to fill the remaining width of the line. 38 | 39 | The progressbar module is very easy to use, yet very powerful. It will also 40 | automatically enable features like auto-resizing when the system supports it. 41 | """ 42 | 43 | __author__ = 'Nilton Volpato, Vadim Markovtsev' 44 | __author_email__ = 'first-name dot last-name @ gmail.com;' \ 45 | 'v.markovtsev@samsung.com' 46 | __date__ = '2015-03-24' 47 | __version__ = '3.0' 48 | 49 | from .compat import * 50 | from .widgets import * 51 | from .progressbar import * 52 | from .spinner import spin 53 | -------------------------------------------------------------------------------- /veles/external/progressbar/compat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # progressbar - Text progress bar library for Python. 5 | # Copyright (c) 2005 Nilton Volpato 6 | # (c) 2015 Samsung Electronics Co., Ltd. 7 | # 8 | # This library is free software; you can redistribute it and/or 9 | # modify it under the terms of the GNU Lesser General Public 10 | # License as published by the Free Software Foundation; either 11 | # version 2.1 of the License, or (at your option) any later version. 12 | # 13 | # This library is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | # Lesser General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU Lesser General Public 19 | # License along with this library; if not, write to the Free Software 20 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 21 | 22 | """Compatibility methods and classes for the progressbar module.""" 23 | 24 | 25 | # Python 3.x (and backports) use a modified iterator syntax 26 | # This will allow 2.x to behave with 3.x iterators 27 | try: 28 | next 29 | except NameError: 30 | def next(iter): 31 | try: 32 | # Try new style iterators 33 | return iter.__next__() 34 | except AttributeError: 35 | # Fallback in case of a "native" iterator 36 | return iter.next() 37 | 38 | 39 | # Python < 2.5 does not have "any" 40 | try: 41 | any 42 | except NameError: 43 | def any(iterator): 44 | for item in iterator: 45 | if item: return True 46 | return False 47 | -------------------------------------------------------------------------------- /veles/external/progressbar/spinner.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # progressbar - Text progress bar library for Python. 3 | # Copyright (c) 2005 Nilton Volpato 4 | # (c) 2015 Samsung Electronics Co., Ltd. 5 | # 6 | # This library is free software; you can redistribute it and/or 7 | # modify it under the terms of the GNU Lesser General Public 8 | # License as published by the Free Software Foundation; either 9 | # version 2.1 of the License, or (at your option) any later version. 10 | # 11 | # This library is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # Lesser General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU Lesser General Public 17 | # License along with this library; if not, write to the Free Software 18 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 19 | 20 | """ 21 | Prints a spinning dash every once and a while. 22 | """ 23 | 24 | from itertools import cycle 25 | import sys 26 | from time import clock 27 | from threading import Lock 28 | try: 29 | from twisted.internet import reactor 30 | except ImportError: 31 | reactor = None 32 | 33 | 34 | __symbols = cycle(('|\r', '/\r', '—\r', '\\\r')) 35 | __lock = Lock() 36 | __last_flush_time = clock() 37 | stdout = None 38 | 39 | 40 | def spin(interval=0.2): 41 | with __lock: 42 | global stdout 43 | if stdout is None or stdout.closed: 44 | stdout = sys.stdout 45 | if stdout.closed: 46 | return 47 | stdout.write(next(__symbols)) 48 | if reactor is None or not reactor.running: 49 | time = clock() 50 | global __last_flush_time 51 | if time - __last_flush_time < interval: 52 | return 53 | stdout.flush() 54 | __last_flush_time = time 55 | else: 56 | reactor.callFromThread(stdout.flush) 57 | -------------------------------------------------------------------------------- /veles/external/pydev/__init__.py: -------------------------------------------------------------------------------- 1 | from .reload import xreload 2 | -------------------------------------------------------------------------------- /veles/external/pydev/dont_trace.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Support for a tag that allows skipping over functions while debugging. 3 | ''' 4 | import linecache 5 | import re 6 | from .constants import DictContains 7 | 8 | # To suppress tracing a method, add the tag @DontTrace 9 | # to a comment either preceding or on the same line as 10 | # the method definition 11 | # 12 | # E.g.: 13 | # #@DontTrace 14 | # def test1(): 15 | # pass 16 | # 17 | # ... or ... 18 | # 19 | # def test2(): #@DontTrace 20 | # pass 21 | DONT_TRACE_TAG = '@DontTrace' 22 | 23 | # Regular expression to match a decorator (at the beginning 24 | # of a line). 25 | RE_DECORATOR = re.compile(r'^\s*@') 26 | 27 | # Mapping from code object to bool. 28 | # If the key exists, the value is the cached result of should_trace_hook 29 | _filename_to_ignored_lines = {} 30 | 31 | def default_should_trace_hook(frame, filename): 32 | ''' 33 | Return True if this frame should be traced, False if tracing should be blocked. 34 | ''' 35 | # First, check whether this code object has a cached value 36 | ignored_lines = _filename_to_ignored_lines.get(filename) 37 | if ignored_lines is None: 38 | # Now, look up that line of code and check for a @DontTrace 39 | # preceding or on the same line as the method. 40 | # E.g.: 41 | # #@DontTrace 42 | # def test(): 43 | # pass 44 | # ... or ... 45 | # def test(): #@DontTrace 46 | # pass 47 | ignored_lines = {} 48 | lines = linecache.getlines(filename) 49 | i_line = 0 # Could use enumerate, but not there on all versions... 50 | for line in lines: 51 | j = line.find('#') 52 | if j >= 0: 53 | comment = line[j:] 54 | if DONT_TRACE_TAG in comment: 55 | ignored_lines[i_line] = 1 56 | 57 | #Note: when it's found in the comment, mark it up and down for the decorator lines found. 58 | k = i_line - 1 59 | while k >= 0: 60 | if RE_DECORATOR.match(lines[k]): 61 | ignored_lines[k] = 1 62 | k -= 1 63 | else: 64 | break 65 | 66 | k = i_line + 1 67 | while k <= len(lines): 68 | if RE_DECORATOR.match(lines[k]): 69 | ignored_lines[k] = 1 70 | k += 1 71 | else: 72 | break 73 | 74 | i_line += 1 75 | 76 | 77 | _filename_to_ignored_lines[filename] = ignored_lines 78 | 79 | func_line = frame.f_code.co_firstlineno - 1 # co_firstlineno is 1-based, so -1 is needed 80 | return not ( 81 | DictContains(ignored_lines, func_line - 1) or #-1 to get line before method 82 | DictContains(ignored_lines, func_line)) #method line 83 | 84 | 85 | should_trace_hook = None 86 | 87 | 88 | def clear_trace_filter_cache(): 89 | ''' 90 | Clear the trace filter cache. 91 | Call this after reloading. 92 | ''' 93 | global should_trace_hook 94 | try: 95 | # Need to temporarily disable a hook because otherwise 96 | # _filename_to_ignored_lines.clear() will never complete. 97 | old_hook = should_trace_hook 98 | should_trace_hook = None 99 | 100 | # Clear the linecache 101 | linecache.clearcache() 102 | _filename_to_ignored_lines.clear() 103 | 104 | finally: 105 | should_trace_hook = old_hook 106 | 107 | 108 | def trace_filter(mode): 109 | ''' 110 | Set the trace filter mode. 111 | 112 | mode: Whether to enable the trace hook. 113 | True: Trace filtering on (skipping methods tagged @DontTrace) 114 | False: Trace filtering off (trace methods tagged @DontTrace) 115 | None/default: Toggle trace filtering. 116 | ''' 117 | global should_trace_hook 118 | if mode is None: 119 | mode = should_trace_hook is None 120 | 121 | if mode: 122 | should_trace_hook = default_should_trace_hook 123 | else: 124 | should_trace_hook = None 125 | 126 | return mode 127 | 128 | -------------------------------------------------------------------------------- /veles/external/pydev/exec1.py: -------------------------------------------------------------------------------- 1 | def Exec(exp, global_vars, local_vars=None): 2 | if local_vars is not None: 3 | exec exp in global_vars, local_vars 4 | else: 5 | exec exp in global_vars -------------------------------------------------------------------------------- /veles/external/pydev/exec2.py: -------------------------------------------------------------------------------- 1 | def Exec(exp, global_vars, local_vars=None): 2 | if local_vars is not None: 3 | exec(exp, global_vars, local_vars) 4 | else: 5 | exec(exp, global_vars) -------------------------------------------------------------------------------- /veles/forge/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'markhor' 2 | -------------------------------------------------------------------------------- /veles/forge/templates/confirmation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | VelesForge 6 | 7 | 8 |

The confirmation email has been sent to {{ email }}

9 | 10 | -------------------------------------------------------------------------------- /veles/forge/templates/email.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Test letter with {{ link }}. 4 | 5 | -------------------------------------------------------------------------------- /veles/forge/templates/email.txt: -------------------------------------------------------------------------------- 1 | Test letter with {{ link }} -------------------------------------------------------------------------------- /veles/forge/templates/email_un.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Test letter with {{ link }}. 4 | 5 | -------------------------------------------------------------------------------- /veles/forge/templates/email_un.txt: -------------------------------------------------------------------------------- 1 | Test letter with {{ link }} -------------------------------------------------------------------------------- /veles/forge/templates/not_registered.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | VelesForge 6 | 7 | 8 |

You are not registered.

9 | 10 | -------------------------------------------------------------------------------- /veles/forge/templates/successful_registration.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | VelesForge 6 | 7 | 8 |

Registered, your token is {{ token }}

9 | 10 | -------------------------------------------------------------------------------- /veles/forge/templates/successful_unregistration.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | VelesForge 6 | 7 | 8 |

Successfully unregistered.

9 | 10 | -------------------------------------------------------------------------------- /veles/forge/templates/unconfirmation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | VelesForge 6 | 7 | 8 |

The confirmation email has been sent to {{ email }}

9 | 10 | -------------------------------------------------------------------------------- /veles/forge_common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Nov 10, 2014 12 | 13 | Common code between VelesForge server and client classes. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | from pkg_resources import Requirement 38 | from six import text_type 39 | 40 | 41 | REQUIRED_MANIFEST_FIELDS = { 42 | "name", "workflow", "configuration", "short_description", 43 | "long_description", "author", "requires" 44 | } 45 | 46 | 47 | def validate_requires(requires): 48 | if not isinstance(requires, list): 49 | raise TypeError("\"requires\" must be an instance of []") 50 | packages = set() 51 | for item in requires: 52 | if not isinstance(item, text_type): 53 | raise TypeError("Each item in \"requires\" must be " 54 | "a requirements.txt style string") 55 | pn = Requirement.parse(item).project_name 56 | if pn in packages: 57 | raise ValueError("Package %s was listed in \"requires\" more than " 58 | "once" % pn) 59 | packages.add(pn) 60 | -------------------------------------------------------------------------------- /veles/genetics/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Sep 8, 2014 12 | 13 | Classes for genetic optimizations. 14 | 15 | In general: 16 | Gene - contains parameter to optimize. 17 | Chromosome - contains set of genes. 18 | Individual - contains set of chromosomes. 19 | Population - contains set of individuals. 20 | 21 | For now: 22 | Chromosome - parameters to optimize. 23 | Population - list of chromosomes. 24 | 25 | ███████████████████████████████████████████████████████████████████████████████ 26 | 27 | Licensed to the Apache Software Foundation (ASF) under one 28 | or more contributor license agreements. See the NOTICE file 29 | distributed with this work for additional information 30 | regarding copyright ownership. The ASF licenses this file 31 | to you under the Apache License, Version 2.0 (the 32 | "License"); you may not use this file except in compliance 33 | with the License. You may obtain a copy of the License at 34 | 35 | http://www.apache.org/licenses/LICENSE-2.0 36 | 37 | Unless required by applicable law or agreed to in writing, 38 | software distributed under the License is distributed on an 39 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 40 | KIND, either express or implied. See the License for the 41 | specific language governing permissions and limitations 42 | under the License. 43 | 44 | ███████████████████████████████████████████████████████████████████████████████ 45 | """ 46 | 47 | 48 | from veles.genetics.config import fix_config, Range 49 | -------------------------------------------------------------------------------- /veles/genre_recognition.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /veles/import_file.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on March 12, 2015 12 | 13 | Common functions which are used for importing workflow modules. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from importlib import import_module 39 | import os 40 | import sys 41 | from types import ModuleType 42 | 43 | 44 | def get_file_package_and_module(file_name): 45 | package_name = os.path.basename(os.path.dirname(file_name)) 46 | module_name = os.path.splitext(os.path.basename(file_name))[0] 47 | return package_name, module_name 48 | 49 | 50 | def import_file_as_package(file_name): 51 | """ 52 | Imports the file as .. Raises 53 | exceptions if fails. 54 | :param file_name: The path to import. 55 | :return: The loaded module. 56 | """ 57 | package_name, module_name = get_file_package_and_module( 58 | file_name) 59 | sys.path.insert(0, os.path.dirname(os.path.dirname(file_name))) 60 | try: 61 | return import_module("%s.%s" % (package_name, module_name)) 62 | finally: 63 | del sys.path[0] 64 | 65 | 66 | def import_file_as_module(file_name): 67 | """ 68 | Imports the file as . Raises exceptions if fails. 69 | :param file_name: The path to import. 70 | :return: The loaded module. 71 | """ 72 | _, module_name = get_file_package_and_module(file_name) 73 | sys.path.insert(0, os.path.dirname(file_name)) 74 | try: 75 | return __import__(module_name) 76 | finally: 77 | del sys.path[0] 78 | 79 | 80 | def try_to_import_file(file_name): 81 | """ 82 | Tries to import the file as Python module. First calls 83 | import_file_as_package() and falls back to import_file_as_module(). If 84 | fails, keeps silent on any errors and returns the occured exceptions. 85 | :param file_name: The path to import. 86 | :return: The loaded module or tuple of length 2 with the exceptions. 87 | """ 88 | try: 89 | return import_file_as_package(file_name) 90 | except Exception as e1: 91 | try: 92 | return import_file_as_module(file_name) 93 | except Exception as e2: 94 | return e1, e2 95 | 96 | 97 | def is_module(obj): 98 | return isinstance(obj, ModuleType) 99 | -------------------------------------------------------------------------------- /veles/interaction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 7, 2014 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | from IPython.config.loader import Config 37 | from IPython.terminal.embed import InteractiveShellEmbed 38 | from IPython.utils.io import devnull 39 | devnull.close() 40 | import select 41 | import sys 42 | from zope.interface import implementer 43 | 44 | from veles.distributable import TriviallyDistributable 45 | from veles.units import Unit, IUnit 46 | 47 | 48 | @implementer(IUnit) 49 | class Shell(Unit, TriviallyDistributable): 50 | """ 51 | Runs embedded IPython 52 | """ 53 | BANNER1 = "\nVELES interactive console" 54 | BANNER2 = "Type in 'workflow' or 'units' to start" 55 | 56 | def __init__(self, workflow, **kwargs): 57 | super(Shell, self).__init__(workflow, **kwargs) 58 | self.cfg = Config() 59 | self.cfg.PromptManager.in_template = "veles [\\#]> " 60 | self.cfg.PromptManager.out_template = "veles [\\#]: " 61 | self.cfg.HistoryManager.enabled = False 62 | 63 | def initialize(self, **kwargs): 64 | if self.interactive: 65 | return 66 | self.shell_ = InteractiveShellEmbed(config=self.cfg, 67 | banner1=Shell.BANNER1, 68 | banner2=Shell.BANNER2) 69 | 70 | def interact(self, extra_locals=None): 71 | workflow = self.workflow # pylint: disable=W0612 72 | units = self.workflow.units # pylint: disable=W0612 73 | exec('\n'.join("%s = extra_locals['%s']" % (k, k) 74 | for k in extra_locals or {})) 75 | self.shell_() 76 | 77 | @staticmethod 78 | def fix_netcat_colors(): 79 | from IPython.core import prompts 80 | for scheme in (prompts.PColLinux, prompts.PColLightBG): 81 | colors = scheme.colors 82 | for key, val in colors.items(): 83 | colors[key] = val.replace('\x01', '').replace('\x02', '') 84 | 85 | def run(self): 86 | if self.interactive or not sys.stdin.isatty(): 87 | return 88 | key = '' 89 | i, _, _ = select.select([sys.stdin], [], [], 0) 90 | for s in i: 91 | if s == sys.stdin: 92 | key = sys.stdin.readline()[0] 93 | break 94 | if key == 'i': 95 | self.interact() 96 | -------------------------------------------------------------------------------- /veles/iplotter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Nov 5, 2014 12 | 13 | Interface which states that the object supports plotting in GraphicsClient 14 | environment. 15 | 16 | ███████████████████████████████████████████████████████████████████████████████ 17 | 18 | Licensed to the Apache Software Foundation (ASF) under one 19 | or more contributor license agreements. See the NOTICE file 20 | distributed with this work for additional information 21 | regarding copyright ownership. The ASF licenses this file 22 | to you under the Apache License, Version 2.0 (the 23 | "License"); you may not use this file except in compliance 24 | with the License. You may obtain a copy of the License at 25 | 26 | http://www.apache.org/licenses/LICENSE-2.0 27 | 28 | Unless required by applicable law or agreed to in writing, 29 | software distributed under the License is distributed on an 30 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 31 | KIND, either express or implied. See the License for the 32 | specific language governing permissions and limitations 33 | under the License. 34 | 35 | ███████████████████████████████████████████████████████████████████████████████ 36 | """ 37 | 38 | 39 | from zope.interface import Interface, Attribute 40 | 41 | 42 | class IPlotter(Interface): 43 | """Interface which states that the object supports plotting in 44 | GraphicsClient environment. 45 | """ 46 | 47 | matplotlib = Attribute("""matplotlib module reference""") 48 | cm = Attribute("""matplotlib.cm (colormap) module reference""") 49 | lines = Attribute("""matplotlib.lines module reference""") 50 | patches = Attribute("""matplotlib.patches module reference""") 51 | pp = Attribute("""matplotlib.pyplot module reference""") 52 | 53 | def redraw(): 54 | """Updates the plot using the changed object's state. 55 | Should be implemented by the class providing this interface. 56 | """ 57 | -------------------------------------------------------------------------------- /veles/json_encoders.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on June 12, 2015 12 | 13 | Numpy support class for json.dump(s). 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | import json 39 | import numpy 40 | 41 | from veles.config import Config 42 | from veles.memory import Array 43 | 44 | 45 | class NumpyJSONEncoder(json.JSONEncoder): 46 | def default(self, obj): 47 | if isinstance(obj, Array): 48 | obj.map_read() 49 | obj = obj.mem 50 | if isinstance(obj, numpy.ndarray): 51 | return obj.tolist() 52 | elif isinstance(obj, numpy.number): 53 | return obj.item() 54 | elif isinstance(obj, (complex, numpy.complex)): 55 | return [obj.real, obj.imag] 56 | elif isinstance(obj, set): 57 | return list(obj) 58 | elif isinstance(obj, bytes): 59 | return obj.decode("charmap") 60 | return json.JSONEncoder.default(self, obj) 61 | 62 | 63 | class ConfigJSONEncoder(NumpyJSONEncoder): 64 | def default(self, obj): 65 | if isinstance(obj, Config): 66 | return obj.__content__ 67 | return super(ConfigJSONEncoder, self).default(obj) 68 | -------------------------------------------------------------------------------- /veles/loader/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jun 7, 2014 12 | 13 | Loader package 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from veles.loader.base import ILoader, Loader, CLASS_NAME, TARGET, TRAIN, \ 39 | VALID, TEST 40 | from veles.loader.fullbatch import IFullBatchLoader, FullBatchLoader, \ 41 | FullBatchLoaderMSE 42 | from veles.loader.file_loader import IFileLoader 43 | try: 44 | from veles.loader.image import IImageLoader, ImageLoader 45 | from veles.loader.file_image import FileImageLoader, \ 46 | AutoLabelFileImageLoader 47 | from veles.loader.fullbatch_image import FullBatchFileImageLoader, \ 48 | FullBatchFileImageLoaderMSE, FullBatchAutoLabelFileImageLoader 49 | from veles.loader.pickles import PicklesImageFullBatchLoader 50 | from veles.loader.interactive import InteractiveLoader, \ 51 | InteractiveImageLoader 52 | except ImportError as e: 53 | import warnings 54 | warnings.warn("Image loaders will be unavailable: %s" % e) 55 | IImageLoader = ImageLoader = AutoLabelFileImageLoader = FileImageLoader = \ 56 | FullBatchFileImageLoader = FullBatchFileImageLoaderMSE = \ 57 | FullBatchAutoLabelFileImageLoader = PicklesImageFullBatchLoader = \ 58 | InteractiveLoader = InteractiveImageLoader = object 59 | 60 | from veles.loader.saver import MinibatchesSaver, MinibatchesLoader 61 | -------------------------------------------------------------------------------- /veles/loader/hdfs_loader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Feb 2, 2015 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | import six 37 | from snakebite.client import Client 38 | from zope.interface import implementer 39 | 40 | from veles.distributable import TriviallyDistributable 41 | from veles.loader.base import UserLoaderRegistry 42 | from veles.mutable import Bool 43 | from veles.units import IUnit, Unit 44 | 45 | 46 | @six.add_metaclass(UserLoaderRegistry) 47 | @implementer(IUnit) 48 | class HDFSTextLoader(Unit, TriviallyDistributable): 49 | def __init__(self, workflow, **kwargs): 50 | super(HDFSTextLoader, self).__init__(workflow, **kwargs) 51 | self.file_name = kwargs["file"] 52 | self.chunk_lines_number = kwargs.get("chunk", 1000) 53 | client_kwargs = dict(kwargs) 54 | del client_kwargs["file"] 55 | if "chunk" in kwargs: 56 | del client_kwargs["chunk"] 57 | self.hdfs_client = Client(**client_kwargs) 58 | self.output = [""] * self.chunk_lines_number 59 | self.finished = Bool() 60 | 61 | def initialize(self): 62 | self.debug("Opened %s", self.hdfs_client.stat([self.file_name])) 63 | self._generator = self.hdfs_client.text([self.file_name]) 64 | 65 | def run(self): 66 | assert not self.finished 67 | try: 68 | for i in range(self.chunk_lines_number): 69 | self.output[i] = next(self._generator) 70 | except StopIteration: 71 | self.finished <<= True 72 | -------------------------------------------------------------------------------- /veles/loader/libsndfile_loader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on May 21, 2013 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | import logging 37 | from ctypes import byref, c_short, c_char_p, POINTER 38 | 39 | import numpy 40 | 41 | import veles.error as error 42 | from veles.loader.file_loader import FileLoaderBase, FileListLoaderBase 43 | from veles.loader.libsndfile import SF_INFO, libsndfile 44 | 45 | 46 | class SndFileMixin(object): 47 | @staticmethod 48 | def open_file(file_name): 49 | info = SF_INFO() 50 | info.format = 0 51 | handle = libsndfile().sf_open( 52 | c_char_p(file_name.encode()), libsndfile.SFM_READ, byref(info)) 53 | if not handle: 54 | raise error.BadFormatError( 55 | "Audio file %s does not exist or is in an unsupported format" % 56 | file_name) 57 | 58 | if info.channels > 2: 59 | raise error.BadFormatError("Audio file " + file_name + 60 | " has more than two channels. " 61 | "Only mono or stereo are allowed.") 62 | return {"handle": handle, "samples": info.frames, 63 | "sampling_rate": info.samplerate, "channels": info.channels, 64 | "info": info} 65 | 66 | @staticmethod 67 | def close_file(opened_data): 68 | libsndfile().sf_close(opened_data["handle"]) 69 | 70 | @staticmethod 71 | def decode_file(file_name): 72 | opened_data = SndFileLoaderBase.open_file(file_name) 73 | handle = opened_data["handle"] 74 | info = opened_data["info"] 75 | data = numpy.zeros(info.frames * info.channels, dtype=numpy.int16) 76 | libsndfile().sf_readf_short(handle, 77 | data.ctypes.data_as(POINTER(c_short)), 78 | info.frames) 79 | libsndfile().sf_close(handle) 80 | logging.info("Loaded " + file_name + ": " + 81 | info.str_format() + ", " + str(info.frames) + 82 | " samples at " + str(info.samplerate) + " Hz with " + 83 | str(info.channels) + " channels") 84 | return {"data": data, "sampling_rate": info.samplerate, 85 | "samples": info.frames, "channels": info.channels, 86 | "name": file_name} 87 | 88 | @staticmethod 89 | def file_format(opened_data): 90 | return opened_data["info"].str_format() 91 | 92 | 93 | class SndFileLoaderBase(SndFileMixin, FileLoaderBase): 94 | """ 95 | Decodes the specified audio file to the raw signed PCM 16 bit format 96 | using libsndfile. 97 | """ 98 | 99 | def __init__(self, workflow, **kwargs): 100 | kwargs["file_type"] = "audio" 101 | kwargs["file_subtypes"] = kwargs.get( 102 | "file_subtypes", ["basic", "ogg", "wav", "wave", "x-wav", 103 | "x-flac"]) 104 | super(SndFileLoaderBase, self).__init__(workflow, **kwargs) 105 | 106 | 107 | class SndFileListLoaderBase(SndFileMixin, FileListLoaderBase): 108 | pass 109 | -------------------------------------------------------------------------------- /veles/mapped_object_registry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Nov 5, 2013 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | class MappedObjectsRegistry(type): 37 | """ 38 | General purpose mapping between names and real classes. 39 | Derive your registry class from this base and the attribute defined by 40 | "mapping" will point to the dictionary between strings and classes, which 41 | metaclass is your registry class. 42 | """ 43 | mapping = "You must define \"mapping\" static attribute in your metaclass" 44 | base = object 45 | 46 | def __init__(cls, name, bases, clsdict): 47 | yours = set(cls.mro()) 48 | mine = set(cls.base.mro()) 49 | left = yours - mine 50 | mapping = getattr(type(cls), cls.mapping, {}) 51 | if len(left) > 1 and "MAPPING" in clsdict: 52 | mapping[clsdict["MAPPING"]] = cls 53 | setattr(type(cls), cls.mapping, mapping) 54 | super(MappedObjectsRegistry, cls).__init__(name, bases, clsdict) 55 | -------------------------------------------------------------------------------- /veles/numpy_ext.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on March 11, 2015 12 | 13 | Numpy extension functions which ensure virtual address persistence. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | import numpy 39 | 40 | 41 | def max_type(num): 42 | """Returns array converted to supported type with maximum precision. 43 | """ 44 | return num.astype(numpy.float64) 45 | 46 | 47 | def eq_addr(a, b): 48 | return a.__array_interface__["data"][0] == b.__array_interface__["data"][0] 49 | 50 | 51 | def assert_addr(a, b): 52 | """Raises an exception if addresses of the supplied arrays differ. 53 | """ 54 | if not eq_addr(a, b): 55 | raise ValueError("Addresses of the arrays are not equal.") 56 | 57 | 58 | def ravel(a): 59 | """numpy.ravel() with address check. 60 | """ 61 | b = a.ravel() 62 | assert_addr(a, b) 63 | return b 64 | 65 | 66 | def reshape(a, shape): 67 | """numpy.reshape() with address check. 68 | """ 69 | b = a.reshape(shape) 70 | assert_addr(a, b) 71 | return b 72 | 73 | 74 | def reshape_transposed(w): 75 | """Reshapes weights as if they were transposed. 76 | """ 77 | a = w.reshape(*w.shape[1::-1]) 78 | assert_addr(a, w) 79 | return a 80 | 81 | 82 | def transpose(a): 83 | """numpy.transpose() with address check. 84 | """ 85 | b = a.transpose() 86 | assert_addr(a, b) 87 | return b 88 | 89 | 90 | def interleave(arr): 91 | """Returns the interleaved array. 92 | 93 | Example: 94 | [10000, 3, 32, 32] => [10000, 32, 32, 3]. 95 | """ 96 | last = arr.shape[-3] 97 | b = numpy.empty((arr.shape[0],) + arr.shape[2:] + (last,), arr.dtype) 98 | if len(b.shape) == 4: 99 | for i in range(last): 100 | b[:, :, :, i] = arr[:, i, :, :] 101 | elif len(b.shape) == 3: 102 | for i in range(last): 103 | b[:, :, i] = arr[i, :, :] 104 | else: 105 | raise ValueError("a should be of shape 4 or 3.") 106 | return b 107 | 108 | 109 | def roundup(num, align): 110 | d = num % align 111 | if d == 0: 112 | return num 113 | return num + (align - d) 114 | 115 | 116 | class NumDiff(object): 117 | """Numeric differentiation helper. 118 | 119 | WARNING: it is invalid for single precision float data type. 120 | """ 121 | 122 | def __init__(self): 123 | self.h = 1.0e-8 124 | self.points = (2.0 * self.h, self.h, -self.h, -2.0 * self.h) 125 | self.coeffs = numpy.array([-1.0, 8.0, -8.0, 1.0], 126 | dtype=numpy.float64) 127 | self.divizor = 12.0 * self.h 128 | self.errs = numpy.zeros_like(self.points) 129 | 130 | @property 131 | def derivative(self): 132 | return (self.errs * self.coeffs).sum() / self.divizor 133 | -------------------------------------------------------------------------------- /veles/opencl_types.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Feb 11, 2014 12 | 13 | Mapping between numpy and opencl types 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | import numpy 39 | 40 | 41 | # : CL type defines 42 | cl_defines = {"float": {"dtype": "float", 43 | "sizeof_dtype": "4"}, 44 | "double": {"dtype": "double", 45 | "sizeof_dtype": "8"}} 46 | 47 | 48 | # : Supported types as OpenCL => numpy dictionary. 49 | dtypes = {"float": numpy.float32, "double": numpy.float64} 50 | 51 | 52 | # : Map between numpy types and opencl. 53 | def numpy_dtype_to_opencl(dtype): 54 | if dtype == numpy.float32: 55 | return "float" 56 | if dtype == numpy.float64: 57 | return "double" 58 | if dtype == numpy.complex64: 59 | return "float2" 60 | if dtype == numpy.complex128: 61 | return "double2" 62 | if dtype == numpy.int8: 63 | return "char" 64 | if dtype == numpy.int16: 65 | return "short" 66 | if dtype == numpy.int32: 67 | return "int" 68 | if dtype == numpy.int64: 69 | return "long" 70 | if dtype == numpy.uint8: 71 | return "uchar" 72 | if dtype == numpy.uint16: 73 | return "ushort" 74 | if dtype == numpy.uint32: 75 | return "uint" 76 | if dtype == numpy.uint64: 77 | return "ulong" 78 | raise ValueError("Unknown dtype: %s" % dtype) 79 | -------------------------------------------------------------------------------- /veles/paths.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Nov 5, 2014 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | import os 37 | 38 | __root__ = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 39 | __home__ = os.path.join(os.environ.get("HOME", "./"), ".veles") 40 | -------------------------------------------------------------------------------- /veles/pickle2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on May 15, 2014 12 | 13 | Enables the interactive debugging of errors occured during pickling 14 | and unpickling. 15 | 16 | ███████████████████████████████████████████████████████████████████████████████ 17 | 18 | Licensed to the Apache Software Foundation (ASF) under one 19 | or more contributor license agreements. See the NOTICE file 20 | distributed with this work for additional information 21 | regarding copyright ownership. The ASF licenses this file 22 | to you under the Apache License, Version 2.0 (the 23 | "License"); you may not use this file except in compliance 24 | with the License. You may obtain a copy of the License at 25 | 26 | http://www.apache.org/licenses/LICENSE-2.0 27 | 28 | Unless required by applicable law or agreed to in writing, 29 | software distributed under the License is distributed on an 30 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 31 | KIND, either express or implied. See the License for the 32 | specific language governing permissions and limitations 33 | under the License. 34 | 35 | ███████████████████████████████████████████████████████████████████████████████ 36 | """ 37 | 38 | 39 | import six 40 | from six.moves import cPickle as pickle # pylint: disable=W0611 41 | import sys 42 | from pickle import PicklingError, UnpicklingError, HIGHEST_PROTOCOL 43 | 44 | 45 | # : The best protocol value for pickle(). 46 | best_protocol = HIGHEST_PROTOCOL 47 | 48 | 49 | def augment__str__(fn): 50 | def __str__(self): 51 | return "%s\n%s" % ( 52 | fn(self), 53 | "To debug this error, add --debug-pickle to the command line " 54 | "arguments or execute from veles.pickle2 " 55 | "import setup_pickle_debug; setup_pickle_debug()") 56 | return __str__ 57 | 58 | 59 | __pe__str__ = PicklingError.__str__ 60 | __upe__str__ = UnpicklingError.__str__ 61 | PicklingError.__str__ = augment__str__(PicklingError.__str__) 62 | UnpicklingError.__str__ = augment__str__(UnpicklingError.__str__) 63 | del augment__str__ 64 | 65 | 66 | def setup_pickle_debug(): 67 | """Enables the interactive debugging of errors occured during pickling 68 | and unpickling. 69 | """ 70 | if six.PY3: 71 | global pickle 72 | pickle.dump = pickle._dump 73 | pickle.dumps = pickle._dumps 74 | pickle.load = pickle._load 75 | pickle.loads = pickle._loads 76 | else: 77 | import pickle # pylint: disable=W0621 78 | for module in sys.modules.values(): 79 | if "pickle" in getattr(module, "__dict__", {}): 80 | module.__dict__["pickle"] = pickle 81 | pickle._Pickler = pickle.Pickler 82 | pickle._Unpickler = pickle.Unpickler 83 | orig_save = pickle._Pickler.save 84 | orig_load = pickle._Unpickler.load 85 | 86 | def save(self, obj): 87 | try: 88 | return orig_save(self, obj) 89 | except (PicklingError, TypeError, AssertionError): 90 | import traceback 91 | import pdb 92 | six.print_("\033[1;31mPickling failure\033[0m", file=sys.stderr) 93 | traceback.print_exc() 94 | # look at obj 95 | pdb.set_trace() 96 | 97 | def load(self): 98 | try: 99 | return orig_load(self) 100 | except (UnpicklingError, ImportError, AssertionError, KeyError): 101 | import traceback 102 | import pdb 103 | six.print_("\033[1;31mUnpickling failure\033[0m", file=sys.stderr) 104 | traceback.print_exc() 105 | # examine the exception 106 | pdb.post_mortem() 107 | 108 | pickle._Pickler.save = save 109 | pickle._Unpickler.load = load 110 | PicklingError.__str__ = __pe__str__ 111 | UnpicklingError.__str__ = __upe__str__ 112 | -------------------------------------------------------------------------------- /veles/plumbing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Apr 1, 2015 4 | 5 | Service units to change the control flow. 6 | 7 | Copyright (c) 2013 Samsung Electronics Co., Ltd. 8 | """ 9 | 10 | 11 | from zope.interface import implementer 12 | 13 | from veles.distributable import IDistributable, TriviallyDistributable 14 | from veles.units import TrivialUnit, Unit, IUnit 15 | 16 | 17 | class Repeater(TrivialUnit): 18 | """Completes a typical control flow cycle, usually joining the first unit 19 | with the last one. 20 | """ 21 | 22 | def __init__(self, workflow, **kwargs): 23 | kwargs["view_group"] = kwargs.get("view_group", "PLUMBING") 24 | kwargs["ignore_gate"] = True 25 | super(Repeater, self).__init__(workflow, **kwargs) 26 | 27 | def link_from(self, *args): 28 | super(Repeater, self).link_from(*args) 29 | if len(self.links_to) > 2: 30 | self.warning( 31 | "Repeater has more than 2 incoming links: %s. Are you sure?", 32 | tuple(self.links_to)) 33 | return self 34 | 35 | 36 | class UttermostPoint(TrivialUnit): 37 | hide_from_registry = True 38 | 39 | def __init__(self, workflow, **kwargs): 40 | kwargs["view_group"] = kwargs.get("view_group", "SERVICE") 41 | super(UttermostPoint, self).__init__(workflow, **kwargs) 42 | 43 | 44 | class StartPoint(UttermostPoint): 45 | """Workflow execution normally starts from this unit. 46 | """ 47 | hide_from_registry = True 48 | 49 | def __init__(self, workflow, **kwargs): 50 | kwargs["name"] = kwargs.get("name", "Start") 51 | super(StartPoint, self).__init__(workflow, **kwargs) 52 | 53 | @Unit.name.getter 54 | def name(self): 55 | if hasattr(self, "_workflow_") and self.workflow is not None: 56 | return "%s of %s" % (self._name, type(self.workflow).__name__) 57 | return Unit.name.fget(self) 58 | 59 | 60 | class EndPoint(UttermostPoint): 61 | """Ends the pipeline execution, normally is the last unit in a workflow. 62 | """ 63 | hide_from_registry = True 64 | 65 | def __init__(self, workflow, **kwargs): 66 | kwargs["name"] = kwargs.get("name", "End") 67 | super(EndPoint, self).__init__(workflow, **kwargs) 68 | 69 | def init_unpickled(self): 70 | super(EndPoint, self).init_unpickled() 71 | # on_workflow_finished() applies to Workflow's run time 72 | del Unit.timers[self.id] 73 | 74 | @Unit.name.getter 75 | def name(self): 76 | if hasattr(self, "_workflow_") and self.workflow is not None: 77 | return "%s of %s" % (self._name, type(self.workflow).__name__) 78 | return Unit.name.fget(self) 79 | 80 | def run(self): 81 | self.workflow.on_workflow_finished() 82 | 83 | def generate_data_for_master(self): 84 | return True 85 | 86 | def apply_data_from_slave(self, data, slave): 87 | if not self.gate_block: 88 | self.workflow.on_workflow_finished() 89 | 90 | 91 | @implementer(IDistributable, IUnit) 92 | class FireStarter(Unit, TriviallyDistributable): 93 | """ 94 | Resets :py:attr:`veles.units.Unit.stopped` of the associated units 95 | exposed as :py:attr:`veles.plumbing.FireStarter.units`. 96 | """ 97 | 98 | def __init__(self, workflow, **kwargs): 99 | kwargs["view_group"] = kwargs.get("view_group", "SERVICE") 100 | super(FireStarter, self).__init__(workflow, **kwargs) 101 | self._units = set(kwargs.get("units", tuple())) 102 | 103 | @property 104 | def units(self): 105 | return self._units 106 | 107 | def initialize(self, **kwargs): 108 | pass 109 | 110 | def run(self): 111 | for unit in self.units: 112 | unit.stopped = False 113 | -------------------------------------------------------------------------------- /veles/portable.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jul 17, 2014 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | import os 37 | import platform 38 | import subprocess 39 | 40 | 41 | def show_file(file_name): 42 | system = platform.system() 43 | if system == "Windows": 44 | os.startfile(file_name) 45 | elif system == "Linux": 46 | subprocess.Popen(["xdg-open", file_name]) 47 | -------------------------------------------------------------------------------- /veles/prng/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jun 27, 2014 12 | 13 | Package for random generators. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from veles.prng.random_generator import RandomGenerator, xorshift128plus, get 39 | # This effectively imports veles.backends and breaks veles.units 40 | # from veles.prng.uniform import Uniform 41 | -------------------------------------------------------------------------------- /veles/publishing/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2015 12 | 13 | Package for publishing results. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | from .publisher import Publisher 38 | import veles.publishing.confluence_backend 39 | -------------------------------------------------------------------------------- /veles/publishing/backend.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2015 12 | 13 | Base publishing backend class. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | from six import add_metaclass 38 | 39 | from veles.logger import Logger 40 | from veles.publishing.registry import PublishingBackendRegistry 41 | 42 | 43 | @add_metaclass(PublishingBackendRegistry) 44 | class Backend(Logger): 45 | def __init__(self, template, **kwargs): 46 | super(Backend, self).__init__() 47 | self.template = template 48 | -------------------------------------------------------------------------------- /veles/publishing/confluence_backend.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2015 12 | 13 | Publishes results to Atlassian Confluence based on Jinja2 templates. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from veles.publishing.confluence import Confluence 39 | from veles.publishing.jinja2_template_backend import Jinja2TemplateBackend 40 | 41 | 42 | class ConfluenceBackend(Jinja2TemplateBackend): 43 | MAPPING = "confluence" 44 | TEMPLATE_EXT = "xml" 45 | 46 | def __init__(self, template, **kwargs): 47 | super(ConfluenceBackend, self).__init__(template, **kwargs) 48 | self.confluence_access = {} 49 | for arg in "server", "username", "password": 50 | self.confluence_access[arg] = kwargs[arg] 51 | self.page = kwargs.get("page") 52 | self.parent = kwargs.get("parent") 53 | self.space = kwargs["space"] 54 | 55 | def render(self, info): 56 | content = super(ConfluenceBackend, self).render(info) 57 | page = self.page 58 | conf = Confluence(**self.confluence_access) 59 | if page is None: 60 | page = info["name"] 61 | index = 1 62 | while conf.get_page_summary(page, self.space) is not None: 63 | page = "%s (%d)" % (info["name"], index) 64 | index += 1 65 | self.info("Uploading the text (%d symbols)...", len(content)) 66 | published = conf.store_page_content( 67 | page, self.space, content, convert_wiki=False, parent=self.parent) 68 | url = published["url"] 69 | for name, data in info["plots"].items(): 70 | self.info("Attaching %s...", name) 71 | conf.attach_file(page, self.space, {"%s.png" % name: data["png"]}) 72 | self.info("Attaching the workflow graph...") 73 | conf.attach_file(page, self.space, 74 | {"workflow.png": info["workflow_graph"]["png"]}) 75 | if info["image"] is not None: 76 | self.info("Attaching the image...") 77 | conf.attach_file(page, self.space, 78 | {info["image"]["name"]: info["image"]["data"]}) 79 | self.info("Successfully published \"%s\" as %s", page, url) 80 | self.url = url 81 | self.page = page 82 | -------------------------------------------------------------------------------- /veles/publishing/ipynb_template.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "worksheets": [ 3 | { 4 | "cells": [ 5 | { 6 | "cell_type": "code", 7 | "execution_count": null, 8 | "metadata": {}, 9 | "outputs": [], 10 | "input": [ 11 | "" 12 | ], 13 | "language": "python" 14 | } 15 | ] 16 | } 17 | ], 18 | "cells": [], 19 | "metadata": {}, 20 | "nbformat": 3, 21 | "nbformat_minor": 0 22 | } -------------------------------------------------------------------------------- /veles/publishing/jinja2_template_backend.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2015 12 | 13 | Base class for publishing backends which use Jinja2 templates. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | import binascii 38 | import os 39 | import sys 40 | from jinja2 import Environment, FileSystemLoader 41 | 42 | from veles.publishing.backend import Backend 43 | 44 | 45 | class Jinja2TemplateBackend(Backend): 46 | def __init__(self, template, **kwargs): 47 | super(Jinja2TemplateBackend, self).__init__(template, **kwargs) 48 | self.environment = Environment( 49 | trim_blocks=kwargs.get("trim_blocks", True), 50 | lstrip_blocks=kwargs.get("lstrip_blocks", True), 51 | loader=FileSystemLoader(os.path.dirname( 52 | sys.modules[type(self).__module__].__file__))) 53 | if template is None: 54 | self.template = self.environment.get_template( 55 | "%s_template.%s" % (self.MAPPING, self.TEMPLATE_EXT)) 56 | else: 57 | self.template = self.environment.from_string(template) 58 | 59 | def render(self, info): 60 | info["hexlify"] = binascii.hexlify 61 | self.info("Rendering the template...") 62 | content = self.template.render(max=max, min=min, **info) 63 | self.debug("Rendered:\n%s", content) 64 | return content 65 | -------------------------------------------------------------------------------- /veles/publishing/latex_template.tex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Samsung/veles/96218b8d04ab099d65bfff350a24412a66c26178/veles/publishing/latex_template.tex -------------------------------------------------------------------------------- /veles/publishing/pdf_backend.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on June 30, 2015 12 | 13 | Extension for MarkdownBackend to write PDF reports. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | import os 39 | from tempfile import mkdtemp 40 | from shutil import rmtree 41 | from six import string_types 42 | from weasyprint import HTML 43 | 44 | from veles.publishing.backend import Backend 45 | from veles.publishing.markdown_backend import MarkdownBackend 46 | 47 | 48 | class PDFBackend(Backend): 49 | MAPPING = "pdf" 50 | 51 | def __init__(self, template, **kwargs): 52 | super(PDFBackend, self).__init__(template, **kwargs) 53 | self.markdown = MarkdownBackend( 54 | template, html=True, file=os.path.join(mkdtemp( 55 | prefix="veles-pdf-publish-"), "report.html"), 56 | image_format="png") 57 | self.file = kwargs["file"] 58 | 59 | @property 60 | def file(self): 61 | return self._file 62 | 63 | @file.setter 64 | def file(self, value): 65 | if value is None: 66 | self._file = None 67 | return 68 | if not isinstance(value, string_types) and not hasattr(value, "write"): 69 | raise TypeError( 70 | "file must either a path or a file-like object (got %s)" % 71 | type(value)) 72 | self._file = value 73 | 74 | def render(self, info): 75 | try: 76 | self.markdown.render(info) 77 | HTML(self.markdown.file).write_pdf(self.file) 78 | self.info("%s is ready", self.file) 79 | finally: 80 | rmtree(os.path.dirname(self.markdown.file)) 81 | -------------------------------------------------------------------------------- /veles/publishing/registry.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2015 12 | 13 | Publishing backends registry. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | from veles.mapped_object_registry import MappedObjectsRegistry 38 | 39 | 40 | class PublishingBackendRegistry(MappedObjectsRegistry): 41 | """Metaclass to record publishing backends. Used by 42 | :class:`veles.publishing.Publisher`. 43 | Classes derived from Unit may contain 'hide' attribute which specifies 44 | whether it should not appear in the list of registered units. Usually 45 | hide = True is applied to base units which must not be used directly, only 46 | subclassed. There is also a 'hide_all' attribute, do disable the 47 | registration of the whole inheritance tree, so that all the children are 48 | automatically hidden. 49 | """ 50 | mapping = "backends" 51 | -------------------------------------------------------------------------------- /veles/result_provider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jun 12, 2015 12 | 13 | Interface and base class to provide flexible workflow run results output. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from zope.interface import Interface 39 | 40 | 41 | class IResultProvider(Interface): 42 | """ 43 | Specifies the contract for supplying the measurable outcome of any model's 44 | execution. It must be implemented by units which measure the reportable 45 | metrics. 46 | """ 47 | 48 | def get_metric_values(): 49 | """ 50 | :return The measurable results of model's execution, e.g., accuracy, 51 | number of errors, RMSE, etc. Technically, they are a dictionary of 52 | metric names (as returned by get_metric_names()) and achieved values. 53 | """ 54 | 55 | def get_metric_names(): 56 | """ 57 | :return The names of metrics returned by get_metric_values(). A set. 58 | """ 59 | -------------------------------------------------------------------------------- /veles/samples/GeneticExample/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Samsung/veles/96218b8d04ab099d65bfff350a24412a66c26178/veles/samples/GeneticExample/__init__.py -------------------------------------------------------------------------------- /veles/samples/GeneticExample/genetics.py: -------------------------------------------------------------------------------- 1 | # -*-coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Sep 21, 2015 12 | 13 | Example of genetic optimization. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | from __future__ import division 37 | 38 | from zope.interface import implementer 39 | 40 | from veles.config import root 41 | from veles.result_provider import IResultProvider 42 | from veles.units import IUnit, Unit 43 | from veles.workflow import Workflow 44 | 45 | 46 | @implementer(IUnit, IResultProvider) 47 | class Optimizer(Unit): 48 | """Computes fitness value. 49 | """ 50 | def __init__(self, workflow, **kwargs): 51 | super(Optimizer, self).__init__(workflow, **kwargs) 52 | self.fitness = 0.0 53 | # Create some attributes here. 54 | 55 | def initialize(self, **kwargs): 56 | # Allocate some resources here. 57 | pass 58 | 59 | def run(self): 60 | # Do the job here. 61 | x = root.test.x 62 | y = root.test.y 63 | value = (x - 0.33) ** 2 * (y - 0.27) ** 2 64 | 65 | # Assign the fitness (the more the better) 66 | self.fitness = -value # looking for a minimum 67 | 68 | def get_metric_names(self): 69 | return {"EvaluationFitness"} 70 | 71 | def get_metric_values(self): 72 | return {"EvaluationFitness": self.fitness} 73 | 74 | 75 | class TestWorkflow(Workflow): 76 | """Workflow for one run of fitness computation. 77 | """ 78 | def __init__(self, workflow, **kwargs): 79 | super(TestWorkflow, self).__init__(workflow, **kwargs) 80 | self.optimizer = Optimizer(self).link_from(self.start_point) 81 | self.end_point.link_from(self.optimizer) 82 | 83 | 84 | def run(load, main): 85 | """Entry point. 86 | """ 87 | load(TestWorkflow) # creates workflow or loads it from snapshot 88 | main() # calls initialize and run on the workflow 89 | -------------------------------------------------------------------------------- /veles/samples/GeneticExample/genetics_config.py: -------------------------------------------------------------------------------- 1 | # -*-coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Sep 21, 2015 12 | 13 | Example of genetic optimization (config file). 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | from __future__ import division 37 | 38 | from veles.config import root 39 | from veles.genetics import Range 40 | 41 | 42 | root.test.update({ 43 | "x": Range(0.0, -1.0, 1.0), 44 | "y": Range(0.0, -1.0, 1.0) 45 | }) 46 | -------------------------------------------------------------------------------- /veles/samples/GeneticExample/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "VELES team", 3 | "configuration": "genetics_config.py", 4 | "long_description": "Simple example of usage Genetic Algorithm. Workflow for one run of fitness computation", 5 | "name": "GeneticExample", 6 | "requires": [ 7 | "veles >= 0.9.2", 8 | "veles.znicz >= 0.8.2" 9 | ], 10 | "short_description": "Simple example of usage Genetic Algorithm", 11 | "workflow": "genetics.py", 12 | "files": ["__init__.py"] 13 | } -------------------------------------------------------------------------------- /veles/samples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Samsung/veles/96218b8d04ab099d65bfff350a24412a66c26178/veles/samples/__init__.py -------------------------------------------------------------------------------- /veles/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Apr 23, 2013 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | -------------------------------------------------------------------------------- /veles/scripts/update_forge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*-coding: utf-8 -*- 3 | """ 4 | .. invisible: 5 | _ _ _____ _ _____ _____ 6 | | | | | ___| | | ___/ ___| 7 | | | | | |__ | | | |__ \ `--. 8 | | | | | __|| | | __| `--. \ 9 | \ \_/ / |___| |___| |___/\__/ / 10 | \___/\____/\_____|____/\____/ 11 | 12 | Created on Oct 20, 2014 13 | 14 | This script update all workflows to VelesForge 15 | 16 | Command to run: FORGE_SERVER="http://velesnet.ml/forge" PYTHONPATH=`pwd` 17 | veles/scripts/update_forge.py 18 | 19 | ███████████████████████████████████████████████████████████████████████████████ 20 | 21 | Licensed to the Apache Software Foundation (ASF) under one 22 | or more contributor license agreements. See the NOTICE file 23 | distributed with this work for additional information 24 | regarding copyright ownership. The ASF licenses this file 25 | to you under the Apache License, Version 2.0 (the 26 | "License"); you may not use this file except in compliance 27 | with the License. You may obtain a copy of the License at 28 | 29 | http://www.apache.org/licenses/LICENSE-2.0 30 | 31 | Unless required by applicable law or agreed to in writing, 32 | software distributed under the License is distributed on an 33 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 34 | KIND, either express or implied. See the License for the 35 | specific language governing permissions and limitations 36 | under the License. 37 | 38 | ███████████████████████████████████████████████████████████████████████████████ 39 | """ 40 | 41 | 42 | import logging 43 | import subprocess 44 | import os 45 | import sys 46 | 47 | from veles.dot_pip import install_dot_pip 48 | install_dot_pip() 49 | from veles.config import root 50 | from veles.scripts.generate_frontend import scan_workflows 51 | from veles.logger import Logger 52 | 53 | 54 | class Main(Logger): 55 | def run(self, server_url): 56 | if server_url is None: 57 | raise ValueError("Server URL must not be None. Looks like you did " 58 | "not set FORGE_SERVER environment variable.") 59 | workflows = scan_workflows(False) 60 | for workflow in workflows: 61 | workflow_folder = os.path.join("veles", os.path.dirname(workflow)) 62 | if root.common.forge.manifest in os.listdir(workflow_folder): 63 | self.info("Update workflow %s in VELESForge" % workflow_folder) 64 | subprocess.call( 65 | ["python3", "-m", "veles", "forge", "upload", "-s", 66 | server_url, "-d", workflow_folder, "-i", 67 | "41a8a51b-b866-4b67-afa3-3ea78805cac0"]) 68 | else: 69 | self.info("Workflow %s is not in Forge" % workflow) 70 | 71 | if __name__ == "__main__": 72 | logging.basicConfig(level=logging.INFO) 73 | sys.exit(Main().run(os.getenv("FORGE_SERVER"))) 74 | -------------------------------------------------------------------------------- /veles/timeit2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Sep 25, 2014 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | from six import PY3 37 | if PY3: 38 | from time import perf_counter 39 | else: 40 | from time import time as perf_counter 41 | 42 | 43 | def timeit(function, *args, **kwargs): 44 | ts = perf_counter() 45 | res = function(*args, **kwargs) 46 | return res, perf_counter() - ts 47 | -------------------------------------------------------------------------------- /veles/txzmq/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jul 3, 2014. 12 | 13 | This file is a part of (almost) compeletely rewritten original txZmq project. 14 | 15 | ZeroMQ integration into Twisted reactor. 16 | 17 | ███████████████████████████████████████████████████████████████████████████████ 18 | 19 | Licensed to the Apache Software Foundation (ASF) under one 20 | or more contributor license agreements. See the NOTICE file 21 | distributed with this work for additional information 22 | regarding copyright ownership. The ASF licenses this file 23 | to you under the Apache License, Version 2.0 (the 24 | "License"); you may not use this file except in compliance 25 | with the License. You may obtain a copy of the License at 26 | 27 | http://www.apache.org/licenses/LICENSE-2.0 28 | 29 | Unless required by applicable law or agreed to in writing, 30 | software distributed under the License is distributed on an 31 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 32 | KIND, either express or implied. See the License for the 33 | specific language governing permissions and limitations 34 | under the License. 35 | 36 | ███████████████████████████████████████████████████████████████████████████████ 37 | """ 38 | 39 | 40 | from veles.txzmq.connection import ZmqConnection, ZmqEndpoint 41 | from veles.txzmq.sharedio import SharedIO 42 | -------------------------------------------------------------------------------- /veles/txzmq/manager.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jul 3, 2014. 12 | 13 | ZeroMQ Twisted factory which is controlling ZeroMQ context. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from zmq import Context 39 | 40 | from twisted.internet import reactor 41 | 42 | 43 | class ZmqContextManager(object): 44 | """ 45 | I control individual ZeroMQ connections. 46 | 47 | Factory creates and destroys ZeroMQ context. 48 | 49 | :var reactor: reference to Twisted reactor used by all the connections 50 | :var ioThreads: number of IO threads ZeroMQ will be using for this context 51 | :vartype ioThreads: int 52 | :var lingerPeriod: number of milliseconds to block when closing socket 53 | (terminating context), when there are some messages pending to be sent 54 | :vartype lingerPeriod: int 55 | 56 | :var connections: set of instantiated :class:`ZmqConnection` 57 | :vartype connections: set 58 | :var context: ZeroMQ context 59 | """ 60 | 61 | reactor = reactor 62 | ioThreads = 1 63 | lingerPeriod = 100 64 | _instance = None 65 | 66 | def __new__(cls): 67 | if not cls._instance: 68 | cls._instance = super(ZmqContextManager, cls).__new__(cls) 69 | cls._instance.initialized = False 70 | return cls._instance 71 | 72 | def __init__(self): 73 | """ 74 | Constructor. 75 | 76 | Create ZeroMQ context. 77 | """ 78 | if not self.initialized: 79 | self.initialized = True 80 | self.connections = set() 81 | self.context = Context(self.ioThreads) 82 | reactor.addSystemEventTrigger('during', 'shutdown', self.shutdown) 83 | 84 | def __repr__(self): 85 | return "ZmqContextManager(%d threads)" % self.ioThreads 86 | 87 | def shutdown(self): 88 | """ 89 | Shutdown factory. 90 | 91 | This is shutting down all created connections 92 | and terminating ZeroMQ context. Also cleans up 93 | Twisted reactor. 94 | """ 95 | if not self.initialized: 96 | return 97 | self.initialized = False 98 | for connection in self.connections.copy(): 99 | connection.shutdown() 100 | 101 | self.connections = None 102 | self.context.term() 103 | self.context = None 104 | -------------------------------------------------------------------------------- /veles/txzmq/sharedio.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jul 3, 2014. 12 | 13 | This file is a part of (almost) compeletely rewritten original txZmq project. 14 | 15 | Buffer exchange via the shared memory. 16 | 17 | ███████████████████████████████████████████████████████████████████████████████ 18 | 19 | Licensed to the Apache Software Foundation (ASF) under one 20 | or more contributor license agreements. See the NOTICE file 21 | distributed with this work for additional information 22 | regarding copyright ownership. The ASF licenses this file 23 | to you under the Apache License, Version 2.0 (the 24 | "License"); you may not use this file except in compliance 25 | with the License. You may obtain a copy of the License at 26 | 27 | http://www.apache.org/licenses/LICENSE-2.0 28 | 29 | Unless required by applicable law or agreed to in writing, 30 | software distributed under the License is distributed on an 31 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 32 | KIND, either express or implied. See the License for the 33 | specific language governing permissions and limitations 34 | under the License. 35 | 36 | ███████████████████████████████████████████████████████████████████████████████ 37 | """ 38 | 39 | 40 | from mmap import mmap 41 | from posix_ipc import SharedMemory, O_CREAT, ExistentialError 42 | 43 | 44 | class SharedIO(object): 45 | """ 46 | A version of BytesIO which is shared between multiple processes, suitable 47 | for IPC. 48 | """ 49 | 50 | CACHE = {} 51 | 52 | def __init__(self, name, size): 53 | self.shmem = None 54 | self.shmem = SharedMemory(name, flags=O_CREAT, mode=0o666, size=size) 55 | self.file = mmap(self.shmem.fd, size) 56 | self.__init_file_methods() 57 | self.__shmem_refs = [1] 58 | 59 | def __del__(self): 60 | if self.shmem is None: 61 | return 62 | self.__shmem_refs[0] -= 1 63 | if self.__shmem_refs[0] == 0: 64 | self.shmem.close_fd() 65 | try: 66 | self.shmem.unlink() 67 | except ExistentialError: 68 | pass 69 | 70 | def __getstate__(self): 71 | return {"name": self.name, 72 | "size": self.size, 73 | "pos": self.tell()} 74 | 75 | def __setstate__(self, state): 76 | name = state["name"] 77 | size = state["size"] 78 | cached = SharedIO.CACHE.get("%s:%d" % (name, size)) 79 | if cached is not None: 80 | assert cached.size == size 81 | self.shmem = cached.shmem 82 | self.file = cached.file 83 | self.__shmem_refs = cached.__shmem_refs 84 | self.__shmem_refs[0] += 1 85 | self.__init_file_methods() 86 | else: 87 | self.__init__(name, size) 88 | SharedIO.CACHE["%s:%d" % (name, size)] = self 89 | self.seek(state["pos"]) 90 | 91 | @property 92 | def name(self): 93 | return self.shmem.name 94 | 95 | @property 96 | def size(self): 97 | return self.shmem.size 98 | 99 | @property 100 | def refs(self): 101 | return self.__shmem_refs[0] 102 | 103 | def __init_file_methods(self): 104 | for name in ("read", "readline", "write", "tell", "close", "seek"): 105 | setattr(self, name, getattr(self.file, name)) 106 | -------------------------------------------------------------------------------- /veles/verified.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jan 30, 2015 12 | 13 | :class:`Verified` definition. 14 | 15 | ███████████████████████████████████████████████████████████████████████████████ 16 | 17 | Licensed to the Apache Software Foundation (ASF) under one 18 | or more contributor license agreements. See the NOTICE file 19 | distributed with this work for additional information 20 | regarding copyright ownership. The ASF licenses this file 21 | to you under the Apache License, Version 2.0 (the 22 | "License"); you may not use this file except in compliance 23 | with the License. You may obtain a copy of the License at 24 | 25 | http://www.apache.org/licenses/LICENSE-2.0 26 | 27 | Unless required by applicable law or agreed to in writing, 28 | software distributed under the License is distributed on an 29 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 30 | KIND, either express or implied. See the License for the 31 | specific language governing permissions and limitations 32 | under the License. 33 | 34 | ███████████████████████████████████████████████████████████████████████████████ 35 | """ 36 | 37 | 38 | from zope.interface.verify import verifyObject, verifyClass 39 | 40 | from veles.compat import from_none 41 | from veles.logger import Logger 42 | import veles.zope_verify_fix # pylint: disable=W0611 43 | 44 | 45 | class Verified(Logger): 46 | """ 47 | Base for all classes which follow any :class:`zope.interface.Interface`. 48 | """ 49 | def verify_interface(self, iface): 50 | if getattr(type(self), "DISABLE_INTERFACE_VERIFICATION", False): 51 | return 52 | if not iface.providedBy(self): 53 | raise NotImplementedError( 54 | "Unit %s does not implement %s interface" 55 | % (repr(self), iface.__name__)) 56 | try: 57 | verifyObject(iface, self) 58 | except Exception as e: 59 | self.error("%s does not pass verifyObject(%s)", self, iface) 60 | raise from_none(e) 61 | try: 62 | verifyClass(iface, self.__class__) 63 | except Exception as e: 64 | self.error("%s does not pass verifyClass(%s)", 65 | self.__class__, iface) 66 | raise from_none(e) 67 | -------------------------------------------------------------------------------- /veles/zope_verify_fix.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | .. invisible: 4 | _ _ _____ _ _____ _____ 5 | | | | | ___| | | ___/ ___| 6 | | | | | |__ | | | |__ \ `--. 7 | | | | | __|| | | __| `--. \ 8 | \ \_/ / |___| |___| |___/\__/ / 9 | \___/\____/\_____|____/\____/ 10 | 11 | Created on Jul 21, 2014 12 | 13 | ███████████████████████████████████████████████████████████████████████████████ 14 | 15 | Licensed to the Apache Software Foundation (ASF) under one 16 | or more contributor license agreements. See the NOTICE file 17 | distributed with this work for additional information 18 | regarding copyright ownership. The ASF licenses this file 19 | to you under the Apache License, Version 2.0 (the 20 | "License"); you may not use this file except in compliance 21 | with the License. You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, 26 | software distributed under the License is distributed on an 27 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 28 | KIND, either express or implied. See the License for the 29 | specific language governing permissions and limitations 30 | under the License. 31 | 32 | ███████████████████████████████████████████████████████████████████████████████ 33 | """ 34 | 35 | 36 | import zope.interface.verify as verify 37 | 38 | 39 | def _incompat(required, implemented): 40 | if (len(implemented['required']) > len(required['required']) and 41 | not required['kwargs']): 42 | return 'implementation requires too many arguments' 43 | if ((len(implemented['positional']) < len(required['positional'])) 44 | and not implemented['varargs']): 45 | return "implementation doesn't allow enough arguments" 46 | if required['kwargs'] and not implemented['kwargs']: 47 | return "implementation doesn't support keyword arguments" 48 | if required['varargs'] and not implemented['varargs']: 49 | return "implementation doesn't support variable arguments" 50 | 51 | verify._incompat = _incompat 52 | --------------------------------------------------------------------------------