├── .gitignore ├── LICENSE ├── README.md ├── addfor_tutorials.yml ├── machine_learning ├── example_data │ ├── Training Dataset.arff │ ├── bumpy_road.csv │ ├── data.csv │ ├── data2.csv │ ├── high_freq.csv │ ├── low_freq.csv │ ├── matlab_test_data_01.mat │ └── matlab_test_data_02.mat ├── images │ ├── AI-ML.png │ ├── Autoencoder_structure.png │ ├── Euclidean_distance.png │ ├── LSTM.001.jpeg │ ├── LSTM3-chain.png │ ├── Manhattan_distance.png │ ├── RNN_true_truncated_backprop.png │ ├── Simple Neural Network.png │ ├── WeightSharing.png │ ├── activation_functions.png │ ├── beta_cost.jpg │ ├── bias_variance.png │ ├── bias_variance_proof.jpg │ ├── camera.png │ ├── cat.jpg │ ├── classification.jpg │ ├── classifier.png │ ├── clustering.jpg │ ├── cnn.jpeg │ ├── computation_graph_example.jpg │ ├── conv_padding.jpg │ ├── conv_stride.jpg │ ├── curse_dimensionality.png │ ├── dataset.jpg │ ├── decision_boundary.jpg │ ├── diags.jpeg │ ├── dichotomies.png │ ├── dichotomies_fat.png │ ├── dropout.jpeg │ ├── early_stopping.png │ ├── enc_base.png │ ├── enc_deep.png │ ├── ensemble_explore_boston.png │ ├── ensemble_explore_hastie.png │ ├── estimating_coefficients.png │ ├── feature_map.png │ ├── feature_map_mod.jpg │ ├── gradient_descent.png │ ├── gradients.png │ ├── k-fold.png │ ├── kernel_svm.png │ ├── lagged.jpeg │ ├── learning_curves.png │ ├── logistic_cost.png │ ├── logistic_function.png │ ├── logistic_regression.jpg │ ├── logistic_regression2.jpg │ ├── maximize_margin.png │ ├── maxpool.jpeg │ ├── ml_process.jpg │ ├── mnist1.png │ ├── mnist2.png │ ├── nesterov.png │ ├── neural_net.jpeg │ ├── neuron.jpeg │ ├── non_convex.png │ ├── non_linear_svm.png │ ├── non_separable_svm.png │ ├── non_separable_svm1.png │ ├── non_separable_svm2.png │ ├── one_vs_all.png │ ├── quadratic_bowl.png │ ├── quadratic_svm.png │ ├── receptive.png │ ├── regression.jpg │ ├── simpleRNN.jpeg │ ├── slope_intercept.png │ ├── softmargin_svm.png │ ├── temp.png │ ├── temp1.png │ ├── temp2.png │ ├── temp3.png │ ├── tf_mnist.png │ ├── tflstm.png │ ├── train_cycle.png │ ├── trainset_example.png │ ├── trainset_multiple_example.png │ ├── tree.png │ ├── unrolledRNN.jpeg │ ├── valley.png │ ├── valley_with_ball.png │ ├── without_momentum.png │ ├── without_momentum_3d.gif │ └── without_momentum_3d.png ├── index.ipynb ├── ml00v04_definitions.ipynb ├── ml01v04_prepare_the_data.ipynb ├── ml02v04_the_scikit-learn_interface.ipynb ├── ml03v04_visualizing_the_data.ipynb ├── ml04v04_dealing_with_bias_and_variance.ipynb ├── ml05v04_ensemble_methods.ipynb ├── ml06v04_ensemble_methods_advanced.ipynb ├── ml07v04_clustering.ipynb ├── ml08v04_linear_models.ipynb ├── ml09v04_gaussian_process.ipynb ├── ml11v04_linear_models_regularization.ipynb ├── ml13v04_support_vector_machines.ipynb ├── ml16v04_forecasting_with_LSTM.ipynb ├── ml17v04_prognostics_using_autoencoder.ipynb ├── ml25v04_tensorflow_basic_concepts.ipynb ├── ml26v04_neural_networks_tensorflow.ipynb ├── ml27v04_convolutional_neural_networks_tensorflow.ipynb ├── ml28v04_recurrent_neural_networks_with_tensorflow.ipynb ├── ml29v04_keras_introduction.ipynb ├── temp │ └── temporary.txt └── utilities │ ├── .theanorc │ ├── __init__.py │ ├── biasvariance.py │ ├── cache.py │ ├── cifar10.py │ ├── dataset.py │ ├── download.py │ ├── logo.png │ └── utils.py ├── numpy ├── images │ ├── IIR_filters_table.jpg │ ├── arrayobject.jpg │ ├── carrayobject.jpg │ ├── convex_function.png │ ├── earrayobject.jpg │ ├── filter_design.jpg │ ├── non_convex_function.png │ ├── non_smooth_function.png │ ├── numpy_array.jpg │ ├── smooth_function.png │ └── tableobject.jpg ├── index.ipynb ├── np01v04_numpy_basics.ipynb ├── np02v04_numpy_PyTables.ipynb ├── np03v04_numpy_plotting.ipynb ├── np04v04_scipy_optimization.ipynb ├── np05v04_scipy_sig_processing_IIRfilter_design.ipynb ├── np06v04_Symbolic_Computation.ipynb ├── temp │ └── temporary.txt └── utilities │ ├── Thumbs.db │ ├── __init__.py │ ├── __init__.pyc │ ├── logo.png │ ├── plot_utilities.py │ ├── plot_utilities.pyc │ └── vlarrayobject.jpg ├── pandas ├── example_data │ ├── CrimeStatebyState.csv.bz2 │ ├── babynames.zip │ ├── cities.csv.bz2 │ ├── company.csv │ ├── generated.html │ ├── generated2.xls │ ├── matlab_variable.mat │ ├── p01_d3.csv.gz │ ├── p01_prices.txt │ ├── p01_volumes.txt │ ├── p03_AAPL.csv │ ├── p03_AAPL.txt │ ├── p03_DAX.csv │ ├── p05_d2.txt │ ├── pd06_pivot_Example.txt │ ├── pd12_peopl.csv.gz │ └── tabella_comuni_italiani.txt ├── index.ipynb ├── pd01v04_basic_data_operativity.ipynb ├── pd02v04_input_output.ipynb ├── pd03v04_time_series.ipynb ├── pd04v04_statistical_tools.ipynb ├── pd05v04_data_organization.ipynb ├── pd06v04_advanced_data_management.ipynb ├── pd07v04_open_data.ipynb ├── pd08v04_babynames.ipynb ├── temp │ └── temporary.txt └── utilities │ ├── Thumbs.db │ ├── __init__.py │ ├── generators.py │ ├── logo.png │ ├── my_finance_plots.py │ ├── my_finance_tools.py │ └── tom.py └── python-ipython ├── example_data ├── my_input.txt ├── p03_AAPL.txt └── small_is_beautiful.txt ├── images └── integration.png ├── index.ipynb ├── py01v04_ipython_notebook_introduction.ipynb ├── py02v04_python_basics.ipynb ├── py03v04_python_getting_started.ipynb ├── py04v04_python_style_guide.ipynb ├── py05v04_python_more_examples.ipynb ├── py06v04_python_object_oriented.ipynb ├── py07v04_python_speed-up_with_C.ipynb ├── py08v04_Unicode.ipynb ├── py09v04_python_regular_expressions.ipynb ├── py10v04_ipython_notebook_widgets.ipynb ├── tmp ├── my_input2.txt ├── my_unicode_example.txt ├── my_unknow_encoder.txt ├── my_utf8_example.txt ├── out_ascii.pkl ├── out_compb.pkl └── temporary.txt └── utilities ├── __init__.py ├── integrate.py ├── integrate.pyd ├── integrate_compiled.so ├── integrate_cy.pyd ├── integrate_cy.pyx ├── integrate_cy.so ├── integrate_hints.py ├── integrate_hints.pyd ├── integrate_hints.so ├── logo.png ├── my_finance_tools.py ├── my_module.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | *.pyc 3 | */temp/* 4 | */tmp/* 5 | !*/temp/temporary.txt 6 | */example_data/* 7 | */MNIST_data/* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Training Material | Addfor S.p.A. 2 | 3 | The following IPython Notebooks are the standard training material distributed with the Addfor trainings. For more information about standard and custom training solutions please visit [Training @ Addfor](https://www.add-for.com/training/). 4 | 5 | All the IPython notebooks are distributed under the Creative Commons Attribution-ShareAlike 4.0 International License. 6 | 7 | ## Installation instructions 8 | 9 | We recommend to install the Anaconda distribution to the latest version: please visit [continuum.io](https://www.continuum.io/downloads) to download Anaconda. The tutorials work with python3 (python2 is no longer supported). After Anaconda installation update the distribution to the latest release: `conda update anaconda`. 10 | 11 | Clone this repository with git; use this command: `git clone --depth 1 https://github.com/addfor/tutorials` if you want to download only the current commit (faster, takes less disk space): 12 | 13 | > Create a shallow clone with a history truncated to the specified number of commits. 14 | 15 | **NOTE:** for Windows users, you can use this [git client](http://gitforwindows.org/), or choose to download: click *Clone or download* and then *Download ZIP* (in this case skip the git clone step). 16 | 17 | Next cd into tutorials and create the environment addfor_tutorials from the file `addfor_tutorials.yml` (make sure the file is in your directory). Issue the command `conda env create -f addfor_tutorials.yml` (the process could take few minutes). After the installation is finished, activate the environment: 18 | 19 | > Windows: `activate myenv` 20 | > macOS and Linux: `source activate myenv` 21 | 22 | All notebooks use our Addutils library: please install [Addutils](https://www.dropbox.com/s/g2vibmklfn2smz3/AddUtils-0.5.4-py34.zip) (for python3) before running the Notebooks. Download the zip file and open the Terminal or Anaconda Prompt: `source activate addfor_tutorials` if environment is not already active, then type `pip install AddUtils-0.5.4-py34.zip` (it should work for python3.4+). 23 | 24 | At this point you are able to run the notebook with: `jupyter-notebook` and navigate through the directory tree. 25 | 26 | **Note**: the first time you run the notebooks you could experience a brief slowdown due to matplotlib building its font cache. It should disappear the next session. 27 | 28 | For more informations visit: [Download training material guidelines @ Addfor](https://www.add-for.com/downloads-tutorials/) 29 | 30 | ## Index 31 | 32 | 1. **Python + IPython/Jupyter** 33 | 1. [An introduction to the IPython notebook](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py01v04_ipython_notebook_introduction.ipynb) 34 | 1. [Python Basic Concepts](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py02v04_python_basics.ipynb) 35 | 1. [Python Getting Started](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py03v04_python_getting_started.ipynb) 36 | 1. [Python Style Guide](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py04v04_python_style_guide.ipynb) 37 | 1. [Python More Examples](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py05v04_python_more_examples.ipynb) 38 | 1. [Object Oriented Programming in Python](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py06v04_python_object_oriented.ipynb) 39 | 1. [Integration of Python with compiled languages](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py07v04_python_speed-up_with_C.ipynb) 40 | 1. [Unicode](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py08v04_Unicode.ipynb) 41 | 1. [Regular Expressions](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/python-ipython/py09v04_python_regular_expressions.ipynb) 42 | 1. **NumPy** 43 | 1. [Numpy Basic Concepts](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np01v04_numpy_basics.ipynb) 44 | 1. [PyTables](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np02v04_numpy_PyTables.ipynb) 45 | 1. [Numpy - Plotting with Matplotlib](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np03v04_numpy_plotting.ipynb) 46 | 1. [Scipy - Optimization](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np04v04_scipy_optimization.ipynb) 47 | 1. [Scipy Signal Processing: IIR Filter Design](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np05v04_scipy_sig_processing_IIRfilter_design.ipynb) 48 | 1. [Symbolic Computation](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/numpy/np06v04_Symbolic_Computation.ipynb) 49 | 1. **Pandas** 50 | 1. [pandas Dataframe - Basic Operativity](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd01v04_basic_data_operativity.ipynb) 51 | 1. [pandas I/O tools and examples](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd02v04_input_output.ipynb) 52 | 1. [Pandas Time series](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd03v04_time_series.ipynb) 53 | 1. [Statistical tools](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd04v04_statistical_tools.ipynb) 54 | 1. [Merge and pivot](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd05v04_data_organization.ipynb) 55 | 1. [Split apply and combine](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd06v04_advanced_data_management.ipynb) 56 | 1. [Sources of Open Data](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd07v04_open_data.ipynb) 57 | 1. [Baby Names](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/pandas/pd08v04_babynames.ipynb) 58 | 1. **Machine learning** 59 | 1. [Definitions and Advices](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml00v04_definitions.ipynb) 60 | 1. [Prepare the Data](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml01v04_prepare_the_data.ipynb) 61 | 1. [The scikit-learn interface](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml02v04_the_scikit-learn_interface.ipynb) 62 | 1. [Visualizing the Data](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml03v04_visualizing_the_data.ipynb) 63 | 1. [Dealing with Bias and Variance](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml04v04_dealing_with_bias_and_variance.ipynb) 64 | 1. [Ensemble Methods](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml05v04_ensemble_methods.ipynb) 65 | 1. [Ensemble Methods Advanced](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml06v04_ensemble_methods_advanced.ipynb) 66 | 1. [Support vector machines (SVMs)](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml13v04_support_vector_machines.ipynb) 67 | 1. [Predict Temporal Series](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml15v04_predict_temporal_series.ipynb) 68 | 1. [Forecasting with LSTM](http://nbviewer.jupyter.org/github/addfor/tutorials/blob/master/machine_learning/ml16v04_forecasting_with_LSTM.ipynb) 69 | 1. [Prognostics using Autoencoder](http://nbviewer.jupyter.org/github/addfor/tutorials/blob/master/machine_learning/ml17v04_prognostics_using_autoencoder.ipynb) 70 | 1. [Theano Basic Concepts](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml20v04_theano_basics.ipynb) 71 | 1. [Explore Neural Network Hyperparameters with Theano and Keras](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml21v04_theano_NN_explore_hyperparameters.ipynb) 72 | 1. [Neural Networks with Nervana Neon library](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml23v04_neon_NN_basics_and_hyperparameters-py27.ipynb) 73 | 1. [Tensorflow Basic concepts](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml25v04_tensorflow_basics.ipynb) 74 | 1. [Explore Neural Network Hyperparameters with TensorFlow](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml26v04_tensorflow_NN_explore_hyperparameters.ipynb) 75 | 1. [TensorFlow for beginners](http://nbviewer.jupyter.org/github/addfor/tutorials/blob/master/machine_learning/ml27v04_tensorflow_for_beginners.ipynb) 76 | 1. [Keras - Theano Benchmark](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml30v04_Keras_NN_test.ipynb) 77 | 1. [Neon Benchmark](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml31v04_Neon_NN_test-py27.ipynb) 78 | 1. [TensorFlow Benchmark](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml32v04_TensorFlow_NN_test.ipynb) 79 | 1. [Neural Network Benchmark Summary](http://nbviewer.ipython.org/github/addfor/tutorials/blob/master/machine_learning/ml33v04_NN_benchmark.ipynb) 80 | -------------------------------------------------------------------------------- /addfor_tutorials.yml: -------------------------------------------------------------------------------- 1 | name: addfor_tutorials 2 | channels: 3 | - defaults 4 | dependencies: 5 | - python=3.6.4 6 | - numpy=1.14.0 7 | - pandas=0.22.0 8 | - scikit-learn=0.19.1 9 | - bokeh=0.12.13 10 | - matplotlib=2.1.2 11 | - nb_conda=2.2.0 12 | - seaborn=0.8.1 13 | - faker=0.8.6 14 | - pandas-datareader=0.5.0 15 | - pytables=3.4.2 16 | - lxml=4.1.1 17 | - xlrd=1.1.0 18 | - pydot=1.2.3 19 | - sympy=1.1.1 20 | - cython=0.27.3 21 | - xlsxwriter=1.0.2 22 | - h5py=2.7.1 23 | - pip: 24 | - liac-arff==2.1.1 25 | - pillow==5.0.0 26 | -------------------------------------------------------------------------------- /machine_learning/example_data/matlab_test_data_01.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/example_data/matlab_test_data_01.mat -------------------------------------------------------------------------------- /machine_learning/example_data/matlab_test_data_02.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/example_data/matlab_test_data_02.mat -------------------------------------------------------------------------------- /machine_learning/images/AI-ML.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/AI-ML.png -------------------------------------------------------------------------------- /machine_learning/images/Autoencoder_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/Autoencoder_structure.png -------------------------------------------------------------------------------- /machine_learning/images/Euclidean_distance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/Euclidean_distance.png -------------------------------------------------------------------------------- /machine_learning/images/LSTM.001.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/LSTM.001.jpeg -------------------------------------------------------------------------------- /machine_learning/images/LSTM3-chain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/LSTM3-chain.png -------------------------------------------------------------------------------- /machine_learning/images/Manhattan_distance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/Manhattan_distance.png -------------------------------------------------------------------------------- /machine_learning/images/RNN_true_truncated_backprop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/RNN_true_truncated_backprop.png -------------------------------------------------------------------------------- /machine_learning/images/Simple Neural Network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/Simple Neural Network.png -------------------------------------------------------------------------------- /machine_learning/images/WeightSharing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/WeightSharing.png -------------------------------------------------------------------------------- /machine_learning/images/activation_functions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/activation_functions.png -------------------------------------------------------------------------------- /machine_learning/images/beta_cost.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/beta_cost.jpg -------------------------------------------------------------------------------- /machine_learning/images/bias_variance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/bias_variance.png -------------------------------------------------------------------------------- /machine_learning/images/bias_variance_proof.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/bias_variance_proof.jpg -------------------------------------------------------------------------------- /machine_learning/images/camera.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/camera.png -------------------------------------------------------------------------------- /machine_learning/images/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/cat.jpg -------------------------------------------------------------------------------- /machine_learning/images/classification.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/classification.jpg -------------------------------------------------------------------------------- /machine_learning/images/classifier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/classifier.png -------------------------------------------------------------------------------- /machine_learning/images/clustering.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/clustering.jpg -------------------------------------------------------------------------------- /machine_learning/images/cnn.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/cnn.jpeg -------------------------------------------------------------------------------- /machine_learning/images/computation_graph_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/computation_graph_example.jpg -------------------------------------------------------------------------------- /machine_learning/images/conv_padding.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/conv_padding.jpg -------------------------------------------------------------------------------- /machine_learning/images/conv_stride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/conv_stride.jpg -------------------------------------------------------------------------------- /machine_learning/images/curse_dimensionality.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/curse_dimensionality.png -------------------------------------------------------------------------------- /machine_learning/images/dataset.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/dataset.jpg -------------------------------------------------------------------------------- /machine_learning/images/decision_boundary.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/decision_boundary.jpg -------------------------------------------------------------------------------- /machine_learning/images/diags.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/diags.jpeg -------------------------------------------------------------------------------- /machine_learning/images/dichotomies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/dichotomies.png -------------------------------------------------------------------------------- /machine_learning/images/dichotomies_fat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/dichotomies_fat.png -------------------------------------------------------------------------------- /machine_learning/images/dropout.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/dropout.jpeg -------------------------------------------------------------------------------- /machine_learning/images/early_stopping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/early_stopping.png -------------------------------------------------------------------------------- /machine_learning/images/enc_base.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/enc_base.png -------------------------------------------------------------------------------- /machine_learning/images/enc_deep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/enc_deep.png -------------------------------------------------------------------------------- /machine_learning/images/ensemble_explore_boston.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/ensemble_explore_boston.png -------------------------------------------------------------------------------- /machine_learning/images/ensemble_explore_hastie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/ensemble_explore_hastie.png -------------------------------------------------------------------------------- /machine_learning/images/estimating_coefficients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/estimating_coefficients.png -------------------------------------------------------------------------------- /machine_learning/images/feature_map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/feature_map.png -------------------------------------------------------------------------------- /machine_learning/images/feature_map_mod.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/feature_map_mod.jpg -------------------------------------------------------------------------------- /machine_learning/images/gradient_descent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/gradient_descent.png -------------------------------------------------------------------------------- /machine_learning/images/gradients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/gradients.png -------------------------------------------------------------------------------- /machine_learning/images/k-fold.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/k-fold.png -------------------------------------------------------------------------------- /machine_learning/images/kernel_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/kernel_svm.png -------------------------------------------------------------------------------- /machine_learning/images/lagged.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/lagged.jpeg -------------------------------------------------------------------------------- /machine_learning/images/learning_curves.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/learning_curves.png -------------------------------------------------------------------------------- /machine_learning/images/logistic_cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/logistic_cost.png -------------------------------------------------------------------------------- /machine_learning/images/logistic_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/logistic_function.png -------------------------------------------------------------------------------- /machine_learning/images/logistic_regression.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/logistic_regression.jpg -------------------------------------------------------------------------------- /machine_learning/images/logistic_regression2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/logistic_regression2.jpg -------------------------------------------------------------------------------- /machine_learning/images/maximize_margin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/maximize_margin.png -------------------------------------------------------------------------------- /machine_learning/images/maxpool.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/maxpool.jpeg -------------------------------------------------------------------------------- /machine_learning/images/ml_process.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/ml_process.jpg -------------------------------------------------------------------------------- /machine_learning/images/mnist1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/mnist1.png -------------------------------------------------------------------------------- /machine_learning/images/mnist2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/mnist2.png -------------------------------------------------------------------------------- /machine_learning/images/nesterov.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/nesterov.png -------------------------------------------------------------------------------- /machine_learning/images/neural_net.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/neural_net.jpeg -------------------------------------------------------------------------------- /machine_learning/images/neuron.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/neuron.jpeg -------------------------------------------------------------------------------- /machine_learning/images/non_convex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/non_convex.png -------------------------------------------------------------------------------- /machine_learning/images/non_linear_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/non_linear_svm.png -------------------------------------------------------------------------------- /machine_learning/images/non_separable_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/non_separable_svm.png -------------------------------------------------------------------------------- /machine_learning/images/non_separable_svm1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/non_separable_svm1.png -------------------------------------------------------------------------------- /machine_learning/images/non_separable_svm2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/non_separable_svm2.png -------------------------------------------------------------------------------- /machine_learning/images/one_vs_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/one_vs_all.png -------------------------------------------------------------------------------- /machine_learning/images/quadratic_bowl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/quadratic_bowl.png -------------------------------------------------------------------------------- /machine_learning/images/quadratic_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/quadratic_svm.png -------------------------------------------------------------------------------- /machine_learning/images/receptive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/receptive.png -------------------------------------------------------------------------------- /machine_learning/images/regression.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/regression.jpg -------------------------------------------------------------------------------- /machine_learning/images/simpleRNN.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/simpleRNN.jpeg -------------------------------------------------------------------------------- /machine_learning/images/slope_intercept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/slope_intercept.png -------------------------------------------------------------------------------- /machine_learning/images/softmargin_svm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/softmargin_svm.png -------------------------------------------------------------------------------- /machine_learning/images/temp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/temp.png -------------------------------------------------------------------------------- /machine_learning/images/temp1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/temp1.png -------------------------------------------------------------------------------- /machine_learning/images/temp2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/temp2.png -------------------------------------------------------------------------------- /machine_learning/images/temp3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/temp3.png -------------------------------------------------------------------------------- /machine_learning/images/tf_mnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/tf_mnist.png -------------------------------------------------------------------------------- /machine_learning/images/tflstm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/tflstm.png -------------------------------------------------------------------------------- /machine_learning/images/train_cycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/train_cycle.png -------------------------------------------------------------------------------- /machine_learning/images/trainset_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/trainset_example.png -------------------------------------------------------------------------------- /machine_learning/images/trainset_multiple_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/trainset_multiple_example.png -------------------------------------------------------------------------------- /machine_learning/images/tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/tree.png -------------------------------------------------------------------------------- /machine_learning/images/unrolledRNN.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/unrolledRNN.jpeg -------------------------------------------------------------------------------- /machine_learning/images/valley.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/valley.png -------------------------------------------------------------------------------- /machine_learning/images/valley_with_ball.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/valley_with_ball.png -------------------------------------------------------------------------------- /machine_learning/images/without_momentum.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/without_momentum.png -------------------------------------------------------------------------------- /machine_learning/images/without_momentum_3d.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/without_momentum_3d.gif -------------------------------------------------------------------------------- /machine_learning/images/without_momentum_3d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/images/without_momentum_3d.png -------------------------------------------------------------------------------- /machine_learning/index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Table of contents" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "* [__Definitions and Advices__](ml00v04_definitions.ipynb)\n", 15 | " * [1 What is Machine Learning ?](ml00v04_definitions.ipynb#1-What-is-Machine-Learning-?)\n", 16 | " * [1.1 Documentation and reference:](ml00v04_definitions.ipynb#1.1-Documentation-and-reference:)\n", 17 | " * [2 Supervised and Unsupervised Learning](ml00v04_definitions.ipynb#2-Supervised-and-Unsupervised-Learning)\n", 18 | " * [3 Cheat Sheet - scikit-learn Algorithm Selection](ml00v04_definitions.ipynb#3-Cheat-Sheet---scikit-learn-Algorithm-Selection)\n", 19 | " * [4 Machine Learning Wisdom](ml00v04_definitions.ipynb#4-Machine-Learning-Wisdom)\n", 20 | "* [__Prepare the Data__](ml01v02_prepare_the_data.ipynb)\n", 21 | " * [1 Datasets available in scikit-learn](ml01v02_prepare_the_data.ipynb#1-Datasets-available-in-scikit-learn)\n", 22 | " * [1.1 Example: the \"Iris\" Packaged Dataset](ml01v02_prepare_the_data.ipynb#1.1-Example:-the-\"Iris\"-Packaged-Dataset)\n", 23 | " * [1.2 Example: the \"Digits\" Packaged Dataset](ml01v02_prepare_the_data.ipynb#1.2-Example:-the-\"Digits\"-Packaged-Dataset)\n", 24 | " * [1.3 Example: the \"S-Curve\" Generated Dataset](ml01v02_prepare_the_data.ipynb#1.3-Example:-the-\"S-Curve\"-Generated-Dataset)\n", 25 | " * [2 Using pandas](ml01v02_prepare_the_data.ipynb#2-Using-pandas)\n", 26 | " * [3 Working with MATLAB files](ml01v02_prepare_the_data.ipynb#3-Working-with-MATLAB-files)\n", 27 | " * [4 Preprocessing Data](ml01v02_prepare_the_data.ipynb#4-Preprocessing-Data)\n", 28 | " * [4.1 Standardizing = Mean Removal + Variance Scaling:](ml01v02_prepare_the_data.ipynb#4.1-Standardizing-=-Mean-Removal-+-Variance-Scaling:)\n", 29 | " * [4.2 Using the preprocessing.StandardScaler with pandas:](ml01v02_prepare_the_data.ipynb#4.2-Using-the-preprocessing.StandardScaler-with-pandas:)\n", 30 | " * [4.3 Normalizing = Dividing by a Norm of the Vector:](ml01v02_prepare_the_data.ipynb#4.3-Normalizing-=-Dividing-by-a-Norm-of-the-Vector:)\n", 31 | " * [5 Features Extraction](ml01v02_prepare_the_data.ipynb#5-Features-Extraction)\n", 32 | " * [5.1 Derived Features:](ml01v02_prepare_the_data.ipynb#5.1-Derived-Features:)\n", 33 | " * [5.2 DictVectorizer uses \"one-hot\" encoder for categorical features:](ml01v02_prepare_the_data.ipynb#5.2-DictVectorizer-uses-\"one-hot\"-encoder-for-categorical-features:)\n", 34 | " * [5.3 The Bag of Words representation:](ml01v02_prepare_the_data.ipynb#5.3-The-Bag-of-Words-representation:)\n", 35 | "* [__The scikit-learn interface__](ml02v02_the_scikit-learn_interface.ipynb)\n", 36 | " * [1 Simple estimator example: fit a linear regression model](ml02v02_the_scikit-learn_interface.ipynb#1-Simple-estimator-example:-fit-a-linear-regression-model)\n", 37 | " * [2 Separate Training and Validation Sets](ml02v02_the_scikit-learn_interface.ipynb#2-Separate-Training-and-Validation-Sets)\n", 38 | " * [2.1 Example: Do a Regression Analysis on MATLAB® data](ml02v02_the_scikit-learn_interface.ipynb#2.1-Example:-Do-a-Regression-Analysis-on-MATLAB®-data)\n", 39 | " * [2.2 Example: Training and a Validation Sets on a Classification Problem](ml02v02_the_scikit-learn_interface.ipynb#2.2-Example:-Training-and-a-Validation-Sets-on-a-Classification-Problem)\n", 40 | " * [3 Cross Validation (CV)](ml02v02_the_scikit-learn_interface.ipynb#3-Cross-Validation-(CV))\n", 41 | " * [3.1 Cross Validation: test many estimators on the same dataset:](ml02v02_the_scikit-learn_interface.ipynb#3.1-Cross-Validation:-test-many-estimators-on-the-same-dataset:)\n", 42 | " * [3.2 Cross Validation: test many hyperparamaters and estimators on the same dataset:](ml02v02_the_scikit-learn_interface.ipynb#3.2-Cross-Validation:-test-many-hyperparamaters-and-estimators-on-the-same-dataset:)\n", 43 | " * [3.3 Model specific Cross Validation:](ml02v02_the_scikit-learn_interface.ipynb#3.3-Model-specific-Cross-Validation:)\n", 44 | " * [3.4 Cross-validation iterators](ml02v02_the_scikit-learn_interface.ipynb#3.4-Cross-validation-iterators)\n", 45 | " * [4 Grid Search: Searching for estimator hyperparameters](ml02v02_the_scikit-learn_interface.ipynb#4-Grid-Search:-Searching-for-estimator-hyperparameters)\n", 46 | " * [4.1 Exhaustive Grid Search](ml02v02_the_scikit-learn_interface.ipynb#4.1-Exhaustive-Grid-Search)\n", 47 | " * [4.2 Randomized Parameter Optimization](ml02v02_the_scikit-learn_interface.ipynb#4.2-Randomized-Parameter-Optimization)\n", 48 | "* [__Visualizing the Data__](ml03v02_visualizing_the_data.ipynb)\n", 49 | " * [1 Principal Component Analysis (PCA)](ml03v02_visualizing_the_data.ipynb#1-Principal-Component-Analysis-(PCA))\n", 50 | " * [2 Linear Discriminant Alanysis (LDA)](ml03v02_visualizing_the_data.ipynb#2-Linear-Discriminant-Alanysis-(LDA))\n", 51 | " * [3 Manifold Learning](ml03v02_visualizing_the_data.ipynb#3-Manifold-Learning)\n", 52 | " * [3.1 Another example un a specific test dataset: the S-Curve](ml03v02_visualizing_the_data.ipynb#3.1-Another-example-un-a-specific-test-dataset:-the-S-Curve)\n", 53 | " * [3.2 Some Tips on Manifold Learning practical use](ml03v02_visualizing_the_data.ipynb#3.2-Some-Tips-on-Manifold-Learning-practical-use)\n", 54 | " * [4 Totally Random Trees Embedding](ml03v02_visualizing_the_data.ipynb#4-Totally-Random-Trees-Embedding)\n" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "---\n", 62 | "\n", 63 | "Visit [www.add-for.com]() for more tutorials and updates.\n", 64 | "\n", 65 | "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License." 66 | ] 67 | } 68 | ], 69 | "metadata": { 70 | "anaconda-cloud": {}, 71 | "kernelspec": { 72 | "display_name": "Python [default]", 73 | "language": "python", 74 | "name": "python3" 75 | }, 76 | "language_info": { 77 | "codemirror_mode": { 78 | "name": "ipython", 79 | "version": 3 80 | }, 81 | "file_extension": ".py", 82 | "mimetype": "text/x-python", 83 | "name": "python", 84 | "nbconvert_exporter": "python", 85 | "pygments_lexer": "ipython3", 86 | "version": "3.5.2" 87 | } 88 | }, 89 | "nbformat": 4, 90 | "nbformat_minor": 0 91 | } 92 | -------------------------------------------------------------------------------- /machine_learning/temp/temporary.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/temp/temporary.txt -------------------------------------------------------------------------------- /machine_learning/utilities/.theanorc: -------------------------------------------------------------------------------- 1 | [global] 2 | device=gpu 3 | mode=FAST_RUN 4 | floatX=float32 5 | allow_gc=False 6 | optimizer_including=local_remove_all_assert 7 | [gcc] 8 | cxxflags=-O3 -funroll-loops -ftracer 9 | [nvcc] 10 | fastmath = True 11 | [cuda] 12 | root=/usr/local/cuda 13 | -------------------------------------------------------------------------------- /machine_learning/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | from . import biasvariance 2 | from . import utils 3 | -------------------------------------------------------------------------------- /machine_learning/utilities/biasvariance.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import bokeh.plotting as bk 3 | 4 | def test_func(x, err=0.5): 5 | return np.random.normal(10 - 1. / (x + 0.1), err) 6 | 7 | def compute_error(x, y, p): 8 | yfit = np.polyval(p, x) 9 | return np.sqrt(np.mean((y - yfit) ** 2)) 10 | 11 | def plot_bias_variance(N=8, random_seed=42, err=0.5): 12 | np.random.seed(random_seed) 13 | x = 10 ** np.linspace(-2, 0, N) 14 | y = test_func(x) 15 | xfit = np.linspace(-0.2, 1.2, 1000) 16 | titles = ['d = 1 (under-fit; high bias)', 17 | 'd = 2', 18 | 'd = 6 (over-fit; high variance)'] 19 | degrees = [1, 2, 6] 20 | 21 | row = [] 22 | for i, d in enumerate(degrees): 23 | fig = bk.figure(plot_width=240, plot_height=240, 24 | title=titles[i], x_range=(-0.2, 1.2), y_range=(0, 12)) 25 | fig.title.text_font_size = '11pt' 26 | fig.xaxis.axis_label_text_font_size = '9pt' 27 | fig.yaxis.axis_label_text_font_size = '9pt' 28 | fig.x(x, y, color='black', size=12) 29 | 30 | p = np.polyfit(x, y, d) 31 | yfit = np.polyval(p, xfit) 32 | fig.line(xfit, yfit, line_color='blue') 33 | 34 | fig.xaxis.axis_label = 'house size' 35 | fig.xaxis.axis_label_text_font_size = '9pt' 36 | if i == 0: 37 | fig.yaxis.axis_label = 'price' 38 | row.append(fig) 39 | 40 | gp = bk.gridplot([row], border_space=0) 41 | bk.show(gp) 42 | 43 | -------------------------------------------------------------------------------- /machine_learning/utilities/cache.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Cache-wrapper for a function or class. 4 | # 5 | # Save the result of calling a function or creating an object-instance 6 | # to harddisk. This is used to persist the data so it can be reloaded 7 | # very quickly and easily. 8 | # 9 | # Implemented in Python 3.5 10 | # 11 | ######################################################################## 12 | # 13 | # This file is part of the TensorFlow Tutorials available at: 14 | # 15 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 16 | # 17 | # Published under the MIT License. See the file LICENSE for details. 18 | # 19 | # Copyright 2016 by Magnus Erik Hvass Pedersen 20 | # 21 | ######################################################################## 22 | 23 | import os 24 | import pickle 25 | import numpy as np 26 | 27 | ######################################################################## 28 | 29 | 30 | def cache(cache_path, fn, *args, **kwargs): 31 | """ 32 | Cache-wrapper for a function or class. If the cache-file exists 33 | then the data is reloaded and returned, otherwise the function 34 | is called and the result is saved to cache. The fn-argument can 35 | also be a class instead, in which case an object-instance is 36 | created and saved to the cache-file. 37 | 38 | :param cache_path: 39 | File-path for the cache-file. 40 | 41 | :param fn: 42 | Function or class to be called. 43 | 44 | :param args: 45 | Arguments to the function or class-init. 46 | 47 | :param kwargs: 48 | Keyword arguments to the function or class-init. 49 | 50 | :return: 51 | The result of calling the function or creating the object-instance. 52 | """ 53 | 54 | # If the cache-file exists. 55 | if os.path.exists(cache_path): 56 | # Load the cached data from the file. 57 | with open(cache_path, mode='rb') as file: 58 | obj = pickle.load(file) 59 | 60 | print("- Data loaded from cache-file: " + cache_path) 61 | else: 62 | # The cache-file does not exist. 63 | 64 | # Call the function / class-init with the supplied arguments. 65 | obj = fn(*args, **kwargs) 66 | 67 | # Save the data to a cache-file. 68 | with open(cache_path, mode='wb') as file: 69 | pickle.dump(obj, file) 70 | 71 | print("- Data saved to cache-file: " + cache_path) 72 | 73 | return obj 74 | 75 | 76 | ######################################################################## 77 | 78 | 79 | def convert_numpy2pickle(in_path, out_path): 80 | """ 81 | Convert a numpy-file to pickle-file. 82 | 83 | The first version of the cache-function used numpy for saving the data. 84 | Instead of re-calculating all the data, you can just convert the 85 | cache-file using this function. 86 | 87 | :param in_path: 88 | Input file in numpy-format written using numpy.save(). 89 | 90 | :param out_path: 91 | Output file written as a pickle-file. 92 | 93 | :return: 94 | Nothing. 95 | """ 96 | 97 | # Load the data using numpy. 98 | data = np.load(in_path) 99 | 100 | # Save the data using pickle. 101 | with open(out_path, mode='wb') as file: 102 | pickle.dump(data, file) 103 | 104 | 105 | ######################################################################## 106 | 107 | if __name__ == '__main__': 108 | # This is a short example of using a cache-file. 109 | 110 | # This is the function that will only get called if the result 111 | # is not already saved in the cache-file. This would normally 112 | # be a function that takes a long time to compute, or if you 113 | # need persistent data for some other reason. 114 | def expensive_function(a, b): 115 | return a * b 116 | 117 | print('Computing expensive_function() ...') 118 | 119 | # Either load the result from a cache-file if it already exists, 120 | # otherwise calculate expensive_function(a=123, b=456) and 121 | # save the result to the cache-file for next time. 122 | result = cache(cache_path='cache_expensive_function.pkl', 123 | fn=expensive_function, a=123, b=456) 124 | 125 | print('result =', result) 126 | 127 | # Newline. 128 | print() 129 | 130 | # This is another example which saves an object to a cache-file. 131 | 132 | # We want to cache an object-instance of this class. 133 | # The motivation is to do an expensive computation only once, 134 | # or if we need to persist the data for some other reason. 135 | class ExpensiveClass: 136 | def __init__(self, c, d): 137 | self.c = c 138 | self.d = d 139 | self.result = c * d 140 | 141 | def print_result(self): 142 | print('c =', self.c) 143 | print('d =', self.d) 144 | print('result = c * d =', self.result) 145 | 146 | print('Creating object from ExpensiveClass() ...') 147 | 148 | # Either load the object from a cache-file if it already exists, 149 | # otherwise make an object-instance ExpensiveClass(c=123, d=456) 150 | # and save the object to the cache-file for the next time. 151 | obj = cache(cache_path='cache_ExpensiveClass.pkl', 152 | fn=ExpensiveClass, c=123, d=456) 153 | 154 | obj.print_result() 155 | 156 | ######################################################################## 157 | -------------------------------------------------------------------------------- /machine_learning/utilities/cifar10.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Functions for downloading the CIFAR-10 data-set from the internet 4 | # and loading it into memory. 5 | # 6 | # Implemented in Python 3.5 7 | # 8 | # Usage: 9 | # 1) Set the variable data_path with the desired storage path. 10 | # 2) Call maybe_download_and_extract() to download the data-set 11 | # if it is not already located in the given data_path. 12 | # 3) Call load_class_names() to get an array of the class-names. 13 | # 4) Call load_training_data() and load_test_data() to get 14 | # the images, class-numbers and one-hot encoded class-labels 15 | # for the training-set and test-set. 16 | # 5) Use the returned data in your own program. 17 | # 18 | # Format: 19 | # The images for the training- and test-sets are returned as 4-dim numpy 20 | # arrays each with the shape: [image_number, height, width, channel] 21 | # where the individual pixels are floats between 0.0 and 1.0. 22 | # 23 | ######################################################################## 24 | # 25 | # This file is part of the TensorFlow Tutorials available at: 26 | # 27 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 28 | # 29 | # Published under the MIT License. See the file LICENSE for details. 30 | # 31 | # Copyright 2016 by Magnus Erik Hvass Pedersen 32 | # 33 | ######################################################################## 34 | 35 | import numpy as np 36 | import pickle 37 | import os 38 | from . import download 39 | from .dataset import one_hot_encoded 40 | 41 | ######################################################################## 42 | 43 | # Directory where you want to download and save the data-set. 44 | # Set this before you start calling any of the functions below. 45 | data_path = "data/CIFAR-10/" 46 | 47 | # URL for the data-set on the internet. 48 | data_url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 49 | 50 | ######################################################################## 51 | # Various constants for the size of the images. 52 | # Use these constants in your own program. 53 | 54 | # Width and height of each image. 55 | img_size = 32 56 | 57 | # Number of channels in each image, 3 channels: Red, Green, Blue. 58 | num_channels = 3 59 | 60 | # Length of an image when flattened to a 1-dim array. 61 | img_size_flat = img_size * img_size * num_channels 62 | 63 | # Number of classes. 64 | num_classes = 10 65 | 66 | ######################################################################## 67 | # Various constants used to allocate arrays of the correct size. 68 | 69 | # Number of files for the training-set. 70 | _num_files_train = 5 71 | 72 | # Number of images for each batch-file in the training-set. 73 | _images_per_file = 10000 74 | 75 | # Total number of images in the training-set. 76 | # This is used to pre-allocate arrays for efficiency. 77 | _num_images_train = _num_files_train * _images_per_file 78 | 79 | ######################################################################## 80 | # Private functions for downloading, unpacking and loading data-files. 81 | 82 | 83 | def _get_file_path(filename=""): 84 | """ 85 | Return the full path of a data-file for the data-set. 86 | 87 | If filename=="" then return the directory of the files. 88 | """ 89 | 90 | return os.path.join(data_path, "cifar-10-batches-py/", filename) 91 | 92 | 93 | def _unpickle(filename): 94 | """ 95 | Unpickle the given file and return the data. 96 | 97 | Note that the appropriate dir-name is prepended the filename. 98 | """ 99 | 100 | # Create full path for the file. 101 | file_path = _get_file_path(filename) 102 | 103 | print("Loading data: " + file_path) 104 | 105 | with open(file_path, mode='rb') as file: 106 | # In Python 3.X it is important to set the encoding, 107 | # otherwise an exception is raised here. 108 | data = pickle.load(file, encoding='bytes') 109 | 110 | return data 111 | 112 | 113 | def _convert_images(raw): 114 | """ 115 | Convert images from the CIFAR-10 format and 116 | return a 4-dim array with shape: [image_number, height, width, channel] 117 | where the pixels are floats between 0.0 and 1.0. 118 | """ 119 | 120 | # Convert the raw images from the data-files to floating-points. 121 | raw_float = np.array(raw, dtype=float) / 255.0 122 | 123 | # Reshape the array to 4-dimensions. 124 | images = raw_float.reshape([-1, num_channels, img_size, img_size]) 125 | 126 | # Reorder the indices of the array. 127 | images = images.transpose([0, 2, 3, 1]) 128 | 129 | return images 130 | 131 | 132 | def _load_data(filename): 133 | """ 134 | Load a pickled data-file from the CIFAR-10 data-set 135 | and return the converted images (see above) and the class-number 136 | for each image. 137 | """ 138 | 139 | # Load the pickled data-file. 140 | data = _unpickle(filename) 141 | 142 | # Get the raw images. 143 | raw_images = data[b'data'] 144 | 145 | # Get the class-numbers for each image. Convert to numpy-array. 146 | cls = np.array(data[b'labels']) 147 | 148 | # Convert the images. 149 | images = _convert_images(raw_images) 150 | 151 | return images, cls 152 | 153 | 154 | ######################################################################## 155 | # Public functions that you may call to download the data-set from 156 | # the internet and load the data into memory. 157 | 158 | 159 | def maybe_download_and_extract(): 160 | """ 161 | Download and extract the CIFAR-10 data-set if it doesn't already exist 162 | in data_path (set this variable first to the desired path). 163 | """ 164 | 165 | download.maybe_download_and_extract(url=data_url, download_dir=data_path) 166 | 167 | 168 | def load_class_names(): 169 | """ 170 | Load the names for the classes in the CIFAR-10 data-set. 171 | 172 | Returns a list with the names. Example: names[3] is the name 173 | associated with class-number 3. 174 | """ 175 | 176 | # Load the class-names from the pickled file. 177 | raw = _unpickle(filename="batches.meta")[b'label_names'] 178 | 179 | # Convert from binary strings. 180 | names = [x.decode('utf-8') for x in raw] 181 | 182 | return names 183 | 184 | 185 | def load_training_data(): 186 | """ 187 | Load all the training-data for the CIFAR-10 data-set. 188 | 189 | The data-set is split into 5 data-files which are merged here. 190 | 191 | Returns the images, class-numbers and one-hot encoded class-labels. 192 | """ 193 | 194 | # Pre-allocate the arrays for the images and class-numbers for efficiency. 195 | images = np.zeros(shape=[_num_images_train, img_size, img_size, num_channels], dtype=float) 196 | cls = np.zeros(shape=[_num_images_train], dtype=int) 197 | 198 | # Begin-index for the current batch. 199 | begin = 0 200 | 201 | # For each data-file. 202 | for i in range(_num_files_train): 203 | # Load the images and class-numbers from the data-file. 204 | images_batch, cls_batch = _load_data(filename="data_batch_" + str(i + 1)) 205 | 206 | # Number of images in this batch. 207 | num_images = len(images_batch) 208 | 209 | # End-index for the current batch. 210 | end = begin + num_images 211 | 212 | # Store the images into the array. 213 | images[begin:end, :] = images_batch 214 | 215 | # Store the class-numbers into the array. 216 | cls[begin:end] = cls_batch 217 | 218 | # The begin-index for the next batch is the current end-index. 219 | begin = end 220 | 221 | return images, cls, one_hot_encoded(class_numbers=cls, num_classes=num_classes) 222 | 223 | 224 | def load_test_data(): 225 | """ 226 | Load all the test-data for the CIFAR-10 data-set. 227 | 228 | Returns the images, class-numbers and one-hot encoded class-labels. 229 | """ 230 | 231 | images, cls = _load_data(filename="test_batch") 232 | 233 | return images, cls, one_hot_encoded(class_numbers=cls, num_classes=num_classes) 234 | 235 | ######################################################################## 236 | -------------------------------------------------------------------------------- /machine_learning/utilities/dataset.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Class for creating a data-set consisting of all files in a directory. 4 | # 5 | # Example usage is shown in the file knifey.py and Tutorial #09. 6 | # 7 | # Implemented in Python 3.5 8 | # 9 | ######################################################################## 10 | # 11 | # This file is part of the TensorFlow Tutorials available at: 12 | # 13 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 14 | # 15 | # Published under the MIT License. See the file LICENSE for details. 16 | # 17 | # Copyright 2016 by Magnus Erik Hvass Pedersen 18 | # 19 | ######################################################################## 20 | 21 | import numpy as np 22 | import os 23 | from .cache import cache 24 | 25 | ######################################################################## 26 | 27 | 28 | def one_hot_encoded(class_numbers, num_classes=None): 29 | """ 30 | Generate the One-Hot encoded class-labels from an array of integers. 31 | 32 | For example, if class_number=2 and num_classes=4 then 33 | the one-hot encoded label is the float array: [0. 0. 1. 0.] 34 | 35 | :param class_numbers: 36 | Array of integers with class-numbers. 37 | Assume the integers are from zero to num_classes-1 inclusive. 38 | 39 | :param num_classes: 40 | Number of classes. If None then use max(class_numbers)+1. 41 | 42 | :return: 43 | 2-dim array of shape: [len(class_numbers), num_classes] 44 | """ 45 | 46 | # Find the number of classes if None is provided. 47 | # Assumes the lowest class-number is zero. 48 | if num_classes is None: 49 | num_classes = np.max(class_numbers) + 1 50 | 51 | return np.eye(num_classes, dtype=float)[class_numbers] 52 | 53 | 54 | ######################################################################## 55 | 56 | 57 | class DataSet: 58 | def __init__(self, in_dir, exts='.jpg'): 59 | """ 60 | Create a data-set consisting of the filenames in the given directory 61 | and sub-dirs that match the given filename-extensions. 62 | 63 | For example, the knifey-spoony data-set (see knifey.py) has the 64 | following dir-structure: 65 | 66 | knifey-spoony/forky/ 67 | knifey-spoony/knifey/ 68 | knifey-spoony/spoony/ 69 | knifey-spoony/forky/test/ 70 | knifey-spoony/knifey/test/ 71 | knifey-spoony/spoony/test/ 72 | 73 | This means there are 3 classes called: forky, knifey, and spoony. 74 | 75 | If we set in_dir = "knifey-spoony/" and create a new DataSet-object 76 | then it will scan through these directories and create a training-set 77 | and test-set for each of these classes. 78 | 79 | The training-set will contain a list of all the *.jpg filenames 80 | in the following directories: 81 | 82 | knifey-spoony/forky/ 83 | knifey-spoony/knifey/ 84 | knifey-spoony/spoony/ 85 | 86 | The test-set will contain a list of all the *.jpg filenames 87 | in the following directories: 88 | 89 | knifey-spoony/forky/test/ 90 | knifey-spoony/knifey/test/ 91 | knifey-spoony/spoony/test/ 92 | 93 | See the TensorFlow Tutorial #09 for a usage example. 94 | 95 | :param in_dir: 96 | Root-dir for the files in the data-set. 97 | This would be 'knifey-spoony/' in the example above. 98 | 99 | :param exts: 100 | String or tuple of strings with valid filename-extensions. 101 | Not case-sensitive. 102 | 103 | :return: 104 | Object instance. 105 | """ 106 | 107 | # Extend the input directory to the full path. 108 | in_dir = os.path.abspath(in_dir) 109 | 110 | # Input directory. 111 | self.in_dir = in_dir 112 | 113 | # Convert all file-extensions to lower-case. 114 | self.exts = tuple(ext.lower() for ext in exts) 115 | 116 | # Names for the classes. 117 | self.class_names = [] 118 | 119 | # Filenames for all the files in the training-set. 120 | self.filenames = [] 121 | 122 | # Filenames for all the files in the test-set. 123 | self.filenames_test = [] 124 | 125 | # Class-number for each file in the training-set. 126 | self.class_numbers = [] 127 | 128 | # Class-number for each file in the test-set. 129 | self.class_numbers_test = [] 130 | 131 | # Total number of classes in the data-set. 132 | self.num_classes = 0 133 | 134 | # For all files/dirs in the input directory. 135 | for name in os.listdir(in_dir): 136 | # Full path for the file / dir. 137 | current_dir = os.path.join(in_dir, name) 138 | 139 | # If it is a directory. 140 | if os.path.isdir(current_dir): 141 | # Add the dir-name to the list of class-names. 142 | self.class_names.append(name) 143 | 144 | # Training-set. 145 | 146 | # Get all the valid filenames in the dir (not sub-dirs). 147 | filenames = self._get_filenames(current_dir) 148 | 149 | # Append them to the list of all filenames for the training-set. 150 | self.filenames.extend(filenames) 151 | 152 | # The class-number for this class. 153 | class_number = self.num_classes 154 | 155 | # Create an array of class-numbers. 156 | class_numbers = [class_number] * len(filenames) 157 | 158 | # Append them to the list of all class-numbers for the training-set. 159 | self.class_numbers.extend(class_numbers) 160 | 161 | # Test-set. 162 | 163 | # Get all the valid filenames in the sub-dir named 'test'. 164 | filenames_test = self._get_filenames(os.path.join(current_dir, 'test')) 165 | 166 | # Append them to the list of all filenames for the test-set. 167 | self.filenames_test.extend(filenames_test) 168 | 169 | # Create an array of class-numbers. 170 | class_numbers = [class_number] * len(filenames_test) 171 | 172 | # Append them to the list of all class-numbers for the test-set. 173 | self.class_numbers_test.extend(class_numbers) 174 | 175 | # Increase the total number of classes in the data-set. 176 | self.num_classes += 1 177 | 178 | def _get_filenames(self, dir): 179 | """ 180 | Create and return a list of filenames with matching extensions in the given directory. 181 | 182 | :param dir: 183 | Directory to scan for files. Sub-dirs are not scanned. 184 | 185 | :return: 186 | List of filenames. Only filenames. Does not include the directory. 187 | """ 188 | 189 | # Initialize empty list. 190 | filenames = [] 191 | 192 | # If the directory exists. 193 | if os.path.exists(dir): 194 | # Get all the filenames with matching extensions. 195 | for filename in os.listdir(dir): 196 | if filename.lower().endswith(self.exts): 197 | filenames.append(filename) 198 | 199 | return filenames 200 | 201 | def get_paths(self, test=False): 202 | """ 203 | Get the full paths for the files in the data-set. 204 | 205 | :param test: 206 | Boolean. Return the paths for the test-set (True) or training-set (False). 207 | 208 | :return: 209 | Iterator with strings for the path-names. 210 | """ 211 | 212 | if test: 213 | # Use the filenames and class-numbers for the test-set. 214 | filenames = self.filenames_test 215 | class_numbers = self.class_numbers_test 216 | 217 | # Sub-dir for test-set. 218 | test_dir = "test/" 219 | else: 220 | # Use the filenames and class-numbers for the training-set. 221 | filenames = self.filenames 222 | class_numbers = self.class_numbers 223 | 224 | # Don't use a sub-dir for test-set. 225 | test_dir = "" 226 | 227 | for filename, cls in zip(filenames, class_numbers): 228 | # Full path-name for the file. 229 | path = os.path.join(self.in_dir, self.class_names[cls], test_dir, filename) 230 | 231 | yield path 232 | 233 | def get_training_set(self): 234 | """ 235 | Return the list of paths for the files in the training-set, 236 | and the list of class-numbers as integers, 237 | and the class-numbers as one-hot encoded arrays. 238 | """ 239 | 240 | return list(self.get_paths()), \ 241 | np.asarray(self.class_numbers), \ 242 | one_hot_encoded(class_numbers=self.class_numbers, 243 | num_classes=self.num_classes) 244 | 245 | def get_test_set(self): 246 | """ 247 | Return the list of paths for the files in the test-set, 248 | and the list of class-numbers as integers, 249 | and the class-numbers as one-hot encoded arrays. 250 | """ 251 | 252 | return list(self.get_paths(test=True)), \ 253 | np.asarray(self.class_numbers_test), \ 254 | one_hot_encoded(class_numbers=self.class_numbers_test, 255 | num_classes=self.num_classes) 256 | 257 | 258 | ######################################################################## 259 | 260 | 261 | def load_cached(cache_path, in_dir): 262 | """ 263 | Wrapper-function for creating a DataSet-object, which will be 264 | loaded from a cache-file if it already exists, otherwise a new 265 | object will be created and saved to the cache-file. 266 | 267 | This is useful if you need to ensure the ordering of the 268 | filenames is consistent every time you load the data-set, 269 | for example if you use the DataSet-object in combination 270 | with Transfer Values saved to another cache-file, see e.g. 271 | Tutorial #09 for an example of this. 272 | 273 | :param cache_path: 274 | File-path for the cache-file. 275 | 276 | :param in_dir: 277 | Root-dir for the files in the data-set. 278 | This is an argument for the DataSet-init function. 279 | 280 | :return: 281 | The DataSet-object. 282 | """ 283 | 284 | print("Creating dataset from the files in: " + in_dir) 285 | 286 | # If the object-instance for DataSet(in_dir=data_dir) already 287 | # exists in the cache-file then reload it, otherwise create 288 | # an object instance and save it to the cache-file for next time. 289 | dataset = cache(cache_path=cache_path, 290 | fn=DataSet, in_dir=in_dir) 291 | 292 | return dataset 293 | 294 | 295 | ######################################################################## 296 | -------------------------------------------------------------------------------- /machine_learning/utilities/download.py: -------------------------------------------------------------------------------- 1 | ######################################################################## 2 | # 3 | # Functions for downloading and extracting data-files from the internet. 4 | # 5 | # Implemented in Python 3.5 6 | # 7 | ######################################################################## 8 | # 9 | # This file is part of the TensorFlow Tutorials available at: 10 | # 11 | # https://github.com/Hvass-Labs/TensorFlow-Tutorials 12 | # 13 | # Published under the MIT License. See the file LICENSE for details. 14 | # 15 | # Copyright 2016 by Magnus Erik Hvass Pedersen 16 | # 17 | ######################################################################## 18 | 19 | import sys 20 | import os 21 | import urllib.request 22 | import tarfile 23 | import zipfile 24 | 25 | ######################################################################## 26 | 27 | 28 | def _print_download_progress(count, block_size, total_size): 29 | """ 30 | Function used for printing the download progress. 31 | Used as a call-back function in maybe_download_and_extract(). 32 | """ 33 | 34 | # Percentage completion. 35 | pct_complete = float(count * block_size) / total_size 36 | 37 | # Status-message. Note the \r which means the line should overwrite itself. 38 | msg = "\r- Download progress: {0:.1%}".format(pct_complete) 39 | 40 | # Print it. 41 | sys.stdout.write(msg) 42 | sys.stdout.flush() 43 | 44 | 45 | ######################################################################## 46 | 47 | 48 | def maybe_download_and_extract(url, download_dir): 49 | """ 50 | Download and extract the data if it doesn't already exist. 51 | Assumes the url is a tar-ball file. 52 | 53 | :param url: 54 | Internet URL for the tar-file to download. 55 | Example: "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 56 | 57 | :param download_dir: 58 | Directory where the downloaded file is saved. 59 | Example: "data/CIFAR-10/" 60 | 61 | :return: 62 | Nothing. 63 | """ 64 | 65 | # Filename for saving the file downloaded from the internet. 66 | # Use the filename from the URL and add it to the download_dir. 67 | filename = url.split('/')[-1] 68 | file_path = os.path.join(download_dir, filename) 69 | 70 | # Check if the file already exists. 71 | # If it exists then we assume it has also been extracted, 72 | # otherwise we need to download and extract it now. 73 | if not os.path.exists(file_path): 74 | # Check if the download directory exists, otherwise create it. 75 | if not os.path.exists(download_dir): 76 | os.makedirs(download_dir) 77 | 78 | # Download the file from the internet. 79 | file_path, _ = urllib.request.urlretrieve(url=url, 80 | filename=file_path, 81 | reporthook=_print_download_progress) 82 | 83 | print() 84 | print("Download finished. Extracting files.") 85 | 86 | if file_path.endswith(".zip"): 87 | # Unpack the zip-file. 88 | zipfile.ZipFile(file=file_path, mode="r").extractall(download_dir) 89 | elif file_path.endswith((".tar.gz", ".tgz")): 90 | # Unpack the tar-ball. 91 | tarfile.open(name=file_path, mode="r:gz").extractall(download_dir) 92 | 93 | print("Done.") 94 | else: 95 | print("Data has apparently already been downloaded and unpacked.") 96 | 97 | 98 | ######################################################################## 99 | -------------------------------------------------------------------------------- /machine_learning/utilities/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/machine_learning/utilities/logo.png -------------------------------------------------------------------------------- /machine_learning/utilities/utils.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | def lazy_property(function): 4 | attribute = '_lazy_' + function.__name__ 5 | 6 | @property 7 | @functools.wraps(function) 8 | def wrapper(self): 9 | if not hasattr(self, attribute): 10 | setattr(self, attribute, function(self)) 11 | return getattr(self, attribute) 12 | return wrapper 13 | 14 | 15 | """ 16 | (Copied from OpenDeep https://github.com/vitruvianscience/opendeep) 17 | 18 | This module contains different utility functions that are not connected 19 | in anyway to the networks presented in the tutorials, but rather help in 20 | processing the outputs into a more understandable way. 21 | 22 | For example ``tile_raster_images`` helps in generating a easy to grasp 23 | image from a set of samples or weights. 24 | 25 | Written by Li Yao (University of Montreal) 26 | https://github.com/yaoli/GSN 27 | """ 28 | 29 | import numpy 30 | 31 | def scale_to_unit_interval(ndar, eps=1e-8): 32 | """ 33 | Scales all values in the ndarray 'ndar' to be between 0 and 1. 34 | 35 | Parameters 36 | ---------- 37 | ndar : numpy.ndarray 38 | The input array to scale values. 39 | eps : float 40 | Small value to avoid divide-by-zero when scaling. 41 | 42 | Returns 43 | ------- 44 | numpy.ndarray 45 | The input array scaled to be between 0 and 1. 46 | """ 47 | ndar = ndar.copy() 48 | ndar -= ndar.min() 49 | ndar *= 1.0 / (ndar.max() + eps) 50 | return ndar 51 | 52 | def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), 53 | scale_rows_to_unit_interval=True, 54 | output_pixel_vals=True): 55 | """ 56 | Transform an array with one flattened image per row, into an array in 57 | which images are reshaped and layed out like tiles on a floor. 58 | 59 | This function is useful for visualizing datasets whose rows are images, 60 | and also columns of matrices for transforming those rows 61 | (such as the first layer of a neural net). 62 | 63 | Parameters 64 | ---------- 65 | X : 2D ndarray or a tuple of 4 channels, elements of which can be 2D ndarrays or None 66 | A 2D array in which every row is a flattened image. 67 | img_shape : tuple 68 | The original (height, width) shape of each image. 69 | tile_shape : tuple 70 | The number of images to tile (rows, cols). 71 | tile_spacing : tuple 72 | The amount of pixels to put between image tiles (like a border size). 73 | scale_rows_to_unit_interval : bool 74 | If the values need to be scaled before being plotted to [0,1] or not. 75 | output_pixel_vals : bool 76 | If output should be pixel values (i.e. int8 values) or floats. 77 | 78 | Returns 79 | ------- 80 | 2D array 81 | Array suitable for viewing as an image. (See:`PIL.Image.fromarray`.) 82 | """ 83 | 84 | assert len(img_shape) == 2 85 | assert len(tile_shape) == 2 86 | assert len(tile_spacing) == 2 87 | 88 | # The expression below can be re-written in a more C style as 89 | # follows : 90 | # 91 | # out_shape = [0,0] 92 | # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - 93 | # tile_spacing[0] 94 | # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - 95 | # tile_spacing[1] 96 | out_shape = [(ishp + tsp) * tshp - tsp for ishp, tshp, tsp 97 | in zip(img_shape, tile_shape, tile_spacing)] 98 | 99 | if isinstance(X, tuple): 100 | assert len(X) == 4 101 | # Create an output numpy ndarray to store the image 102 | if output_pixel_vals: 103 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 104 | dtype='uint8') 105 | else: 106 | out_array = numpy.zeros((out_shape[0], out_shape[1], 4), 107 | dtype=X.dtype) 108 | 109 | #colors default to 0, alpha defaults to 1 (opaque) 110 | if output_pixel_vals: 111 | channel_defaults = [0, 0, 0, 255] 112 | else: 113 | channel_defaults = [0., 0., 0., 1.] 114 | 115 | for i in xrange(4): 116 | if X[i] is None: 117 | # if channel is None, fill it with zeros of the correct 118 | # dtype 119 | dt = out_array.dtype 120 | if output_pixel_vals: 121 | dt = 'uint8' 122 | out_array[:, :, i] = numpy.zeros(out_shape, 123 | dtype=dt) + channel_defaults[i] 124 | else: 125 | # use a recurrent call to run the channel and store it 126 | # in the output 127 | out_array[:, :, i] = tile_raster_images( 128 | X[i], img_shape, tile_shape, tile_spacing, 129 | scale_rows_to_unit_interval, output_pixel_vals) 130 | return out_array 131 | 132 | else: 133 | # if we are dealing with only one channel 134 | H, W = img_shape 135 | Hs, Ws = tile_spacing 136 | 137 | # generate a matrix to store the output 138 | dt = X.dtype 139 | if output_pixel_vals: 140 | dt = 'uint8' 141 | out_array = numpy.zeros(out_shape, dtype=dt) 142 | 143 | for tile_row in xrange(tile_shape[0]): 144 | for tile_col in xrange(tile_shape[1]): 145 | if tile_row * tile_shape[1] + tile_col < X.shape[0]: 146 | this_x = X[tile_row * tile_shape[1] + tile_col] 147 | if scale_rows_to_unit_interval: 148 | # if we should scale values to be between 0 and 1 149 | # do this by calling the `scale_to_unit_interval` 150 | # function 151 | this_img = scale_to_unit_interval( 152 | this_x.reshape(img_shape)) 153 | else: 154 | this_img = this_x.reshape(img_shape) 155 | # add the slice to the corresponding position in the 156 | # output array 157 | c = 1 158 | if output_pixel_vals: 159 | c = 255 160 | out_array[ 161 | tile_row * (H + Hs): tile_row * (H + Hs) + H, 162 | tile_col * (W + Ws): tile_col * (W + Ws) + W 163 | ] = this_img * c 164 | return out_array 165 | -------------------------------------------------------------------------------- /numpy/images/IIR_filters_table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/IIR_filters_table.jpg -------------------------------------------------------------------------------- /numpy/images/arrayobject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/arrayobject.jpg -------------------------------------------------------------------------------- /numpy/images/carrayobject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/carrayobject.jpg -------------------------------------------------------------------------------- /numpy/images/convex_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/convex_function.png -------------------------------------------------------------------------------- /numpy/images/earrayobject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/earrayobject.jpg -------------------------------------------------------------------------------- /numpy/images/filter_design.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/filter_design.jpg -------------------------------------------------------------------------------- /numpy/images/non_convex_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/non_convex_function.png -------------------------------------------------------------------------------- /numpy/images/non_smooth_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/non_smooth_function.png -------------------------------------------------------------------------------- /numpy/images/numpy_array.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/numpy_array.jpg -------------------------------------------------------------------------------- /numpy/images/smooth_function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/smooth_function.png -------------------------------------------------------------------------------- /numpy/images/tableobject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/images/tableobject.jpg -------------------------------------------------------------------------------- /numpy/index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "* [__Numpy Basic Concepts__](np01v01_numpy_basics.ipynb)\n", 8 | " * [1 What is Numpy ?](np01v01_numpy_basics.ipynb#1-What-is-Numpy-?)\n", 9 | " * [1.1 Documentation and reference:](np01v01_numpy_basics.ipynb#1.1-Documentation-and-reference:)\n", 10 | " * [2 Array Creation](np01v01_numpy_basics.ipynb#2-Array-Creation)\n", 11 | " * [2.1 Array creation functions](np01v01_numpy_basics.ipynb#2.1-Array-creation-functions)\n", 12 | " * [2.2 Sequences and reshaping](np01v01_numpy_basics.ipynb#2.2-Sequences-and-reshaping)\n", 13 | " * [2.3 Sparse Matrices](np01v01_numpy_basics.ipynb#2.3-Sparse-Matrices)\n", 14 | " * [2.4 Random Numbers](np01v01_numpy_basics.ipynb#2.4-Random-Numbers)\n", 15 | " * [2.5 Casting](np01v01_numpy_basics.ipynb#2.5-Casting)\n", 16 | " * [3 Basic Visualization with matplotlib](np01v01_numpy_basics.ipynb#3-Basic-Visualization-with-matplotlib)\n", 17 | " * [3.1 D Plotting](np01v01_numpy_basics.ipynb#3.1-D-Plotting)\n", 18 | " * [4 Basic Linear Algebra](np01v01_numpy_basics.ipynb#4-Basic-Linear-Algebra)\n", 19 | " * [4.1 Determinant of a square matrix](np01v01_numpy_basics.ipynb#4.1-Determinant-of-a-square-matrix)\n", 20 | " * [4.2 Inverse of a square matrix](np01v01_numpy_basics.ipynb#4.2-Inverse-of-a-square-matrix)\n", 21 | " * [4.3 Advanced Linear Algebra](np01v01_numpy_basics.ipynb#4.3-Advanced-Linear-Algebra)\n", 22 | " * [5 Slicing -Indexing for MATLAB® Users-](np01v01_numpy_basics.ipynb#5-Slicing--Indexing-for-MATLAB®-Users-)\n", 23 | " * [5.1 Indexing single elements](np01v01_numpy_basics.ipynb#5.1-Indexing-single-elements)\n", 24 | " * [5.2 Indexing by rows and columns](np01v01_numpy_basics.ipynb#5.2-Indexing-by-rows-and-columns)\n", 25 | " * [6 File Input / Output](np01v01_numpy_basics.ipynb#6-File-Input-/-Output)\n", 26 | "* [__PyTables__](np02v01_numpy_PyTables.ipynb)\n", 27 | " * [1 The Array Object](np02v01_numpy_PyTables.ipynb#1-The-Array-Object)\n", 28 | " * [2 The CArray Object](np02v01_numpy_PyTables.ipynb#2-The-CArray-Object)\n", 29 | " * [3 The Table Object](np02v01_numpy_PyTables.ipynb#3-The-Table-Object)\n", 30 | " * [4 The EArray Object](np02v01_numpy_PyTables.ipynb#4-The-EArray-Object)\n", 31 | "* [__Numpy - Plotting with Matplotlib__](np03v01_numpy_plotting.ipynb)\n", 32 | " * [1 Simple 2D Plots](np03v01_numpy_plotting.ipynb#1-Simple-2D-Plots)\n", 33 | " * [2 Subplots](np03v01_numpy_plotting.ipynb#2-Subplots)\n", 34 | " * [3 Axes](np03v01_numpy_plotting.ipynb#3-Axes)\n", 35 | " * [4 Example: Cycle Trough many colors](np03v01_numpy_plotting.ipynb#4-Example:-Cycle-Trough-many-colors)\n", 36 | " * [5 Example: Filled Plots](np03v01_numpy_plotting.ipynb#5-Example:-Filled-Plots)\n", 37 | " * [6 Example: Custom Scatter Plot](np03v01_numpy_plotting.ipynb#6-Example:-Custom-Scatter-Plot)\n", 38 | " * [7 Example: Custom Histogram](np03v01_numpy_plotting.ipynb#7-Example:-Custom-Histogram)\n", 39 | " * [8 Example: Simple 3D Plot - Surface](np03v01_numpy_plotting.ipynb#8-Example:-Simple-3D-Plot---Surface)\n", 40 | " * [9 Example: Colormaps](np03v01_numpy_plotting.ipynb#9-Example:-Colormaps)\n", 41 | " * [10 Example: Skew and Kurtosis](np03v01_numpy_plotting.ipynb#10-Example:-Skew-and-Kurtosis)\n", 42 | "* [__Scipy - Optimization__](np04v01_scipy_optimization.ipynb)\n", 43 | " * [1 Brief Introduction to Optimization](np04v01_scipy_optimization.ipynb#1-Brief-Introduction-to-Optimization)\n", 44 | " * [1.1 Dimensionality of the problem](np04v01_scipy_optimization.ipynb#1.1-Dimensionality-of-the-problem)\n", 45 | " * [1.2 Convex versus non-convex optimization](np04v01_scipy_optimization.ipynb#1.2-Convex-versus-non-convex-optimization)\n", 46 | " * [1.3 Smooth and non-smooth problems](np04v01_scipy_optimization.ipynb#1.3-Smooth-and-non-smooth-problems)\n", 47 | " * [2 A Simpe Optimization Example](np04v01_scipy_optimization.ipynb#2-A-Simpe-Optimization-Example)\n", 48 | " * [3 Finding the roots of a scalar function](np04v01_scipy_optimization.ipynb#3-Finding-the-roots-of-a-scalar-function)\n", 49 | " * [4 Curve Fitting](np04v01_scipy_optimization.ipynb#4-Curve-Fitting)\n", 50 | " * [5 Optimizing with Box-Bound Constraints](np04v01_scipy_optimization.ipynb#5-Optimizing-with-Box-Bound-Constraints)\n", 51 | "* [__1 Scipy Signal Processing: IIR Filter Design__](np05v01_scipy_sig_processing_IIRfilter_design.ipynb)\n", 52 | " * [1 Filter Specification](np05v01_scipy_sig_processing_IIRfilter_design.ipynb#1-Filter-Specification)\n", 53 | " * [2 Filter Design](np05v01_scipy_sig_processing_IIRfilter_design.ipynb#2-Filter-Design)\n", 54 | " * [3 Frequency and Phase Response](np05v01_scipy_sig_processing_IIRfilter_design.ipynb#3-Frequency-and-Phase-Response)\n", 55 | " * [4 Step and Impulse Response](np05v01_scipy_sig_processing_IIRfilter_design.ipynb#4-Step-and-Impulse-Response)\n", 56 | "* [__Symbolic Computation__](np06v01_Symbolic_Computation.ipynb)\n", 57 | " * [1 Basic manipulation](np06v01_Symbolic_Computation.ipynb#1-Basic-manipulation)\n", 58 | " * [2 Solve](np06v01_Symbolic_Computation.ipynb#2-Solve)\n", 59 | " * [3 Algebra](np06v01_Symbolic_Computation.ipynb#3-Algebra)\n", 60 | " * [3.1 Expand and Factor](np06v01_Symbolic_Computation.ipynb#3.1-Expand-and-Factor)\n", 61 | " * [3.2 Symplify](np06v01_Symbolic_Computation.ipynb#3.2-Symplify)\n", 62 | " * [3.3 apart and together](np06v01_Symbolic_Computation.ipynb#3.3-apart-and-together)\n", 63 | " * [4 Calculus](np06v01_Symbolic_Computation.ipynb#4-Calculus)\n", 64 | " * [4.1 Differentiation and Integration](np06v01_Symbolic_Computation.ipynb#4.1-Differentiation-and-Integration)\n", 65 | " * [4.2 Sum and Products](np06v01_Symbolic_Computation.ipynb#4.2-Sum-and-Products)\n", 66 | " * [4.3 Limits](np06v01_Symbolic_Computation.ipynb#4.3-Limits)\n", 67 | " * [4.4 Series](np06v01_Symbolic_Computation.ipynb#4.4-Series)\n", 68 | " * [5 Linear algebra](np06v01_Symbolic_Computation.ipynb#5-Linear-algebra)\n", 69 | " * [5.1 Matrices](np06v01_Symbolic_Computation.ipynb#5.1-Matrices)\n", 70 | " * [6 Solving Equations](np06v01_Symbolic_Computation.ipynb#6-Solving-Equations)\n" 71 | ] 72 | } 73 | ], 74 | "metadata": {}, 75 | "nbformat": 4, 76 | "nbformat_minor": 0 77 | } -------------------------------------------------------------------------------- /numpy/temp/temporary.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/temp/temporary.txt -------------------------------------------------------------------------------- /numpy/utilities/Thumbs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/utilities/Thumbs.db -------------------------------------------------------------------------------- /numpy/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | from .plot_utilities import plot_01 2 | 3 | -------------------------------------------------------------------------------- /numpy/utilities/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/utilities/__init__.pyc -------------------------------------------------------------------------------- /numpy/utilities/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/utilities/logo.png -------------------------------------------------------------------------------- /numpy/utilities/plot_utilities.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import bokeh.plotting as bk 3 | 4 | def plot_01(): 5 | image = np.random.randn(120, 120) 6 | fig = bk.figure() 7 | fig.image([image], x=[0], y=[0], dw=[1], dh=[1], palette='OrRd9') 8 | return fig 9 | 10 | if __name__ == '__main__': 11 | fig = plot_01() 12 | bk.show(fig) 13 | -------------------------------------------------------------------------------- /numpy/utilities/plot_utilities.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/utilities/plot_utilities.pyc -------------------------------------------------------------------------------- /numpy/utilities/vlarrayobject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/numpy/utilities/vlarrayobject.jpg -------------------------------------------------------------------------------- /pandas/example_data/CrimeStatebyState.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/CrimeStatebyState.csv.bz2 -------------------------------------------------------------------------------- /pandas/example_data/babynames.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/babynames.zip -------------------------------------------------------------------------------- /pandas/example_data/cities.csv.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/cities.csv.bz2 -------------------------------------------------------------------------------- /pandas/example_data/company.csv: -------------------------------------------------------------------------------- 1 | Company,Value,expenses,employees,Dimension 2 | Dapibus Company,96 008,7124,78,Big 3 | Pede Blandit Congue Company,61 562,8454,60,Big 4 | Pede Suspendisse Associates,54 728,6641,4,Small 5 | Dictum Associates,16 802,6498,89,Big 6 | Dui Cras Pellentesque Ltd,93 954,5040,97,Big 7 | Ligula Industries,67 131,2169,26,Small 8 | Quis Consulting,50 171,1896,1,Small 9 | Ipsum Primis Consulting,59 808,3866,50,Big 10 | Velit Quisque Varius PC,32 458,6488,74,Big 11 | Pede Et Industries,87 795,5070,54,Small 12 | Tortor Integer LLP,24 259,4940,83,Big 13 | Suspendisse Tristique LLC,46 868,6538,85,Big 14 | Velit Cras Lorem Associates,54 467,6746,76,Big 15 | Non Vestibulum Corp.,42 683,441,81,Big 16 | Venenatis Associates,69 313,4222,15,Small 17 | Elit Erat Consulting,70 729,2154,54,Big 18 | Phasellus At LLP,50 704,6201,79,Big 19 | Eleifend Vitae Company,76 979,2783,5,Small 20 | Nec Quam Curabitur Company,93 879,9150,72,Small 21 | Dapibus Gravida Aliquam LLC,44 292,6322,85,Big 22 | Donec Tempus LLP,76 095,3245,98,Big 23 | Ut Industries,53 086,1543,19,Small 24 | Tellus LLC,52 341,4517,39,Small 25 | Aliquam Institute,82 638,1757,46,Big 26 | Pharetra Ut Pharetra Associates,50 931,8194,88,Big 27 | Sociis Natoque Institute,18 942,9154,28,Small 28 | Congue Foundation,93 274,8854,91,Big 29 | Tempor Lorem Eget PC,39 332,2784,64,Big 30 | Non Inc.,81 641,7963,60,Small 31 | Eu Enim Limited,39 547,1847,42,Small 32 | Ipsum Primis Company,38 414,2354,65,Big 33 | Aliquam Corp.,7 108,2708,96,Big 34 | Orci Tincidunt Adipiscing Foundation,3 512,5730,18,Small 35 | Nunc Lectus Corporation,64 802,6044,71,Big 36 | A Arcu Sed Foundation,38 989,3500,85,Big 37 | Etiam Industries,94 376,1589,3,Small 38 | Curabitur Ut Company,19 663,490,98,Big 39 | Parturient Ltd,76 408,2750,91,Big 40 | Sit Inc.,0 700,4621,28,Small 41 | Eget Inc.,87 539,123,61,Small 42 | Vulputate Ullamcorper LLC,57 084,2640,28,Big 43 | Ipsum Dolor Institute,4 533,8109,39,Big 44 | Praesent Eu Associates,14 678,5726,22,Small 45 | Tristique Pellentesque Tellus Associates,57 845,3345,7,Small 46 | Lobortis Tellus Justo Institute,73 097,9832,7,Big 47 | Sit Incorporated,50 811,2473,32,Big 48 | In Tincidunt Congue Foundation,97 949,4712,54,Small 49 | Accumsan Foundation,49 265,8817,57,Big 50 | Nunc Quis Arcu LLC,32 628,7626,5,Big 51 | Molestie Arcu PC,71 333,1602,52,Small 52 | Purus Maecenas Libero Associates,63 720,397,42,Small 53 | Pharetra Company,12 599,5591,27,Big 54 | Per Conubia Nostra Consulting,47 759,6314,74,Big 55 | Pellentesque Massa Foundation,31 081,2143,8,Small 56 | Nisi Aenean LLP,81 471,9904,25,Big 57 | Neque Morbi Quis LLP,33 711,7388,76,Big 58 | Eros Proin Ultrices Foundation,45 248,8842,3,Small 59 | Metus In Lorem Foundation,91 207,9710,47,Small 60 | Ac Mattis Corporation,34 014,1604,71,Big 61 | Ante Lectus Convallis Associates,16 396,223,13,Big 62 | Quam A Felis Inc.,64 656,1163,35,Small 63 | Quam Elementum Company,35 281,8793,78,Small 64 | Quis Urna Nunc PC,47 258,5227,60,Big 65 | Lorem LLP,56 759,7643,48,Big 66 | Nulla Tempor Augue Consulting,30 122,547,36,Small 67 | Laoreet Libero Et Associates,54 799,4976,2,Small 68 | Adipiscing Lobortis Risus Corporation,2 274,1549,32,Big 69 | At Pretium LLC,71 508,5802,50,Big 70 | Malesuada Inc.,96 400,7850,2,Small 71 | Scelerisque Neque Sed LLP,54 690,2358,63,Small 72 | Sagittis Semper Nam Associates,86 689,8589,19,Big 73 | Eu Consulting,32 837,9806,3,Big 74 | Dapibus Gravida Aliquam Inc.,84 278,9552,8,Small 75 | Sodales Purus Foundation,97 506,9110,6,Small 76 | Sed LLP,39 908,3986,84,Big 77 | Risus In Foundation,63 746,7785,44,Big 78 | Ultricies Limited,29 023,1281,53,Small 79 | Ullamcorper Nisl Arcu Company,72 141,7325,28,Small 80 | Natoque Corporation,15 235,3231,48,Big 81 | Nulla Consulting,51 637,9654,79,Big 82 | Enim Associates,79 663,4485,56,Small 83 | Elit Fermentum Risus Institute,72 291,3615,59,Small 84 | Cum Inc.,48 305,7682,81,Big 85 | Nisl Quisque Fringilla Corp.,47 620,8494,26,Big 86 | Interdum Nunc Institute,26 733,5672,27,Small 87 | Lacus Corporation,83 805,1964,81,Small 88 | Lorem Semper Auctor Consulting,48 217,7579,65,Big 89 | Lectus Quis LLP,84 659,2707,79,Big 90 | Sed LLP,99 252,3015,24,Small 91 | Eget Institute,52 530,3655,4,Small -------------------------------------------------------------------------------- /pandas/example_data/generated.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | 513 | 514 | 515 | 516 | 517 | 518 | 519 | 520 | 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 531 | 532 | 533 | 534 | 535 | 536 | 537 | 538 | 539 | 540 | 541 | 542 | 543 | 544 | 545 | 546 | 547 | 548 | 549 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 579 | 580 | 581 | 582 | 583 | 584 | 585 | 586 | 587 | 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 | 604 | 605 | 606 | 607 | 608 | 609 | 610 | 611 | 612 | 613 | 614 | 615 | 616 | 617 | 618 | 619 | 620 | 621 | 622 | 623 | 624 | 625 | 626 | 627 | 628 | 629 | 630 | 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | 640 | 641 | 642 | 643 | 644 | 645 | 646 | 647 | 648 | 649 | 650 | 651 | 652 | 653 | 654 | 655 | 656 | 657 | 658 | 659 | 660 | 661 | 662 | 663 | 664 | 665 | 666 | 667 | 668 | 669 | 670 | 671 | 672 | 673 | 674 | 675 | 676 | 677 | 678 | 679 | 680 | 681 | 682 | 683 | 684 | 685 | 686 | 687 | 688 | 689 | 690 | 691 | 692 | 693 | 694 | 695 | 696 | 697 | 698 | 699 | 700 | 701 | 702 | 703 | 704 | 705 | 706 | 707 | 708 | 709 | 710 | 711 | 712 | 713 | 714 | 715 | 716 | 717 | 718 | 719 | 720 | 721 | 722 | 723 | 724 | 725 | 726 | 727 | 728 | 729 | 730 | 731 | 732 | 733 | 734 | 735 | 736 | 737 | 738 | 739 | 740 | 741 | 742 | 743 | 744 | 745 | 746 | 747 | 748 | 749 | 750 | 751 | 752 | 753 | 754 | 755 | 756 | 757 | 758 | 759 | 760 | 761 | 762 | 763 | 764 | 765 | 766 | 767 | 768 | 769 | 770 | 771 | 772 | 773 | 774 | 775 | 776 | 777 | 778 | 779 | 780 | 781 | 782 | 783 | 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 801 | 802 | 803 | 804 | 805 | 806 | 807 | 808 | 809 | 810 | 811 | 812 | 813 | 814 | 815 | 816 | 817 |
Nomi maschiliStradeCittàCoordinateEntemail
Drew5672 Accumsan RoadWhitburn-54.68429, -67.21709Ac Consultinga.aliquet.vel@vitae.co.uk
FinnAp #170-4074 Interdum Rd.Pelotas81.93087, 168.14556Varius Incorporatednec.malesuada.ut@primisin.com
MartinAp #471-6260 Etiam AveAliano33.99272, -148.70584Arcu Industrieslacus@risus.net
ZephaniaAp #720-7951 Aliquam AveSenneville-56.82569, -72.80435Tristique PCarcu@sempereratin.co.uk
Devin3538 Dui St.Zwettl-Niederösterreich-11.90874, -149.61123Imperdiet Dictum PClectus.rutrum.urna@massaQuisqueporttitor.ca
ZeusP.O. Box 311, 8130 Parturient St.Birmingham27.97063, -82.8825Convallis Dolor Quisque Associatesmagnis.dis@enim.org
Darius6352 Arcu. RoadEckville24.72404, -88.57903Mollis Duis Corporationnunc.sit.amet@orci.net
AbdulAp #175-4584 Aliquam RoadStamford45.71206, -100.45745Commodo At Libero Corporationipsum.Suspendisse.sagittis@risusvariusorci.co.uk
Tanek470-2620 Eget RoadBillings7.32109, -19.88892Vitae LtdNunc@pharetranibh.edu
MichaelP.O. Box 695, 1340 Vitae St.Worcester-27.5433, 95.33864Pellentesque A Companyin.tempus.eu@nasceturridiculusmus.ca
Leroy9413 Euismod AveCap-Rouge-62.97398, -138.46845Consequat ConsultingCurae.Donec.tincidunt@dui.edu
Palmer2634 Neque. St.Wiekevorst75.50936, -128.0153Mattis LLCaliquet.nec.imperdiet@nislQuisque.com
AmirAp #417-6582 Mauris AvenueDenderbelle-5.46157, -101.51034Nisl Arcu Associatesac.arcu.Nunc@tempusnonlacinia.org
Odysseus909 Lectus St.Macduff-69.08257, 78.2101Risus Incorporatedin.hendrerit@rutrum.co.uk
NormanAp #160-3680 Enim AveMainz30.37536, -11.33456Turpis In Condimentum Companyeleifend@aliquetmolestietellus.com
Victor7796 Lacus. StreetMorro Reatino68.25806, -19.4463Quis Institutetincidunt.tempus@tinciduntvehicula.org
AhmedP.O. Box 751, 2001 Nulla Rd.Ourense63.51301, 68.03157Nam Corp.mollis.non.cursus@natoque.edu
Raymond902-6019 Quis AveQuenast-73.40019, 33.47767Sapien Limitedmauris@Maecenasmalesuadafringilla.ca
Dorian8232 Dolor StreetHomburg87.94352, 59.10144Sit Amet Ante Corp.primis@sem.ca
Charles606 Ligula. RoadAlert Bay-9.7252, 96.34346Mus Donec Dignissim PCut@Aliquam.org
Ivor6558 Gravida StreetNeerharen-37.43899, 88.29297Arcu Vestibulum Ltdmalesuada.fames@atnisi.edu
Maxwell9325 Massa St.Rabbi15.90718, 47.99992Lorem Sit Consultingac.risus@felisorci.com
JudahP.O. Box 632, 937 Nam Av.Dreieich80.002, -118.97661Nec Mollis Vitae Limitedac.nulla@semper.com
IraP.O. Box 129, 1090 Eros. StreetSalles-89.26375, 64.90289Tellus Institutefeugiat@pede.org
Hu977-4679 Vulputate, Av.Dereham-47.67706, -25.44331Eu Lacus Foundationmetus.Vivamus.euismod@Loremipsum.net
Josiah3418 Nisl Rd.Mayerthorpe75.77996, 40.90747Eros Proin LLCdolor@dictumProin.net
James932-7994 Eu, Rd.Feldkirchen in Kärnten79.52423, -179.26425Nam Porttitor Corporationipsum@non.ca
EmeryAp #793-4876 Elit. AveAtlanta-85.10023, -69.87912Maecenas Iaculis Aliquet LLCeuismod@atiaculisquis.ca
MacaulayP.O. Box 628, 7267 Aliquet, Av.Hartlepool64.51014, 102.71763Consectetuer Adipiscing Elit Corp.nonummy@justo.net
HunterP.O. Box 733, 7118 Vitae AvenueGrimbergen-81.05205, 151.02904Non Ante Bibendum Corp.enim@adipiscingelit.ca
OttoAp #372-9714 Vulputate RoadGuápiles-62.28744, -27.15641Luctus Consultingurna.Nullam@interdum.ca
Eaton383-8286 Vivamus St.Lusevera-80.75233, 62.81803In Dolor Fusce LLPamet@mauris.edu
Stewart373-3287 Suspendisse St.Frankenthal-24.48467, -92.12742Magna LimitedInteger@malesuadaaugueut.ca
GavinAp #205-165 Vel, AveBonnyville-82.21383, -87.51734Mollis Integer Consultingdictum.Proin.eget@estmollisnon.net
Wallace8750 Aliquet, AveShrewsbury-84.82663, 111.14283Ullamcorper Corp.quam.Pellentesque@tincidunt.net
BrunoP.O. Box 109, 6155 Donec St.Fogliano Redipuglia6.466, 131.52495Sapien Consultingadipiscing@enimcondimentumeget.edu
AristotleAp #785-7702 Curabitur Av.Alexandria-50.83322, -126.75836Eleifend Egestas CompanyDuis.elementum@eueuismodac.edu
Malcolm2091 Sit AvenueSant'Elena-10.15008, 147.76807Diam Lorem Auctor Incorporatedsollicitudin@purussapien.edu
AhmedP.O. Box 729, 5824 Vulputate, AvenueFresno35.95406, -137.59643Magna Sed Dui Corp.enim.Etiam@etmalesuadafames.org
Branden168-1602 Sit RoadMontecarotto3.51691, -173.96222Netus Et Ltdut.nisi.a@diamloremauctor.edu
ChristopherP.O. Box 221, 5027 Dictum. Av.San Pietro Avellana-79.93685, -60.65692Dolor Dapibus Gravida Consultingenim.Etiam.gravida@quislectus.ca
ErasmusAp #129-1093 Nam St.Lapscheure50.5885, -101.19138Libero Proin Sed Corporationmalesuada.fringilla@egestas.net
Brock764 Sed, Rd.Altmünster2.03395, 90.56518Eleifend Nunc Institutevelit.Sed@urnaUt.co.uk
Lev859-2384 Ut Rd.Independence-23.7638, -97.93639Leo Vivamus Corp.Cras.pellentesque.Sed@Donecsollicitudin.org
AustinP.O. Box 220, 8336 Auctor AvenueLingen-21.08924, 57.4924Quam Inc.lacus.Mauris.non@dolorelit.com
KennanP.O. Box 298, 2240 Tincidunt RoadWhangarei87.56903, 44.87917Nullam Corporationnisl.Quisque@Vivamus.co.uk
NathanAp #965-4519 Tincidunt, StreetHuntly-8.09177, -57.27253Fames Ac Corporationmattis.ornare.lectus@ullamcorperviverra.org
Trevor103-7287 Ante Rd.Worksop-15.69221, 140.61859Feugiat Nec Incorporatedvitae.erat@felisadipiscingfringilla.net
Kaseem9878 Varius. Rd.Assiniboia-22.71377, -156.83994Ipsum Donec LLPsagittis.lobortis@porttitor.co.uk
Tyler9950 Donec RoadFalerone-87.35054, -82.91209Et Netus Et FoundationDuis@et.co.uk
Quinlan965-2125 Elit, Rd.Cochrane-24.07967, -20.55343Quis InstituteVivamus.nisi@atlacus.org
Stuart433-7718 Vitae, AvenueInverness19.04821, 86.06715Mollis Phasellus PCamet@metussitamet.edu
AhmedAp #291-2209 Sit StreetKitimat-21.10603, 128.90385Massa Mauris Vestibulum Corporationhendrerit.a@convallisconvallis.co.uk
Alec2283 Purus. St.Lugo89.32731, -48.75947At Arcu Limitedvelit.dui@necante.edu
OttoAp #143-4216 A Av.Socchieve-44.82965, -11.13392Taciti Sociosqu Ad Inc.sed@fermentumfermentumarcu.co.uk
ChristianP.O. Box 985, 8371 Fusce Av.Hope-23.94499, -164.94965Morbi Limitedtortor.dictum.eu@montesnasceturridiculus.org
DylanP.O. Box 823, 8721 Nulla Av.Le Havre6.93024, 32.90061Sapien Cras Corp.et.commodo.at@consectetuercursus.edu
Harrison441-4884 Aenean Rd.Burlington-81.06878, -173.84985Aenean Associatesnatoque.penatibus.et@cursusdiam.edu
Lucas523-532 Lorem StreetHexham5.87827, 126.2566Sed Tortor Foundationtristique.aliquet@Etiamvestibulum.org
JordanP.O. Box 198, 3987 Dui, St.Eindhoven76.61261, 40.39837Ad Litora Companynec.imperdiet@odioauctor.ca
Alan725-4387 Cubilia AvenueTrier29.88243, 44.98172Dolor LLPdui.lectus.rutrum@ipsum.ca
Damian183-4709 Sed, RoadVillers-la-Tour-42.32656, 62.54334Non Ante Foundationmauris.ut.mi@Duisacarcu.co.uk
Tate2461 Duis Rd.Pike Creek51.44993, 2.47053Orci LimitedVestibulum@velitegestas.co.uk
Dante866-6242 Luctus RoadAisemont-19.81444, 57.91328Facilisis Eget Incorporatedpulvinar@egetlaoreetposuere.net
Vaughan5784 Pede. St.Melilla-8.39545, -10.49761Cursus Instituteest@Etiam.com
Wang721-3768 Quisque St.Molino dei Torti0.80125, 77.84189Feugiat Instituteeu.enim@orcilacusvestibulum.ca
Guy109-6486 Malesuada AveBrecon-76.63174, 25.39433Etiam Instituteadipiscing.ligula.Aenean@aultricies.org
MarvinAp #775-7562 Arcu. St.Merchtem15.38707, 13.48797Aliquam Companylectus.Cum.sociis@parturientmontes.org
IgnatiusP.O. Box 304, 1051 Id St.Girona65.69551, -16.82965Per Inceptos Hymenaeos Corp.enim.Nunc@mollisnec.co.uk
Steven3904 Metus. St.Springdale-45.4299, 44.83996A Tortor Nunc Ltdnunc.nulla.vulputate@Duisdignissim.org
ToddAp #755-7349 Dui. Rd.Sobral78.41465, 94.18776Sed Pharetra Felis Limitedleo@ut.org
Damian2580 Consequat StreetPanketal-28.92551, 51.02208Enim Corp.tortor.dictum.eu@tellusPhaselluselit.edu
AlecAp #725-1303 Augue Rd.Ramara-77.02162, 131.07978Lectus Quis Companygravida@libero.com
KamalAp #912-6265 Parturient AveZiano di Fiemme-28.75265, -7.01129In Nec Orci Industriesarcu@acurnaUt.ca
KeatonP.O. Box 365, 7063 Tincidunt St.Smoky Lake61.46388, -175.89871Nunc Corp.ornare@lacus.edu
ReeceAp #433-9606 Ridiculus St.Maaseik-56.52698, 132.6406Nullam CompanySed@Suspendisse.org
KevinAp #115-5267 Libero StreetWolvertem66.66246, -32.46632Turpis Inc.Donec.sollicitudin@non.net
ZacheryP.O. Box 997, 2668 Amet AvenueUcluelet-19.15975, 118.68758Nibh PCMauris@nequeNullamut.co.uk
Maxwell1501 Justo. RoadLanark County44.14049, 7.57982Lectus Quis LLPdis.parturient@utdolordapibus.net
KevinP.O. Box 929, 4954 Risus. RoadNelson81.08858, -66.26228Mauris Ltdipsum@vitaesemper.net
Fitzgerald6400 Aenean StreetKessenich47.85688, 126.79015Nunc Corp.In.ornare@nibhAliquamornare.org
BakerP.O. Box 330, 2672 In St.Castel Volturno61.90888, 128.88712Congue Incorporatedmollis@interdumlibero.net
Quamar7444 Nunc StreetPiegaro16.34312, 22.36637Pede Consultingac@faucibusorci.org
Barry1891 Adipiscing AvenueVergnies-8.41602, 104.04352Purus LLPin@odiotristiquepharetra.edu
EmmanuelP.O. Box 445, 5233 Dui. StreetHamilton-33.25641, -75.3938Metus FoundationAliquam.ultrices.iaculis@velarcuCurabitur.net
Rooney5453 Commodo AveAylmer79.36747, -111.07236Nisl Elementum Purus PCenim.diam.vel@scelerisquescelerisquedui.com
Thaddeus7267 Felis RoadCappelle sul Tavo47.32545, -3.65071Ante Ipsum Inc.lorem.ac@quistristiqueac.org
RaymondAp #718-3769 Sed Rd.Tauranga-28.88573, -118.15865In Faucibus Companyturpis.egestas@risusaultricies.ca
BarclayAp #183-3638 Et Rd.Appelterre-Eichem18.0446, -69.26686Augue Limiteddiam.luctus@vehiculaetrutrum.ca
GriffithP.O. Box 157, 1837 Nunc Rd.Calice al Cornoviglio-65.22249, -133.28451Sit Amet LLPac.fermentum@condimentum.co.uk
Vernon553-6292 Nunc Rd.Milwaukee81.64991, 67.30965Risus At Fringilla Inc.lobortis.quam@nasceturridiculus.co.uk
NasimP.O. Box 365, 6330 Nunc Rd.Paupisi19.7798, -52.08825Magnis Dis CorporationCras.pellentesque.Sed@Cras.org
Tad836-3697 Diam Av.Airdrie87.74627, -74.20844Venenatis Corporationvestibulum@insodales.org
Upton323-9704 Vitae, AveArendonk57.89839, -86.62495Ullamcorper PCfeugiat@ullamcorperviverraMaecenas.org
DonovanP.O. Box 732, 3757 Libero Rd.Uberaba31.91303, -129.19524Scelerisque Dui Consultingconvallis.dolor.Quisque@arcu.net
Leo316-6835 Consequat RoadLang-64.65703, 55.18568Maecenas Incorporatedtellus.Phasellus.elit@lectusCumsociis.co.uk
Solomon496-903 Molestie RoadCirencester-43.72767, 62.50631Sem Elit Pharetra AssociatesSuspendisse.tristique.neque@eulacusQuisque.co.uk
CarterAp #815-6626 Tellus. RoadWanaka-44.85397, 174.41126Quam Companyimperdiet.nec@torquentper.ca
FerdinandAp #937-3634 Cum AvenueSesto al Reghena-43.8973, -9.9513Vel Quam Dignissim Incorporatedluctus.felis@aliquetmolestietellus.com
CadmanAp #182-1522 Sagittis RoadBolzano Vicentino23.99758, 136.69107Nullam Inc.mauris@consequat.com
-------------------------------------------------------------------------------- /pandas/example_data/generated2.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/generated2.xls -------------------------------------------------------------------------------- /pandas/example_data/matlab_variable.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/matlab_variable.mat -------------------------------------------------------------------------------- /pandas/example_data/p01_d3.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/p01_d3.csv.gz -------------------------------------------------------------------------------- /pandas/example_data/p01_prices.txt: -------------------------------------------------------------------------------- 1 | Date,AAPL,GOOG,JNJ,XOM 2 | 2018-01-26,171.509995,1175.839966,145.330002,89.0 3 | 2018-01-29,167.96000700000002,1175.579956,143.679993,88.010002 4 | 2018-01-30,166.970001,1163.689941,142.429993,86.779999 5 | 2018-01-31,167.429993,1169.939941,138.190002,87.300003 6 | 2018-02-01,167.779999,1167.699951,140.020004,89.07 7 | 2018-02-02,160.5,1111.900024,137.679993,84.529999 8 | -------------------------------------------------------------------------------- /pandas/example_data/p01_volumes.txt: -------------------------------------------------------------------------------- 1 | Date,AAPL,JNJ,XOM 2 | 2018-01-26,39143000,8065600,10514400 3 | 2018-01-29,50640400,8013400,9315700 4 | 2018-01-30,46048200,8376100,12408000 5 | 2018-01-31,32478900,13137400,14404700 6 | -------------------------------------------------------------------------------- /pandas/example_data/p05_d2.txt: -------------------------------------------------------------------------------- 1 | ,City,Name,Revenues,State 2 | 0,New York,Roy,1250, NY 3 | 1, frisco,Johnn,840,CA 4 | 2,houston,Jim,349, tx 5 | 3, taft,Paul,1100, OK 6 | 4,venice,Ross,900, IL 7 | -------------------------------------------------------------------------------- /pandas/example_data/pd06_pivot_Example.txt: -------------------------------------------------------------------------------- 1 | ,date,variable,value 2 | 0,2000-01-03 00:00:00,A,0.3970180055787111 3 | 1,2000-01-04 00:00:00,A,0.6211088243433798 4 | 2,2000-01-05 00:00:00,A,-2.5082841002429745 5 | 3,2000-01-03 00:00:00,B,0.21979588677202364 6 | 4,2000-01-04 00:00:00,B,1.3710270580007788 7 | 5,2000-01-05 00:00:00,B,-1.0055042898048039 8 | 6,2000-01-03 00:00:00,C,-0.39114867992720065 9 | 7,2000-01-04 00:00:00,C,1.209345749759747 10 | 8,2000-01-05 00:00:00,C,-0.05961654675455294 11 | 9,2000-01-03 00:00:00,D,-0.4645580899791357 12 | 10,2000-01-04 00:00:00,D,-0.1736052396445405 13 | 11,2000-01-05 00:00:00,D,-1.6496321207508482 14 | -------------------------------------------------------------------------------- /pandas/example_data/pd12_peopl.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/example_data/pd12_peopl.csv.gz -------------------------------------------------------------------------------- /pandas/index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "* [__pandas Dataframe - Basic Operativity__](pd01v01_basic_data_operativity.ipynb)\n", 8 | " * [1 File I/O and DataFrame Generation](pd01v01_basic_data_operativity.ipynb#1-File-I/O-and-DataFrame-Generation)\n", 9 | " * [1.1 Create DataFrames with read_csv](pd01v01_basic_data_operativity.ipynb#1.1-Create-DataFrames-with-read_csv)\n", 10 | " * [1.2 Create DataFrames from Python Dictionaries](pd01v01_basic_data_operativity.ipynb#1.2-Create-DataFrames-from-Python-Dictionaries)\n", 11 | " * [1.3 Create DataFrames from Items](pd01v01_basic_data_operativity.ipynb#1.3-Create-DataFrames-from-Items)\n", 12 | " * [1.4 Create DataFrames fron Numpy Arrays](pd01v01_basic_data_operativity.ipynb#1.4-Create-DataFrames-fron-Numpy-Arrays)\n", 13 | " * [1.5 DataFrames can be converted in Numpy Arrays](pd01v01_basic_data_operativity.ipynb#1.5-DataFrames-can-be-converted-in-Numpy-Arrays)\n", 14 | " * [1.6 DataFrames, Series and Panels](pd01v01_basic_data_operativity.ipynb#1.6-DataFrames,-Series-and-Panels)\n", 15 | " * [2 Automatic Data Alignment](pd01v01_basic_data_operativity.ipynb#2-Automatic-Data-Alignment)\n", 16 | " * [3 Indexing](pd01v01_basic_data_operativity.ipynb#3-Indexing)\n", 17 | " * [3.1 Label-Based Indexing](pd01v01_basic_data_operativity.ipynb#3.1-Label-Based-Indexing)\n", 18 | " * [3.2 Position-Based Indexing](pd01v01_basic_data_operativity.ipynb#3.2-Position-Based-Indexing)\n", 19 | " * [3.3 Advanced Indexing - .ix](pd01v01_basic_data_operativity.ipynb#3.3-Advanced-Indexing---.ix)\n", 20 | " * [4 DataFrame Basic Operations](pd01v01_basic_data_operativity.ipynb#4-DataFrame-Basic-Operations)\n", 21 | " * [4.1 Reindex/Reorder rows and columns](pd01v01_basic_data_operativity.ipynb#4.1-Reindex/Reorder-rows-and-columns)\n", 22 | " * [4.2 Calculate new columns](pd01v01_basic_data_operativity.ipynb#4.2-Calculate-new-columns)\n", 23 | " * [4.3 Deleting rows and columns](pd01v01_basic_data_operativity.ipynb#4.3-Deleting-rows-and-columns)\n", 24 | " * [4.4 Inserting colums in a specific position](pd01v01_basic_data_operativity.ipynb#4.4-Inserting-colums-in-a-specific-position)\n", 25 | " * [4.5 Check if a value or a list of given values are contained in a specific column](pd01v01_basic_data_operativity.ipynb#4.5-Check-if-a-value-or-a-list-of-given-values-are-contained-in-a-specific-column)\n", 26 | " * [4.6 Rename columns](pd01v01_basic_data_operativity.ipynb#4.6-Rename-columns)\n", 27 | " * [4.7 Iterate efficiently through rows](pd01v01_basic_data_operativity.ipynb#4.7-Iterate-efficiently-through-rows)\n", 28 | " * [5 Duplicated Data](pd01v01_basic_data_operativity.ipynb#5-Duplicated-Data)\n", 29 | " * [5.1 Find duplicated data in columns](pd01v01_basic_data_operativity.ipynb#5.1-Find-duplicated-data-in-columns)\n", 30 | " * [5.2 Remove Duplicates](pd01v01_basic_data_operativity.ipynb#5.2-Remove-Duplicates)\n", 31 | " * [6 Working with Large Arrays](pd01v01_basic_data_operativity.ipynb#6-Working-with-Large-Arrays)\n", 32 | " * [6.1 Control the DataFrame memory occupation](pd01v01_basic_data_operativity.ipynb#6.1-Control-the-DataFrame-memory-occupation)\n", 33 | " * [6.2 Explore large arrays](pd01v01_basic_data_operativity.ipynb#6.2-Explore-large-arrays)\n", 34 | " * [7 Column pct_change and shift](pd01v01_basic_data_operativity.ipynb#7-Column-pct_change-and-shift)\n", 35 | " * [8 Reindex](pd01v01_basic_data_operativity.ipynb#8-Reindex)\n", 36 | " * [9 More on Indexing: Multi Index](pd01v01_basic_data_operativity.ipynb#9-More-on-Indexing:-Multi-Index)\n", 37 | " * [10 Package Options](pd01v01_basic_data_operativity.ipynb#10-Package-Options)\n", 38 | "* [__pandas I/O tools and examples__](pd02v01_input_output.ipynb)\n", 39 | " * [1 Matlab Variables](pd02v01_input_output.ipynb#1-Matlab-Variables)\n", 40 | " * [1.1 Import a Matlab variable from file](pd02v01_input_output.ipynb#1.1-Import-a-Matlab-variable-from-file)\n", 41 | " * [2 Importing a compressed CSV](pd02v01_input_output.ipynb#2-Importing-a-compressed-CSV)\n", 42 | " * [3 Importing and visualizing geographical data](pd02v01_input_output.ipynb#3-Importing-and-visualizing-geographical-data)\n", 43 | " * [4 Importing JSON files](pd02v01_input_output.ipynb#4-Importing-JSON-files)\n", 44 | " * [5 Importing HTML](pd02v01_input_output.ipynb#5-Importing-HTML)\n", 45 | " * [6 Importing Excel](pd02v01_input_output.ipynb#6-Importing-Excel)\n", 46 | " * [7 Working with SQL and databases](pd02v01_input_output.ipynb#7-Working-with-SQL-and-databases)\n", 47 | " * [7.1 Write SQL](pd02v01_input_output.ipynb#7.1-Write-SQL)\n", 48 | " * [7.2 Import SQL](pd02v01_input_output.ipynb#7.2-Import-SQL)\n", 49 | " * [8 Working with HDF5](pd02v01_input_output.ipynb#8-Working-with-HDF5)\n", 50 | " * [8.1 Storer format](pd02v01_input_output.ipynb#8.1-Storer-format)\n", 51 | " * [8.2 Table format](pd02v01_input_output.ipynb#8.2-Table-format)\n", 52 | " * [8.3 Querying a Table](pd02v01_input_output.ipynb#8.3-Querying-a-Table)\n", 53 | "* [__Pandas Time series__](pd03v01_time_series.ipynb)\n", 54 | " * [1 Timestamps and DatetimeIndex](pd03v01_time_series.ipynb#1-Timestamps-and-DatetimeIndex)\n", 55 | " * [2 DateOffsets objects](pd03v01_time_series.ipynb#2-DateOffsets-objects)\n", 56 | " * [3 Indexing with a DateTime index](pd03v01_time_series.ipynb#3-Indexing-with-a-DateTime-index)\n", 57 | " * [4 Frequency conversion](pd03v01_time_series.ipynb#4-Frequency-conversion)\n", 58 | " * [5 Filling gaps](pd03v01_time_series.ipynb#5-Filling-gaps)\n", 59 | "* [__Statistical tools__](pd04v01_statistical_tools.ipynb)\n", 60 | " * [1 Percent change](pd04v01_statistical_tools.ipynb#1-Percent-change)\n", 61 | " * [2 Covariance](pd04v01_statistical_tools.ipynb#2-Covariance)\n", 62 | " * [3 Correlation](pd04v01_statistical_tools.ipynb#3-Correlation)\n", 63 | " * [4 Rolling moments and Binary rolling moments](pd04v01_statistical_tools.ipynb#4-Rolling-moments-and-Binary-rolling-moments)\n", 64 | " * [5 A pratical example: Return indexes and cumulative returns](pd04v01_statistical_tools.ipynb#5-A-pratical-example:-Return-indexes-and-cumulative-returns)\n", 65 | "* [__Merge and pivot__](pd05v01_data_organization.ipynb)\n", 66 | " * [1 Concat](pd05v01_data_organization.ipynb#1-Concat)\n", 67 | " * [2 Append](pd05v01_data_organization.ipynb#2-Append)\n", 68 | " * [3 Join](pd05v01_data_organization.ipynb#3-Join)\n", 69 | " * [4 Merge](pd05v01_data_organization.ipynb#4-Merge)\n", 70 | " * [5 Pivoting](pd05v01_data_organization.ipynb#5-Pivoting)\n", 71 | " * [6 Stack and Unstack](pd05v01_data_organization.ipynb#6-Stack-and-Unstack)\n", 72 | "* [__Split apply and combine__](pd06v01_advanced_data_management.ipynb)\n", 73 | " * [1 Groupby](pd06v01_advanced_data_management.ipynb#1-Groupby)\n", 74 | " * [2 Aggregate](pd06v01_advanced_data_management.ipynb#2-Aggregate)\n", 75 | " * [3 Apply](pd06v01_advanced_data_management.ipynb#3-Apply)\n", 76 | " * [4 A pratical example: Normalize by year](pd06v01_advanced_data_management.ipynb#4-A-pratical-example:-Normalize-by-year)\n", 77 | " * [5 A practical example: Group and standardize by dimension](pd06v01_advanced_data_management.ipynb#5-A-practical-example:-Group-and-standardize-by-dimension)\n", 78 | "* [__Sources of Open Data__](pd07v01_open_data.ipynb)\n", 79 | " * [1 Yahoo! Finance](pd07v01_open_data.ipynb#1-Yahoo!-Finance)\n", 80 | " * [1.1 Plotting timeseries with bokeh:](pd07v01_open_data.ipynb#1.1-Plotting-timeseries-with-bokeh:)\n", 81 | " * [1.2 Plotting candlesticks with bokeh:](pd07v01_open_data.ipynb#1.2-Plotting-candlesticks-with-bokeh:)\n", 82 | " * [1.3 Plotting data ranges with bokeh:](pd07v01_open_data.ipynb#1.3-Plotting-data-ranges-with-bokeh:)\n", 83 | " * [1.4 Plotting multiple plots with matplotlib:](pd07v01_open_data.ipynb#1.4-Plotting-multiple-plots-with-matplotlib:)\n", 84 | " * [2 Google Finance](pd07v01_open_data.ipynb#2-Google-Finance)\n", 85 | " * [3 Federal Reserve Economic Data](pd07v01_open_data.ipynb#3-Federal-Reserve-Economic-Data)\n", 86 | " * [4 World Bank](pd07v01_open_data.ipynb#4-World-Bank)\n", 87 | "* [__Baby Names__](pd08v01_babynames.ipynb)\n", 88 | " * [1 Load and prepare the data](pd08v01_babynames.ipynb#1-Load-and-prepare-the-data)\n", 89 | " * [2 Pivoting](pd08v01_babynames.ipynb#2-Pivoting)\n", 90 | " * [3 Splitting](pd08v01_babynames.ipynb#3-Splitting)\n", 91 | " * [4 Using 'groupby'](pd08v01_babynames.ipynb#4-Using-'groupby')\n" 92 | ] 93 | } 94 | ], 95 | "metadata": {}, 96 | "nbformat": 4, 97 | "nbformat_minor": 0 98 | } -------------------------------------------------------------------------------- /pandas/temp/temporary.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/temp/temporary.txt -------------------------------------------------------------------------------- /pandas/utilities/Thumbs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/utilities/Thumbs.db -------------------------------------------------------------------------------- /pandas/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | from .tom import TomTom # (TomTom, FileGenerator) 2 | from .my_finance_tools import (side_by_side, csv_preview) 3 | from .my_finance_plots import montecarloPlot 4 | -------------------------------------------------------------------------------- /pandas/utilities/generators.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from pandas_datareader.data import DataReader 3 | from pandas_datareader._utils import RemoteDataError 4 | import numpy as np 5 | from .tom import TomTom 6 | import os 7 | import shutil 8 | 9 | 10 | def p01_d2csv(tomtom): 11 | d = {'a' : pd.Series(['one','one','two','three','two']), 12 | 'b' : pd.Series(['x','y','y','x','y']), 13 | 'c' : pd.Series(np.random.randn(5))} 14 | d2 = pd.DataFrame(d) 15 | d2.to_csv(tomtom.get_tmp_name('p01_d2.csv')) 16 | 17 | def p01_d3csv(tomtom): 18 | comuni = pd.read_csv(tomtom.get_example_name('tabella_comuni_italiani.txt'), 19 | sep=';', header=0) 20 | # d3 = pd.DataFrame(np.random.randn(1000, 100), columns=comuni['Comune'].ix[0:199]) 21 | comuni.to_csv(tomtom.get_tmp_name('p01_d3.csv'), index=False) 22 | 23 | def p01_d4csv(tomtom): 24 | idx = [('Fra', 'one', 'x'), 25 | ('Fra', 'two', 'y'), 26 | ('Fra', 'two', 'z'), 27 | ('Ger', 'one', 'x'), 28 | ('Jap', 'one', 'x'), 29 | ('Jap', 'two', 'x'), 30 | ('USA', 'one', 'y'), 31 | ('USA', 'one', 'z')] 32 | index = pd.MultiIndex.from_tuples(idx, names=['Country', 'Number', 'Dir']) 33 | d4 = pd.DataFrame(np.random.randn(8,3), index=index) 34 | d4.to_csv(tomtom.get_tmp_name('p01_d4.csv')) 35 | 36 | def p01_prices(tomtom): 37 | try: 38 | symbols = ['AAPL', 'JNJ', 'XOM', 'GOOG'] 39 | data = dict([(sym, DataReader(sym, 'yahoo')['Close']) for sym in symbols]) 40 | df = pd.DataFrame.from_dict(data) 41 | df.ix[-7:-1].to_csv(tomtom.get_tmp_name('p01_prices.txt')) 42 | except RemoteDataError: 43 | print('Error while reading data, revert to stored file in example_data') 44 | shutil.copy('example_data/p01_prices.txt', 'temp') 45 | 46 | def p01_volumes(tomtom): 47 | try: 48 | symbols = ['AAPL', 'JNJ', 'XOM'] 49 | data = dict([(sym, DataReader(sym, 'yahoo')['Volume']) for sym in symbols]) 50 | df = pd.DataFrame.from_dict(data) 51 | df.ix[-7:-3].to_csv(tomtom.get_tmp_name('p01_volumes.txt')) 52 | except RemoteDataError: 53 | print('Error while reading data, revert to stored file in example_data') 54 | shutil.copy('example_data/p01_volumes.txt', 'temp') 55 | 56 | def p03_DAX(tomtom): 57 | try: 58 | DAX = DataReader('^GDAXI','yahoo',start = '01/01/2000') 59 | DAX.to_csv(tomtom.get_tmp_name('p03_DAX.csv')) 60 | except RemoteDataError: 61 | print('Error while reading data, revert to stored file in example_data') 62 | shutil.copy('example_data/p03_DAX.csv', 'temp') 63 | 64 | def p03_AAPL(tomtom): 65 | try: 66 | DAX = DataReader('AAPL','yahoo',start = '01/01/2000') 67 | DAX.to_csv(tomtom.get_tmp_name('p03_AAPL.csv')) 68 | except RemoteDataError: 69 | print('Error while reading data, revert to stored file in example_data') 70 | shutil.copy('example_data/p03_AAPL.csv', 'temp') 71 | 72 | def p06_d3csv(tomtom): 73 | d2 = pd.DataFrame({'City' : ['New York', ' frisco', 'houston', ' taft', 'venice'], 74 | 'State' : [' NY ', 'CA', ' tx ', ' OK', ' IL'], 75 | 'Name' : ['Roy', 'Johnn', 'Jim', 'Paul', 'Ross'], 76 | 'Revenues' : ['1250', '840', '349', '1100', '900']}) 77 | d2.to_csv(tomtom.get_tmp_name('p06_d2.txt')) 78 | 79 | def p06_d2csv(tomtom): 80 | d3 = pd.DataFrame({'Quantity' : ['1-one', '1-one', '2-two', '3-three'] * 6, 81 | 'Axis' : ['X', 'Y', 'Z'] * 8, 82 | 'Type' : ['foo', 'foo', 'foo', 'bar', 'bar', 'bar'] * 4, 83 | 'N1' : np.random.randn(24), 84 | 'N2' : np.random.randn(24)}) 85 | d3.to_csv(tomtom.get_tmp_name('p06_d3.txt')) 86 | 87 | 88 | def p07_d1csv(tomtom): 89 | d1 = pd.DataFrame({'State' : ['NE','KY','CO','CO','KY','KY','CO','NE','CO'], 90 | 'City' : ['Page','Stone','Rye','Rye','Dema','Keavy','Rye', 91 | 'Cairo', 'Dumont'], 92 | 'Views' : [10, 9, 3, 7, 4, 2, 1, 8, 12], 93 | 'Likes' : [4, 3, 0, 2, 1, 1, 0, 3, 7]}) 94 | d1.to_csv(tomtom.get_tmp_name('p07_d1.txt')) 95 | 96 | def p07_d2csv(tomtom): 97 | import random; random.seed(0) 98 | import string 99 | N = 1000 100 | def rands(n): 101 | choices = string.ascii_uppercase 102 | return ''.join([random.choice(choices) for _ in range(n)]) 103 | 104 | tickers = np.array([rands(5) for _ in range(N)]) 105 | 106 | # Create a DataFrame containing 3 columns representing 107 | # hypothetical, but random portfolios for a subset of tickers: 108 | d2 = pd.DataFrame({'Momentum' : np.random.randn(500) / 200 + 0.03, 109 | 'Value' : np.random.randn(500) / 200 + 0.08, 110 | 'ShortInterest' : np.random.randn(500) / 200 - 0.02}, 111 | index=tickers.take(np.random.permutation(N)[:500])) 112 | 113 | # Next, let's create a random industry classification for the tickers. 114 | ind_names = np.array(['FINANCIAL', 'TECH']) 115 | sampler = np.random.randint(0, len(ind_names), N) 116 | industries = pd.Series(ind_names.take(sampler), index=tickers, name='industry') 117 | d2['Industry'] = industries 118 | 119 | d2.to_csv(tomtom.get_tmp_name('p07_d2.csv')) 120 | 121 | def p07_portfolioh5(tomtom): 122 | import random; random.seed(0) 123 | import string 124 | N = 1000 125 | def rands(n): 126 | choices = string.ascii_uppercase 127 | return ''.join([random.choice(choices) for _ in range(n)]) 128 | 129 | tickers = np.array([rands(5) for _ in range(N)]) 130 | fac1, fac2, fac3 = np.random.rand(3, 1000) 131 | ticker_subset = tickers.take(np.random.permutation(N)[:1000]) 132 | 133 | # portfolio = weighted sum of factors plus noise 134 | portfolio = pd.Series(0.7 * fac1 - 1.2 * fac2 + 0.3 * fac3 + np.random.rand(1000), 135 | index=ticker_subset) 136 | factors = pd.DataFrame({'f1': fac1, 'f2': fac2, 'f3': fac3}, 137 | index=ticker_subset) 138 | 139 | h5file = pd.HDFStore(tomtom.get_tmp_name('p07_portfolio.h5')) 140 | h5file['factors'] = factors 141 | h5file['portfolio'] = portfolio 142 | h5file.close() 143 | 144 | def baby_names(tomtom): 145 | import zipfile 146 | path = tomtom.get_example_name('babynames.zip') 147 | opath = tomtom.get_tmp_name("") 148 | z = zipfile.ZipFile(path, "r") 149 | z.extractall(path=opath) 150 | 151 | generators = { 152 | 'baby_names/': baby_names, 153 | 'p07_portfolio.h5': p07_portfolioh5, 154 | 'p07_d2.csv': p07_d2csv, 155 | 'p07_d1.txt': p07_d1csv, 156 | 'p06_d3.txt': p06_d3csv, 157 | 'p06_d2.txt': p06_d2csv, 158 | 'p03_DAX.csv': p03_DAX, 159 | 'p03_AAPL.csv': p03_AAPL, 160 | 'p01_prices.txt': p01_prices, 161 | 'p01_d2.csv': p01_d2csv, 162 | 'p01_d3.csv': p01_d3csv, 163 | 'p01_d4.csv': p01_d4csv, 164 | 'p01_volumes.txt': p01_volumes, 165 | } 166 | 167 | def generate_all(): 168 | tomtom = TomTom() 169 | for filename, gen in generators.items(): 170 | path = tomtom.get_tmp_name(filename) 171 | if not os.path.exists(path): 172 | print("Generating {}...".format(filename)) 173 | gen(tomtom) 174 | else: 175 | print("Skipped {} (already existing)".format(filename)) 176 | 177 | if __name__ == '__main__': 178 | generate_all() 179 | -------------------------------------------------------------------------------- /pandas/utilities/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/pandas/utilities/logo.png -------------------------------------------------------------------------------- /pandas/utilities/my_finance_plots.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pylab import * 3 | from pandas import * 4 | #from pandas.io.data import * 5 | 6 | def montecarloPlot(S): 7 | cols, rows = S.shape 8 | a = arange(0,cols) 9 | x = append(a,a[::-1]) 10 | 11 | figure(figsize=(10, 8), dpi=80) 12 | gs = plt.GridSpec(4, 4) 13 | ax1 = plt.subplot(gs[:, 0:3]) 14 | 15 | cmap = cm.jet 16 | for i in [0, 1, 5]: 17 | lower_bound = np.percentile(S, i, axis=1) 18 | upper_bound = np.percentile(S, 100-i, axis=1) 19 | y= append(lower_bound,upper_bound[::-1]) 20 | fill(x,y,'r',facecolor=cmap(i*255/50), alpha=0.3+i/100.) 21 | 22 | ax2 = plt.subplot(gs[:, 3]) 23 | 24 | hist( S.ravel(), bins= 101, orientation='horizontal' ) 25 | ax2.xaxis.set_major_locator(MaxNLocator(4)) 26 | xlabels = ax2.get_xticklabels() 27 | for label in xlabels: 28 | label.set_rotation(45) 29 | 30 | setp(ax2.get_xticklabels(), visible=False) 31 | setp(ax2.get_yticklabels(), visible=False); grid(True); show() 32 | -------------------------------------------------------------------------------- /pandas/utilities/my_finance_tools.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import matplotlib.finance as fin 3 | import pandas as pd 4 | from pandas import Index, DataFrame 5 | # from pandas.core.datetools import BMonthEnd 6 | 7 | # MY FINANCE TOOLS ------------------------------------------------------------- 8 | ''' Created by: Addfor S.p.A. 9 | This module provides few example finance and utility functions 10 | ''' 11 | 12 | def csv_preview(filename, lines_to_print=5): 13 | ''' 14 | TODO - Add a control to define how many columns to print: 15 | start_column = 0 16 | end_column = 79 17 | ''' 18 | with open(filename) as fid: 19 | for _ in range(lines_to_print): 20 | line = fid.readline() 21 | print(line,) 22 | 23 | def side_by_side(*objs, **kwds): 24 | space = kwds.get('space', 4) 25 | reprs = [repr(obj).split('\n') for obj in objs] 26 | print('-'*40) 27 | print(pd.core.common.adjoin(space, *reprs)) 28 | print('-'*40) 29 | 30 | def getQuotes(symbol, start, end): 31 | '''getQuotes documentation''' 32 | quotes = fin.quotes_historical_yahoo(symbol, start, end) 33 | dates, opn, close, high, low, volume = zip(*quotes) 34 | data = {'open': opn, 'close': close, 'high' : high, 35 | 'low' : low, 'volume': volume} 36 | 37 | dates = Index([datetime.fromordinal(int(d)) for d in dates]) 38 | return DataFrame(data, index=dates) 39 | 40 | 41 | # def toMonthly(frame, how): 42 | # '''toMonthly documentation''' 43 | # offset = BMonthEnd() 44 | # return frame.groupby(offset.rollforward).aggregate(how) 45 | -------------------------------------------------------------------------------- /pandas/utilities/tom.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | TEMP_DIR="temp" 4 | EXAMPLE_DIR="example_data" 5 | 6 | #from generators import generators 7 | 8 | class TomTom(object): 9 | """ 10 | This is an utility class that provides objects capable of telling where to save temporary files. 11 | It implements Borg design pattern. 12 | """ 13 | __shared__state = {} 14 | def __init__(self): 15 | self.__dict__ = self.__shared__state 16 | self.cwd = os.getcwd() 17 | self.sep = os.path.sep 18 | 19 | def get_tmp_name(self, name): 20 | """ 21 | Return an absolute path for a temporary (output) file named name. 22 | """ 23 | return self.cwd + self.sep + TEMP_DIR + self.sep + name 24 | 25 | def get_tmp_dir(self): 26 | return self.cwd + self.sep + TEMP_DIR 27 | 28 | def get_example_name(self, name): 29 | """ 30 | Return an absolute path for an example (input) file named name. 31 | """ 32 | return self.cwd + self.sep + EXAMPLE_DIR + self.sep + name 33 | 34 | 35 | 36 | # class FileGenerator(object): 37 | # __shared__state = {} 38 | # def __init__(self): 39 | # self.__dict__ = self.__shared__state 40 | # self._tom = TomTom() 41 | 42 | # def get_example(self, file): 43 | # path = self._tom.get_tmp_name(file) 44 | # if not os.path.exists(path): 45 | # generators[file](self._tom) 46 | 47 | # return path 48 | -------------------------------------------------------------------------------- /python-ipython/example_data/my_input.txt: -------------------------------------------------------------------------------- 1 | First Second 2 | 10 0.32432 3 | 20 1.324 4 | 21 7.237923 5 | 36 .83298932 6 | 56 237.327823 7 | -------------------------------------------------------------------------------- /python-ipython/example_data/small_is_beautiful.txt: -------------------------------------------------------------------------------- 1 | No system or machinery or economic doctrine or theory stands on its own feet: it is invariably built on a metaphysical foundation, that is to say, upon man's basic outlook on life, its meaning and its purpose. I have talked about the religion of economics, the idol worship of material possessions, of consumption and the so-called standard of living, and the fateful propensity that rejoices in the fact that 'what were luxuries to our fathers have become necessities for us.' 2 | Systems are never more no less than incarnations of man's most basic attitudes. . . . General evidence of material progress would suggest that the modern private enterprise system is--or has been--the most perfect instrument for the pursuit of personal enrichment. The modern private enterprise system ingeniously employs the human urges of greed and envy as its motive power, but manages to overcome the most blatant deficiencies of laissez-faire by means of Keynesian economic management, a bit of redistributive taxation, and the 'countervailing power' of the trade unions. 3 | Can such a system conceivably deal with the problems we are now having to face? The answer is self-evident: greed and envy demand continuous and limitless economic growth of a material kind, without proper regard for conservation, and this type of growth cannot possibly fit into a finite environment. We must therefore study the essential nature of the private enterprise system and the possibilities of evolving an alternative system which might fit the new situation. 4 | -------------------------------------------------------------------------------- /python-ipython/images/integration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/images/integration.png -------------------------------------------------------------------------------- /python-ipython/index.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "* [__An introduction to the IPython notebook__](py01v01_ipython_notebook_introduction.ipynb)\n", 8 | " * [1 Before starting...](py01v01_ipython_notebook_introduction.ipynb#1-Before-starting...)\n", 9 | " * [2 User interface](py01v01_ipython_notebook_introduction.ipynb#2-User-interface)\n", 10 | " * [2.1 Main editing area](py01v01_ipython_notebook_introduction.ipynb#2.1-Main-editing-area)\n", 11 | " * [2.2 Menu](py01v01_ipython_notebook_introduction.ipynb#2.2-Menu)\n", 12 | " * [2.3 Header bar](py01v01_ipython_notebook_introduction.ipynb#2.3-Header-bar)\n", 13 | " * [2.4 The pager at the bottom](py01v01_ipython_notebook_introduction.ipynb#2.4-The-pager-at-the-bottom)\n", 14 | " * [3 Tab completion and tooltips](py01v01_ipython_notebook_introduction.ipynb#3-Tab-completion-and-tooltips)\n", 15 | " * [4 Plotting support](py01v01_ipython_notebook_introduction.ipynb#4-Plotting-support)\n", 16 | " * [5 Interactive Widgets](py01v01_ipython_notebook_introduction.ipynb#5-Interactive-Widgets)\n", 17 | " * [5.1 Plot controlled by Widgets](py01v01_ipython_notebook_introduction.ipynb#5.1-Plot-controlled-by-Widgets)\n", 18 | " * [5.2 Factorization Example using Interactive Widgets and SimPy](py01v01_ipython_notebook_introduction.ipynb#5.2-Factorization-Example-using-Interactive-Widgets-and-SimPy)\n", 19 | " * [6 Markdown cells can contain formatted text and code](py01v01_ipython_notebook_introduction.ipynb#6-Markdown-cells-can-contain-formatted-text-and-code)\n", 20 | " * [7 Rich display: include anything a browser can show](py01v01_ipython_notebook_introduction.ipynb#7-Rich-display:-include-anything-a-browser-can-show)\n", 21 | " * [7.1 Videos](py01v01_ipython_notebook_introduction.ipynb#7.1-Videos)\n", 22 | " * [7.2 Cloud Data Analysis and Graphing Tools](py01v01_ipython_notebook_introduction.ipynb#7.2-Cloud-Data-Analysis-and-Graphing-Tools)\n", 23 | " * [7.3 External Websites](py01v01_ipython_notebook_introduction.ipynb#7.3-External-Websites)\n", 24 | " * [8 Loading external codes](py01v01_ipython_notebook_introduction.ipynb#8-Loading-external-codes)\n", 25 | "* [__Python Basic Concepts__](py02v01_python_basics.ipynb)\n", 26 | " * [1 Why Python?](py02v01_python_basics.ipynb#1-Why-Python?)\n", 27 | " * [2 Python101](py02v01_python_basics.ipynb#2-Python101)\n", 28 | " * [2.1 Two important language features:](py02v01_python_basics.ipynb#2.1-Two-important-language-features:)\n", 29 | " * [2.2 The bricks of Python are:](py02v01_python_basics.ipynb#2.2-The-bricks-of-Python-are:)\n", 30 | " * [2.3 Variables, as everything else, in Python are objects](py02v01_python_basics.ipynb#2.3-Variables,-as-everything-else,-in-Python-are-objects)\n", 31 | " * [2.4 Mutable / Immutable Objects](py02v01_python_basics.ipynb#2.4-Mutable-/-Immutable-Objects)\n", 32 | " * [2.5 Scripts, modules and namespaces](py02v01_python_basics.ipynb#2.5-Scripts,-modules-and-namespaces)\n", 33 | " * [3 Strings](py02v01_python_basics.ipynb#3-Strings)\n", 34 | " * [4 String formatting](py02v01_python_basics.ipynb#4-String-formatting)\n", 35 | " * [5 Lists](py02v01_python_basics.ipynb#5-Lists)\n", 36 | " * [6 Slicing](py02v01_python_basics.ipynb#6-Slicing)\n", 37 | " * [7 Sets](py02v01_python_basics.ipynb#7-Sets)\n", 38 | " * [8 Tuples](py02v01_python_basics.ipynb#8-Tuples)\n", 39 | " * [9 Dictionaries](py02v01_python_basics.ipynb#9-Dictionaries)\n", 40 | " * [10 Counters](py02v01_python_basics.ipynb#10-Counters)\n", 41 | " * [11 IF - FOR - WHILE](py02v01_python_basics.ipynb#11-IF---FOR---WHILE)\n", 42 | " * [11.1 IF](py02v01_python_basics.ipynb#11.1-IF)\n", 43 | " * [11.2 FOR - ELSE](py02v01_python_basics.ipynb#11.2-FOR---ELSE)\n", 44 | " * [11.3 WHILE](py02v01_python_basics.ipynb#11.3-WHILE)\n", 45 | "* [__Python Getting Started__](py03v01_python_getting_started.ipynb)\n", 46 | " * [1 Comment your code !](py03v01_python_getting_started.ipynb#1-Comment-your-code-!)\n", 47 | " * [2 Functions](py03v01_python_getting_started.ipynb#2-Functions)\n", 48 | " * [2.1 Local Functions](py03v01_python_getting_started.ipynb#2.1-Local-Functions)\n", 49 | " * [2.2 External Functions](py03v01_python_getting_started.ipynb#2.2-External-Functions)\n", 50 | " * [2.3 Private Methods](py03v01_python_getting_started.ipynb#2.3-Private-Methods)\n", 51 | " * [2.4 Anonymous Functions (lambda functions)](py03v01_python_getting_started.ipynb#2.4-Anonymous-Functions-(lambda-functions))\n", 52 | " * [3 File I/O](py03v01_python_getting_started.ipynb#3-File-I/O)\n", 53 | " * [3.1 Simple I/O](py03v01_python_getting_started.ipynb#3.1-Simple-I/O)\n", 54 | " * [3.2 Pickle / cPickle](py03v01_python_getting_started.ipynb#3.2-Pickle-/-cPickle)\n", 55 | " * [4 Operating System](py03v01_python_getting_started.ipynb#4-Operating-System)\n", 56 | " * [4.1 General Info](py03v01_python_getting_started.ipynb#4.1-General-Info)\n", 57 | " * [4.2 sys — System-specific parameters and functions](py03v01_python_getting_started.ipynb#4.2-sys-—-System-specific-parameters-and-functions)\n", 58 | " * [4.3 os — Miscellaneous operating system interfaces](py03v01_python_getting_started.ipynb#4.3-os-—-Miscellaneous-operating-system-interfaces)\n", 59 | " * [4.4 glob — Unix style pathname pattern expansion](py03v01_python_getting_started.ipynb#4.4-glob-—-Unix-style-pathname-pattern-expansion)\n", 60 | "* [__Python Style Guide__](py04v01_python_style_guide.ipynb)\n", 61 | " * [1 PEP8](py04v01_python_style_guide.ipynb#1-PEP8)\n", 62 | " * [2 Syntax and Naming Conventions](py04v01_python_style_guide.ipynb#2-Syntax-and-Naming-Conventions)\n", 63 | " * [2.1 Avoid using semicolons at the end of lines](py04v01_python_style_guide.ipynb#2.1-Avoid-using-semicolons-at-the-end-of-lines)\n", 64 | " * [2.2 Avoid having multiple statements on a single line](py04v01_python_style_guide.ipynb#2.2-Avoid-having-multiple-statements-on-a-single-line)\n", 65 | " * [2.3 Indent your code block with 4 spaces](py04v01_python_style_guide.ipynb#2.3-Indent-your-code-block-with-4-spaces)\n", 66 | " * [2.4 Imports: Should always be on separate lines, if refered to different modules.](py04v01_python_style_guide.ipynb#2.4-Imports:-Should-always-be-on-separate-lines,-if-refered-to-different-modules.)\n", 67 | " * [2.5 Whitespace: required or not?](py04v01_python_style_guide.ipynb#2.5-Whitespace:-required-or-not?)\n", 68 | " * [2.6 Naming: names in python should be chosen according to the following conventions:](py04v01_python_style_guide.ipynb#2.6-Naming:-names-in-python-should-be-chosen-according-to-the-following-conventions:)\n", 69 | " * [3 Working with Data](py04v01_python_style_guide.ipynb#3-Working-with-Data)\n", 70 | " * [3.1 Avoid using a temporary variable when swapping two variables](py04v01_python_style_guide.ipynb#3.1-Avoid-using-a-temporary-variable-when-swapping-two-variables)\n", 71 | " * [3.2 Use tuples to unpack data](py04v01_python_style_guide.ipynb#3.2-Use-tuples-to-unpack-data)\n", 72 | " * [3.3 Strings: Use join and list to create a string from a list and viceversa](py04v01_python_style_guide.ipynb#3.3-Strings:-Use-join-and-list-to-create-a-string-from-a-list-and-viceversa)\n", 73 | " * [3.4 Strings: Use startswith and endswith instead of string slicing to check for prefixes or suffixes.](py04v01_python_style_guide.ipynb#3.4-Strings:-Use-startswith-and-endswith-instead-of-string-slicing-to-check-for-prefixes-or-suffixes.)\n", 74 | " * [3.5 Dictionaries: Use the default parameter of dict.get to provide default values](py04v01_python_style_guide.ipynb#3.5-Dictionaries:-Use-the-default-parameter-of-dict.get-to-provide-default-values)\n", 75 | " * [3.6 File IO: Use Context Managers to ensure resources are properly managed](py04v01_python_style_guide.ipynb#3.6-File-IO:-Use-Context-Managers-to-ensure-resources-are-properly-managed)\n", 76 | " * [4 Control Structures](py04v01_python_style_guide.ipynb#4-Control-Structures)\n", 77 | " * [4.1 Don't compare boolean values to True or False using ==](py04v01_python_style_guide.ipynb#4.1-Don't-compare-boolean-values-to-True-or-False-using-==)\n", 78 | " * [4.2 Avoid placing conditional branch on the same line as the colon](py04v01_python_style_guide.ipynb#4.2-Avoid-placing-conditional-branch-on-the-same-line-as-the-colon)\n", 79 | " * [4.3 Comparisons to singletons like None should always be done with is or is not.](py04v01_python_style_guide.ipynb#4.3-Comparisons-to-singletons-like-None-should-always-be-done-with-is-or-is-not.)\n", 80 | " * [4.4 Avoid repeating variable name in compound if Statement](py04v01_python_style_guide.ipynb#4.4-Avoid-repeating-variable-name-in-compound-if-Statement)\n", 81 | " * [4.5 Use list comprehensions to create lists that are subsets of existing data](py04v01_python_style_guide.ipynb#4.5-Use-list-comprehensions-to-create-lists-that-are-subsets-of-existing-data)\n", 82 | " * [4.6 Indexes in for Loops](py04v01_python_style_guide.ipynb#4.6-Indexes-in-for-Loops)\n", 83 | " * [4.7 range` and `xrange`](py04v01_python_style_guide.ipynb#4.7-range`-and-`xrange`)\n", 84 | "* [__Python More Examples__](py05v01_python_more_examples.ipynb)\n", 85 | " * [1 Working with Win32Com - Excel example](py05v01_python_more_examples.ipynb#1-Working-with-Win32Com---Excel-example)\n", 86 | " * [2 Sending an e-mail from a gmail account](py05v01_python_more_examples.ipynb#2-Sending-an-e-mail-from-a-gmail-account)\n", 87 | " * [3 Automating google search](py05v01_python_more_examples.ipynb#3-Automating-google-search)\n", 88 | "* [__Object Oriented Python and Exceptions__](py06v01_python_object_oriented.ipynb)\n", 89 | " * [1 Understanding Objects](py06v01_python_object_oriented.ipynb#1-Understanding-Objects)\n", 90 | " * [1.1 Creating a class](py06v01_python_object_oriented.ipynb#1.1-Creating-a-class)\n", 91 | " * [1.2 Instantiating a class](py06v01_python_object_oriented.ipynb#1.2-Instantiating-a-class)\n", 92 | " * [1.3 Accessing attributes](py06v01_python_object_oriented.ipynb#1.3-Accessing-attributes)\n", 93 | " * [1.4 Class methods](py06v01_python_object_oriented.ipynb#1.4-Class-methods)\n", 94 | " * [1.5 Accessing methods](py06v01_python_object_oriented.ipynb#1.5-Accessing-methods)\n", 95 | " * [1.6 Is __init__ always required?](py06v01_python_object_oriented.ipynb#1.6-Is-__init__-always-required?)\n", 96 | " * [2 Inheritance](py06v01_python_object_oriented.ipynb#2-Inheritance)\n", 97 | " * [3 Duck typing and Polymorphism](py06v01_python_object_oriented.ipynb#3-Duck-typing-and-Polymorphism)\n", 98 | " * [4 Legacy style classes](py06v01_python_object_oriented.ipynb#4-Legacy-style-classes)\n", 99 | " * [5 How long does a class have to be?](py06v01_python_object_oriented.ipynb#5-How-long-does-a-class-have-to-be?)\n", 100 | "* [__Object Oriented Programming in Python__](py06v02_python_object_oriented.ipynb)\n", 101 | " * [1 How to define classes](py06v02_python_object_oriented.ipynb#1-How-to-define-classes)\n", 102 | " * [1.1 Protect your abstraction](py06v02_python_object_oriented.ipynb#1.1-Protect-your-abstraction)\n", 103 | " * [2 Inheritance](py06v02_python_object_oriented.ipynb#2-Inheritance)\n", 104 | " * [2.1 Overriding methods](py06v02_python_object_oriented.ipynb#2.1-Overriding-methods)\n", 105 | " * [3 Encapsulation](py06v02_python_object_oriented.ipynb#3-Encapsulation)\n", 106 | " * [3.1 Composition](py06v02_python_object_oriented.ipynb#3.1-Composition)\n", 107 | " * [3.2 Dynamic Extension](py06v02_python_object_oriented.ipynb#3.2-Dynamic-Extension)\n", 108 | " * [3.2.1 Example](py06v02_python_object_oriented.ipynb#3.2.1-Example)\n", 109 | " * [3.3 Polymorphism and DuckTyping](py06v02_python_object_oriented.ipynb#3.3-Polymorphism-and-DuckTyping)\n", 110 | " * [3.4 Legacy style classes](py06v02_python_object_oriented.ipynb#3.4-Legacy-style-classes)\n", 111 | " * [3.5 How long does a class has to be?](py06v02_python_object_oriented.ipynb#3.5-How-long-does-a-class-has-to-be?)\n", 112 | "* [__Integration of Python with compiled languages__](py07v01_python_speed-up_with_C.ipynb)\n", 113 | " * [1 Introduction](py07v01_python_speed-up_with_C.ipynb#1-Introduction)\n", 114 | " * [2 Speeding up Python code with Cython](py07v01_python_speed-up_with_C.ipynb#2-Speeding-up-Python-code-with-Cython)\n", 115 | " * [2.1 References](py07v01_python_speed-up_with_C.ipynb#2.1-References)\n", 116 | " * [3 Integrating C/C++ code into Python with Weave](py07v01_python_speed-up_with_C.ipynb#3-Integrating-C/C++-code-into-Python-with-Weave)\n", 117 | " * [3.1 References](py07v01_python_speed-up_with_C.ipynb#3.1-References)\n", 118 | " * [4 Access to external libraries with ctypes](py07v01_python_speed-up_with_C.ipynb#4-Access-to-external-libraries-with-ctypes)\n", 119 | " * [4.1 References](py07v01_python_speed-up_with_C.ipynb#4.1-References)\n", 120 | " * [5 Embedding Python in C](py07v01_python_speed-up_with_C.ipynb#5-Embedding-Python-in-C)\n", 121 | " * [5.1 References](py07v01_python_speed-up_with_C.ipynb#5.1-References)\n", 122 | "* [__Unicode__](py08v01_Unicode.ipynb)\n", 123 | " * [1 The World and Unicode](py08v01_Unicode.ipynb#1-The-World-and-Unicode)\n", 124 | " * [2 Encoding and decoding in Python 2.x](py08v01_Unicode.ipynb#2-Encoding-and-decoding-in-Python-2.x)\n", 125 | " * [3 Implicit conversion in Python 2.x](py08v01_Unicode.ipynb#3-Implicit-conversion-in-Python-2.x)\n", 126 | " * [4 Bytes Outside, Unicode Inside](py08v01_Unicode.ipynb#4-Bytes-Outside,-Unicode-Inside)\n", 127 | "* [__Regular Expressions__](py09v01_python_regular_expressions.ipynb)\n", 128 | " * [1 Finding wanted words and pieces of information (in a text complexity)](py09v01_python_regular_expressions.ipynb#1-Finding-wanted-words-and-pieces-of-information-(in-a-text-complexity))\n", 129 | " * [2 Python easy solutions for simple problems](py09v01_python_regular_expressions.ipynb#2-Python-easy-solutions-for-simple-problems)\n", 130 | " * [2.1 Some of the built-in python string functions may solve some of the easiest tasks:](py09v01_python_regular_expressions.ipynb#2.1-Some-of-the-built-in-python-string-functions-may-solve-some-of-the-easiest-tasks:)\n", 131 | " * [2.1.1 find](py09v01_python_regular_expressions.ipynb#2.1.1-find)\n", 132 | " * [2.1.2 strip, lstrip, rstrip](py09v01_python_regular_expressions.ipynb#2.1.2-strip,-lstrip,-rstrip)\n", 133 | " * [2.1.3 replace](py09v01_python_regular_expressions.ipynb#2.1.3-replace)\n", 134 | " * [2.1.4 functions for identifying the type of character](py09v01_python_regular_expressions.ipynb#2.1.4-functions-for-identifying-the-type-of-character)\n", 135 | " * [2.1.5 a slightly more complex example](py09v01_python_regular_expressions.ipynb#2.1.5-a-slightly-more-complex-example)\n", 136 | " * [2.2 The Power of Regular Expressions](py09v01_python_regular_expressions.ipynb#2.2-The-Power-of-Regular-Expressions)\n", 137 | " * [2.3 Python re module](py09v01_python_regular_expressions.ipynb#2.3-Python-re-module)\n" 138 | ] 139 | } 140 | ], 141 | "metadata": {}, 142 | "nbformat": 4, 143 | "nbformat_minor": 0 144 | } -------------------------------------------------------------------------------- /python-ipython/py05v04_python_more_examples.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Python More Examples" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "data": { 17 | "text/html": [ 18 | "
" 81 | ], 82 | "text/plain": [ 83 | "" 84 | ] 85 | }, 86 | "execution_count": 1, 87 | "metadata": {}, 88 | "output_type": "execute_result" 89 | } 90 | ], 91 | "source": [ 92 | "import addutils.toc ; addutils.toc.js(ipy_notebook=True)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 2, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "text/html": [ 103 | "\n", 241 | "\n" 252 | ], 253 | "text/plain": [ 254 | "" 255 | ] 256 | }, 257 | "execution_count": 2, 258 | "metadata": {}, 259 | "output_type": "execute_result" 260 | } 261 | ], 262 | "source": [ 263 | "import os\n", 264 | "import numpy as np\n", 265 | "import pandas as pd\n", 266 | "from addutils import css_notebook\n", 267 | "css_notebook()" 268 | ] 269 | }, 270 | { 271 | "cell_type": "markdown", 272 | "metadata": {}, 273 | "source": [ 274 | "## 1 Working with Excel " 275 | ] 276 | }, 277 | { 278 | "cell_type": "markdown", 279 | "metadata": {}, 280 | "source": [ 281 | "XlsxWriter is a Python module for creating Excel XLSX files." 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": 3, 287 | "metadata": { 288 | "collapsed": true 289 | }, 290 | "outputs": [], 291 | "source": [ 292 | "import xlsxwriter" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "Let’s start by creating a simple spreadsheet using Python and the XlsxWriter module." 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": 4, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "current_dir = os.path.abspath(os.path.curdir)\n", 309 | "file_path = os.path.join(current_dir, 'tmp', 'my_excel_file.xlsx')\n", 310 | "workbook = xlsxwriter.Workbook(file_path)" 311 | ] 312 | }, 313 | { 314 | "cell_type": "markdown", 315 | "metadata": {}, 316 | "source": [ 317 | "Now we add a spreadsheet. If we pass a string we can specify the name of the sheet." 318 | ] 319 | }, 320 | { 321 | "cell_type": "code", 322 | "execution_count": 5, 323 | "metadata": { 324 | "collapsed": true 325 | }, 326 | "outputs": [], 327 | "source": [ 328 | "worksheet = workbook.add_worksheet('My First Try')" 329 | ] 330 | }, 331 | { 332 | "cell_type": "markdown", 333 | "metadata": {}, 334 | "source": [ 335 | "Cells can be written with any value, and can be referenced either with row and column or with an identifier like 'D3'. Remember that positional reference is zero based." 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": 6, 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "data": { 345 | "text/plain": [ 346 | "0" 347 | ] 348 | }, 349 | "execution_count": 6, 350 | "metadata": {}, 351 | "output_type": "execute_result" 352 | } 353 | ], 354 | "source": [ 355 | "worksheet.write(0, 0, 1024)\n", 356 | "worksheet.write('C2', 'Some Text')" 357 | ] 358 | }, 359 | { 360 | "cell_type": "markdown", 361 | "metadata": {}, 362 | "source": [ 363 | "How to assign numerical values and formulas to cells (notice that formulas are assigned as strings):" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 7, 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "data": { 373 | "text/plain": [ 374 | "0" 375 | ] 376 | }, 377 | "execution_count": 7, 378 | "metadata": {}, 379 | "output_type": "execute_result" 380 | } 381 | ], 382 | "source": [ 383 | "worksheet.write(5, 4, 12)\n", 384 | "worksheet.write(6, 4, 15)\n", 385 | "worksheet.write(7, 4, '=E6+E7')" 386 | ] 387 | }, 388 | { 389 | "cell_type": "markdown", 390 | "metadata": {}, 391 | "source": [ 392 | "It is also possible to format cells:" 393 | ] 394 | }, 395 | { 396 | "cell_type": "code", 397 | "execution_count": 8, 398 | "metadata": { 399 | "collapsed": true 400 | }, 401 | "outputs": [], 402 | "source": [ 403 | "# Add a bold format to use to highlight cells.\n", 404 | "bold = workbook.add_format({'bold': True})" 405 | ] 406 | }, 407 | { 408 | "cell_type": "markdown", 409 | "metadata": {}, 410 | "source": [ 411 | "and use it to format a cell:" 412 | ] 413 | }, 414 | { 415 | "cell_type": "code", 416 | "execution_count": 9, 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "data": { 421 | "text/plain": [ 422 | "0" 423 | ] 424 | }, 425 | "execution_count": 9, 426 | "metadata": {}, 427 | "output_type": "execute_result" 428 | } 429 | ], 430 | "source": [ 431 | "worksheet.write(7, 3, 'Total:', bold)" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "It is also possible to format the size of the cell:" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": 10, 444 | "metadata": {}, 445 | "outputs": [ 446 | { 447 | "data": { 448 | "text/plain": [ 449 | "0" 450 | ] 451 | }, 452 | "execution_count": 10, 453 | "metadata": {}, 454 | "output_type": "execute_result" 455 | } 456 | ], 457 | "source": [ 458 | "# Adjust the column width.\n", 459 | "worksheet.set_column('D:E', 25)" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "Closing the workbook, saves the .xslx file." 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": 11, 472 | "metadata": { 473 | "collapsed": true 474 | }, 475 | "outputs": [], 476 | "source": [ 477 | "workbook.close()" 478 | ] 479 | }, 480 | { 481 | "cell_type": "markdown", 482 | "metadata": {}, 483 | "source": [ 484 | "The file can be read with pandas:" 485 | ] 486 | }, 487 | { 488 | "cell_type": "code", 489 | "execution_count": 12, 490 | "metadata": { 491 | "collapsed": true 492 | }, 493 | "outputs": [], 494 | "source": [ 495 | "a = pd.read_excel(file_path)" 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "metadata": {}, 501 | "source": [ 502 | "as you can see the first column is interpreted as a column name in pandas. It is possible to write data to excel directly from pandas using the same engine (xlsxwriter)" 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": 13, 508 | "metadata": { 509 | "collapsed": true 510 | }, 511 | "outputs": [], 512 | "source": [ 513 | "# Specify a writer\n", 514 | "writer = pd.ExcelWriter(file_path, engine='xlsxwriter')\n", 515 | "# Create some data\n", 516 | "expenses = {key: np.random.randint(100, size=10) for key in ['Rent', 'Gas', 'Food', 'Gym']}\n", 517 | "expenses = pd.DataFrame(expenses)\n", 518 | "# Write your DataFrame to a file \n", 519 | "expenses.to_excel(writer, 'Sheet1', index=False)\n", 520 | "# Save the result \n", 521 | "writer.save()" 522 | ] 523 | }, 524 | { 525 | "cell_type": "markdown", 526 | "metadata": {}, 527 | "source": [ 528 | "## 2 Sending an e-mail from a gmail account" 529 | ] 530 | }, 531 | { 532 | "cell_type": "markdown", 533 | "metadata": {}, 534 | "source": [ 535 | "This example is about sending mails using Python smtplib through an SMTP server." 536 | ] 537 | }, 538 | { 539 | "cell_type": "code", 540 | "execution_count": null, 541 | "metadata": { 542 | "collapsed": true 543 | }, 544 | "outputs": [], 545 | "source": [ 546 | "sender = 'ACCOUNT E-MAIL'\n", 547 | "password = 'PASSWORD'\n", 548 | "recipient = 'RECIPIENT'\n", 549 | "subject = 'A mail using Python'\n", 550 | "body = 'This is the body of the message'" 551 | ] 552 | }, 553 | { 554 | "cell_type": "markdown", 555 | "metadata": {}, 556 | "source": [ 557 | "**NOTE**: If you want to use Gmail, please consider that it has strong security mechanism and it may block your mail. " 558 | ] 559 | }, 560 | { 561 | "cell_type": "markdown", 562 | "metadata": {}, 563 | "source": [ 564 | "The smtplib module defines an SMTP client session object that can be used to send mail to any Internet machine with an SMTP or ESMTP listener daemon.\n", 565 | "The first step is to create a SMTP connection to the server. The `smtplib.SMTP` class encapsulates an SMTP connection. If the optional host and port parameters are given, the SMTP `connect()` method is called with those parameters during initialization. An `SMTPConnectError` is raised if the specified host doesn’t respond correctly. The optional timeout parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used).\n", 566 | "\n", 567 | "Next we will identify ourself to an ESMTP server using HELO. The SMTP HELO clause is the stage of the SMTP protocol where SMTP servers get introduced each other. EHLO is just like HELO except that the server’s response text provides computer-readable information about the server’s abilities.\n", 568 | "\n", 569 | "Remember Google’s SMTP server is ‘smtp.gmail.com’ and the port is 587." 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": null, 575 | "metadata": { 576 | "collapsed": true 577 | }, 578 | "outputs": [], 579 | "source": [ 580 | "import smtplib\n", 581 | "session = smtplib.SMTP('smtp.gmail.com', 587)" 582 | ] 583 | }, 584 | { 585 | "cell_type": "markdown", 586 | "metadata": {}, 587 | "source": [ 588 | "Next we call `SMTP.starttls` function to put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP commands that follow will be encrypted. You should then call `ehlo()` again. If keyfile and certfile are provided, these are passed to the socket module’s `ssl()` function.If there has been no previous EHLO or HELO command this session, this method tries ESMTP EHLO first." 589 | ] 590 | }, 591 | { 592 | "cell_type": "code", 593 | "execution_count": null, 594 | "metadata": {}, 595 | "outputs": [], 596 | "source": [ 597 | "session.starttls()\n", 598 | "session.ehlo()" 599 | ] 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "metadata": {}, 604 | "source": [ 605 | "OK, now we are safe to login to the server using SMTP.login(user, password). After successful login we use SMTP.sendmail(from_addr, to_addrs, msg[, mail_options, rcpt_options]) to send mails via Gmail." 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "execution_count": null, 611 | "metadata": {}, 612 | "outputs": [], 613 | "source": [ 614 | "session.login(sender, password)" 615 | ] 616 | }, 617 | { 618 | "cell_type": "code", 619 | "execution_count": null, 620 | "metadata": { 621 | "collapsed": true 622 | }, 623 | "outputs": [], 624 | "source": [ 625 | "headers = [\"from: \" + sender,\n", 626 | " \"subject: \" + subject,\n", 627 | " \"to: \" + recipient,\n", 628 | " \"mime-version: 1.0\",\n", 629 | " \"content-type: text/html\"]\n", 630 | "headers = \"\\r\\n\".join(headers)" 631 | ] 632 | }, 633 | { 634 | "cell_type": "code", 635 | "execution_count": null, 636 | "metadata": { 637 | "collapsed": true 638 | }, 639 | "outputs": [], 640 | "source": [ 641 | "session.sendmail(sender, recipient, headers + \"\\r\\n\\r\\n\" + body)" 642 | ] 643 | }, 644 | { 645 | "cell_type": "markdown", 646 | "metadata": {}, 647 | "source": [ 648 | "## 3 Automating DuckDuckGo search" 649 | ] 650 | }, 651 | { 652 | "cell_type": "markdown", 653 | "metadata": {}, 654 | "source": [ 655 | "Using duckduckgo API is possible to query duckduckgo search engine. Say we are going to search \"barack obama\"" 656 | ] 657 | }, 658 | { 659 | "cell_type": "code", 660 | "execution_count": 14, 661 | "metadata": { 662 | "collapsed": true 663 | }, 664 | "outputs": [], 665 | "source": [ 666 | "import json\n", 667 | "import urllib\n", 668 | "\n", 669 | "searchfor = 'barack obama'" 670 | ] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "metadata": {}, 675 | "source": [ 676 | "To do this we need to create a specially crafted url to pass to the google api. `urllib` is a standard `python` library that allows to create and retrive URL(s).\n" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": 15, 682 | "metadata": { 683 | "collapsed": true 684 | }, 685 | "outputs": [], 686 | "source": [ 687 | "query = urllib.parse.urlencode({'q': searchfor})\n", 688 | "url = 'http://api.duckduckgo.com/?format=json&{}'.format(query)\n", 689 | "search_response = urllib.request.urlopen(url)\n", 690 | "search_results = search_response.read()" 691 | ] 692 | }, 693 | { 694 | "cell_type": "markdown", 695 | "metadata": {}, 696 | "source": [ 697 | "Now we have a json encoded string containing our results, we are going to use `json.loads` in order to get what we are interested to (say Total results and top hits)." 698 | ] 699 | }, 700 | { 701 | "cell_type": "code", 702 | "execution_count": 16, 703 | "metadata": {}, 704 | "outputs": [ 705 | { 706 | "name": "stdout", 707 | "output_type": "stream", 708 | "text": [ 709 | "Related Topics: 34\n", 710 | "Total results: 0\n", 711 | "Top 0 hits:\n" 712 | ] 713 | } 714 | ], 715 | "source": [ 716 | "results = json.loads(search_results.decode(encoding='UTF-8'))\n", 717 | "related_topics = results['RelatedTopics']\n", 718 | "hits = results['Results']\n", 719 | "print('Related Topics: {}'.format(len(related_topics)))\n", 720 | "print('Total results: {}'.format(len(hits)))\n", 721 | "print('Top {} hits:'.format(len(hits)))\n", 722 | "for h in hits:\n", 723 | " print(' ', h['FirstURL'])" 724 | ] 725 | }, 726 | { 727 | "cell_type": "markdown", 728 | "metadata": {}, 729 | "source": [ 730 | "---\n", 731 | "\n", 732 | "Visit [www.add-for.com]() for more tutorials and updates.\n", 733 | "\n", 734 | "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License." 735 | ] 736 | } 737 | ], 738 | "metadata": { 739 | "kernelspec": { 740 | "display_name": "Python [conda env:addfor_tutorials]", 741 | "language": "python", 742 | "name": "conda-env-addfor_tutorials-py" 743 | }, 744 | "language_info": { 745 | "codemirror_mode": { 746 | "name": "ipython", 747 | "version": 3 748 | }, 749 | "file_extension": ".py", 750 | "mimetype": "text/x-python", 751 | "name": "python", 752 | "nbconvert_exporter": "python", 753 | "pygments_lexer": "ipython3", 754 | "version": "3.6.4" 755 | } 756 | }, 757 | "nbformat": 4, 758 | "nbformat_minor": 1 759 | } 760 | -------------------------------------------------------------------------------- /python-ipython/py07v04_python_speed-up_with_C.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Integration of Python with compiled languages" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import addutils.toc ; addutils.toc.js(ipy_notebook=True)" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "from addutils import css_notebook\n", 26 | "css_notebook()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## 1 Introduction" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "Python users and developers can use many tools for integrating Python with other languages. For example it's possible to link Python code to external libraries, such as C or Fortran ones, usually with a more efficient processing and a general speed up. These tools are tipically multi-platform, so that you can develop in Linux, Windows or MacOS, at your will. Here we will consider only the most quoted tools, i.e. **Cython**, **Weave** and **ctypes**. Other tools, such as F2PY (Fortran, C) and SWIG, are available. SWIG (Simplified Wrapper Interface Generator) automates the generation of wrapper code for interfacing C/C++ with many languages (Guile, Java, Ocaml, Perl, Pike, PHP, Python, Ruby, and Tcl). A good primer on F2PY and SWIG is in Langtangen - A Primer on Scientific Programming with Python.\n", 41 | "CXX, Boost.Python, and SCXX are C++ tools that simplify programming with the Python C API." 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "## 2 Speeding up Python code with Cython" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": {}, 54 | "source": [ 55 | "Cython is (almost) a superset of Python. With Cython you can assign C types to Python variables, coding in something similar to Python but getting the speed advantages of C. The resulting run time can be much shorter, even with improvements of 200-300x." 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "Cython code is translated in C code using Python setup scripts based on the `distutils` library. An example is below: it represents the setup script for the three compiled extensions that we will consider in this paragraph. Setup scripts are run from the command lines.\n", 63 | "\n", 64 | "NB: the following examples are meant to be run in MacOSX. Before running them on other Operating Systems it is necessary to install a C compiler and to modify them accordingly." 65 | ] 66 | }, 67 | { 68 | "cell_type": "markdown", 69 | "metadata": {}, 70 | "source": [ 71 | "```python\n", 72 | "from distutils.core import setup\n", 73 | "from distutils.extension import Extension\n", 74 | "from Cython.Distutils import build_ext\n", 75 | "\n", 76 | "setup(\n", 77 | " cmdclass = {'build_ext': build_ext},\n", 78 | " ext_modules = [\n", 79 | " Extension(\"integrate_compiled\",\n", 80 | " [\"integrate.py\"],\n", 81 | " ),\n", 82 | " Extension(\"integrate_hints\",\n", 83 | " [\"integrate_hints.py\"],\n", 84 | " ),\n", 85 | " Extension(\"integrate_cy\",\n", 86 | " [\"integrate_cy.pyx\"],\n", 87 | " ),\n", 88 | " ])\n", 89 | "```" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "The C code is compiled as a Python library, in **pyd** format, that can be imported in Python scripts as a normal Python module (such as Numpy and Pandas)." 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "metadata": {}, 102 | "source": [ 103 | "We consider as a specific example a case of integration of a power function, deriving from a presentation by van der Walt at the 2010 Summer School in Trento. Execution times for Python and Cython versions have been measured on a normal laptop (Windows Vista, 4 GB RAM)." 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "We try to compute the approximate integral of $$\\int_{a}^{b} f(x)dx$$ by using rectangular discretisation." 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "# integrate.py\n", 127 | "from __future__ import division\n", 128 | "\n", 129 | "def f(x):\n", 130 | " return x**4 - 3 * x\n", 131 | "\n", 132 | "def integrate_f(a, b, N):\n", 133 | " \"\"\"Rectangle integration of a function.\n", 134 | "\n", 135 | " Parameters\n", 136 | " ----------\n", 137 | " a, b : ints\n", 138 | " Interval over which to integrate.\n", 139 | " N : int\n", 140 | " Number of intervals to use in the discretisation.\n", 141 | "\n", 142 | " \"\"\"\n", 143 | " s = 0\n", 144 | " dx = (b - a) / N\n", 145 | " for i in range(N):\n", 146 | " s += f(a + i * dx)\n", 147 | " return s * dx" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "After running the setup script on the integrate.py script, we obtain a integrate.pyd module that we can import and test." 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "metadata": {}, 161 | "outputs": [], 162 | "source": [ 163 | "import timeit\n", 164 | "\n", 165 | "setup_statement=\"import utilities.integrate as i0\"\n", 166 | "\n", 167 | "s = \"\"\"\\\n", 168 | "i0.integrate_f(0,10,100000)\n", 169 | "\"\"\"\n", 170 | "t = timeit.Timer( s, setup_statement )\n", 171 | "\n", 172 | "print(\"Execution time: {:.2f} msec/pass\".format( 1000 * t.timeit(number=100)/100 ))" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "If we modify the code with Cython decorators ('@cython.locals'), we can attribute a type to the function arguments and to local variables, as is the norm for static languages. This code is still compatible with Python. " 180 | ] 181 | }, 182 | { 183 | "cell_type": "code", 184 | "execution_count": null, 185 | "metadata": {}, 186 | "outputs": [], 187 | "source": [ 188 | "from __future__ import division\n", 189 | "import cython\n", 190 | "\n", 191 | "@cython.locals(x=cython.double)\n", 192 | "def f(x):\n", 193 | " # Note: x**4 translates to slow code in Cython\n", 194 | " # Use x*x*x*x to avoid calling 'pow'\n", 195 | " return x**4 - 3 * x\n", 196 | "\n", 197 | "@cython.locals(a=cython.double, b=cython.double,\n", 198 | " N=cython.int, s=cython.double,\n", 199 | " dx=cython.double, i=cython.int)\n", 200 | "def integrate_f(a, b, N):\n", 201 | "\n", 202 | " s = 0\n", 203 | " dx = (b - a) / N\n", 204 | " for i in range(N):\n", 205 | " s += f(a + i * dx)\n", 206 | " return s * dx" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "If we compile the code with the setup scripts, we obtain a `pyd dll` that we can import and use." 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": null, 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [ 222 | "setup_statement=\"import utilities.integrate_hints as i1\"\n", 223 | "\n", 224 | "s = \"\"\"i1.integrate_f(0,10,100000)\"\"\"\n", 225 | "t = timeit.Timer( s, setup_statement )\n", 226 | "\n", 227 | "print(\"Execution time: {:.2f} msec/pass\".format( 1000 * t.timeit(number=100)/100))" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "The execution time should get a 3x improvement." 235 | ] 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "We can further optimize the code, using cdef (c definitions), in a way that **this code is no longer compatible with Python**, but, when compiled, it run much faster than the original code." 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "metadata": {}, 247 | "source": [ 248 | "```python\n", 249 | "# cython: cdivision=True\n", 250 | "\n", 251 | "# ^^^ Could also use @cython.cdivision(True) decorator\n", 252 | "\n", 253 | "cdef double f(double x):\n", 254 | " return x*x*x*x - 3 * x\n", 255 | "\n", 256 | "def integrate_f(double a, double b, int N):\n", 257 | " cdef double s = 0\n", 258 | " cdef double dx = (b - a) / N\n", 259 | " cdef int i\n", 260 | " for i in range(N):\n", 261 | " s += f(a + i * dx)\n", 262 | " return s * dx\n", 263 | "```" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": null, 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "setup_statement=\"import utilities.integrate_cy as i2\"\n", 273 | "\n", 274 | "s = \"\"\"\\\n", 275 | "i2.integrate_f(0,10,100000)\n", 276 | "\"\"\"\n", 277 | "t = timeit.Timer( s, setup_statement )\n", 278 | "\n", 279 | "print(\"Execution time: {:.2f} msec/pass\".format( 1000 * t.timeit(number=100)/100 ))\n" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "metadata": {}, 285 | "source": [ 286 | "We obtain an improvement of about 150x, and the code is still quite readable, at the (small) price of having the cython code automatically translated into C by the setup scripts." 287 | ] 288 | }, 289 | { 290 | "cell_type": "markdown", 291 | "metadata": {}, 292 | "source": [ 293 | "### 2.1 References" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "metadata": {}, 299 | "source": [ 300 | "Cython for NumPy users - http://docs.cython.org/src/userguide/numpy_tutorial.html#numpy-tutorial\n", 301 | " \n", 302 | "A good introduction is: van der Walt, Summer School 2010 Trento - https://python.g-node.org/python-autumnschool-2010/start\n", 303 | " \n", 304 | " " 305 | ] 306 | }, 307 | { 308 | "cell_type": "markdown", 309 | "metadata": {}, 310 | "source": [ 311 | "## 3 Integrating C/C++ code into Python with Weave" 312 | ] 313 | }, 314 | { 315 | "cell_type": "markdown", 316 | "metadata": {}, 317 | "source": [ 318 | "Can we insert C code into Python scripts? With Weave it is possible.\n", 319 | "Thanks to the 'inline' function, it is possible to integrate C code lines as text into a Python script. At the first execution of the script, the C code is compiled, so that the total run time can get longer (generally a few seconds), but the compiled code is saved in cache in the computer and is then directly executed the next times. Usually the compiled code is tens of times faster than the interpreted version. " 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "metadata": {}, 325 | "source": [ 326 | "There is also another Weave functionality, 'blitz', that allows to directly compile a single Python expression into C++. For instance, the following Python expression (from http://docs.scipy.org/doc/scipy/reference/tutorial/weave.html):" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [ 335 | "from scipy import * # or from NumPy import *\n", 336 | "import numpy as np\n", 337 | "a = np.zeros((512,512), float)\n", 338 | "b = np.ones((512,512), float)" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": null, 344 | "metadata": {}, 345 | "outputs": [], 346 | "source": [ 347 | "a[1:-1,1:-1] = (b[1:-1,1:-1] + b[2:,1:-1] + 3*b[:-2,1:-1] \\\n", 348 | " + b[1:-1,2:] + b[1:-1,:-2]) / 6.0\n", 349 | "print a" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "can be compiled with blitz in this way:" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "import scipy.weave\n", 366 | "from scipy import * # or from NumPy import *\n", 367 | "\n", 368 | "a = np.zeros((512,512), float)\n", 369 | "b = np.ones((512,512), float)\n", 370 | "\n", 371 | "expr = \"a[1:-1,1:-1] = (b[1:-1,1:-1] + b[2:,1:-1] + 3*b[:-2,1:-1]\" \\\n", 372 | " \"+ b[1:-1,2:] + b[1:-1,:-2]) / 5.0\"\n", 373 | "scipy.weave.blitz(expr)\n", 374 | "print a" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "Generally the increase in run speed with 'blitz' is lower to the one that we obtain with 'inline'." 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": {}, 387 | "source": [ 388 | "An example of stochastic simulation of stocks with 'inline' is here exemplified. The C code is inserted as a text string. It do FOR loops on a Numpy array (named S), passed by reference. The function arguments are passed as list elements in the weave.inline arguments. " 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "Original version:" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": null, 401 | "metadata": {}, 402 | "outputs": [], 403 | "source": [ 404 | "import numpy as np\n", 405 | "import pandas as pd\n", 406 | "from pandas.io.data import read_csv\n", 407 | "\n", 408 | "AAPL = read_csv('example_data/p03_AAPL.txt',\n", 409 | " index_col='Date', parse_dates=True)\n", 410 | "\n", 411 | "AAPL['Ret'] = np.log(AAPL.Close/AAPL.Close.shift(1)) \n", 412 | "vol = np.std(AAPL['Ret'])*np.sqrt(252) \n", 413 | "r = 0.025 # Constant Short Rate\n", 414 | "S0 = AAPL['Close'][-1] # End Value \n", 415 | "K = S0 * 1.1 # OTM Call Option \n", 416 | "T = 1.0 # Maturity Year\n", 417 | "M = 100; dt = T / M # Time Steps \n", 418 | "I = 10000 # Simulation Paths\n", 419 | "S = np.zeros((M + 1, I))\n", 420 | "S[0,:] = S0\n", 421 | "for t in range(1, M + 1):\n", 422 | " ran = np.random.standard_normal(I)\n", 423 | " S[t, :] = S[t - 1, :] * np.exp((r - vol**2 / 2) * dt + vol * np.sqrt(dt) * ran)\n" 424 | ] 425 | }, 426 | { 427 | "cell_type": "markdown", 428 | "metadata": {}, 429 | "source": [ 430 | "Modified version:" 431 | ] 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": null, 436 | "metadata": {}, 437 | "outputs": [], 438 | "source": [ 439 | "from scipy import weave\n", 440 | "from scipy.weave import converters\n", 441 | "# changes for C compatibility\n", 442 | "vol_c = float(vol) # vol is a np array, would give error in C code\n", 443 | "ran = np.random.normal(0, 1, (M+1)*I)\n", 444 | "ran = np.reshape(ran, (M+1,I)) \n", 445 | "\n", 446 | "S = np.zeros((M+1,I))\n", 447 | "\n", 448 | "code = r\"\"\"\n", 449 | "\n", 450 | "for ( int i = 0; i < I; i++ ) S(0,i) = S0;\n", 451 | "\n", 452 | "for ( int t = 1; t < M+1; t++ )\n", 453 | "{\n", 454 | " for ( int i = 0; i < I; i++ ) S(t,i) = S(t-1,i) * exp( (r - vol_c * vol_c/2.0) * dt + vol_c * sqrt(dt) * ran(t,i) );\n", 455 | "}\n", 456 | "\n", 457 | "\"\"\"\n", 458 | " \n", 459 | "weave.inline(code,['S', 'M', 'dt', 'I', 'vol_c','r', 'S0', 'ran'], type_converters=converters.blitz, compiler='gcc')\n" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "### 3.1 References" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "Weave (scipy.weave) - http://docs.scipy.org/doc/scipy/reference/tutorial/weave.html\n", 474 | " \n", 475 | "Weave (Sage) - http://www.sagemath.org/doc/numerical_sage/weave.html" 476 | ] 477 | }, 478 | { 479 | "cell_type": "markdown", 480 | "metadata": {}, 481 | "source": [ 482 | "## 4 Access to external libraries with ctypes" 483 | ] 484 | }, 485 | { 486 | "cell_type": "markdown", 487 | "metadata": {}, 488 | "source": [ 489 | "Numerical libraries such as Blas, Lapack or Atlas are routinely available in Python. We can access other libraries from Python with the ctypes module. ctypes presents many functionalities: it has wrappers for C data, it allows to create pointers and to return the referenced values, it can call functions from the C library (libc)." 490 | ] 491 | }, 492 | { 493 | "cell_type": "markdown", 494 | "metadata": {}, 495 | "source": [ 496 | "As an example, we consider the creation of a simple function in a shared library, created and tested in C in Linux (Ubuntu) (from http://stackoverflow.com/questions/5862915/passing-numpy-arrays-to-a-c-function-for-input-and-output). The C source code is the following:" 497 | ] 498 | }, 499 | { 500 | "cell_type": "raw", 501 | "metadata": {}, 502 | "source": [ 503 | "/* double_array.c */\n", 504 | "\n", 505 | "#include \n", 506 | "\n", 507 | "void cdouble(const double *indata, size_t size, double *outdata) \n", 508 | "{\n", 509 | " size_t i;\n", 510 | " for (i = 0; i < size; ++i)\n", 511 | " outdata[i] = indata[i] * 2.0;\n", 512 | "}" 513 | ] 514 | }, 515 | { 516 | "cell_type": "markdown", 517 | "metadata": {}, 518 | "source": [ 519 | "We can compile this file as shared library in Linux:" 520 | ] 521 | }, 522 | { 523 | "cell_type": "raw", 524 | "metadata": {}, 525 | "source": [ 526 | "gcc -fPIC -shared -o double_array.so double_array.c" 527 | ] 528 | }, 529 | { 530 | "cell_type": "markdown", 531 | "metadata": {}, 532 | "source": [ 533 | "and use this new library from Python:" 534 | ] 535 | }, 536 | { 537 | "cell_type": "raw", 538 | "metadata": {}, 539 | "source": [ 540 | "import numpy as np\n", 541 | "import ctypes\n", 542 | "from numpy.ctypeslib import ndpointer\n", 543 | "lib = ctypes.cdll.LoadLibrary('./double_array.so')\n", 544 | "\n", 545 | "cdouble = lib.cdouble\n", 546 | "cdouble.restype = None\n", 547 | "cdouble.argtypes = [ndpointer(ctypes.c_double),\n", 548 | " ctypes.c_size_t,\n", 549 | " ndpointer(ctypes.c_double)]\n", 550 | "\t\t\t\t\n", 551 | "indata = np.ones((10,10))\n", 552 | "outdata = np.empty((10,10))\n", 553 | "cdouble(indata, indata.size, outdata)\n", 554 | "\n", 555 | "print 'array originale: %s' % indata\n", 556 | "print 'nuovo array: %s' % outdata" 557 | ] 558 | }, 559 | { 560 | "cell_type": "markdown", 561 | "metadata": {}, 562 | "source": [ 563 | "A tested compilation setting for Windows with gcc that allows to create dll recognised by ctypes is:" 564 | ] 565 | }, 566 | { 567 | "cell_type": "raw", 568 | "metadata": {}, 569 | "source": [ 570 | "gcc -c -DBUILD_DLL sourcename.c gcc -shared -o sourcename.dll sourcename.o " 571 | ] 572 | }, 573 | { 574 | "cell_type": "markdown", 575 | "metadata": {}, 576 | "source": [ 577 | "See: http://codespeak.net/pipermail/cython-dev/2009-May/005517.html" 578 | ] 579 | }, 580 | { 581 | "cell_type": "markdown", 582 | "metadata": {}, 583 | "source": [ 584 | "### 4.1 References" 585 | ] 586 | }, 587 | { 588 | "cell_type": "markdown", 589 | "metadata": {}, 590 | "source": [ 591 | "ctypes — A foreign function library for Python - http://docs.python.org/library/ctypes.html\n", 592 | " \n", 593 | "ctypes (Sage) - http://www.sagemath.org/doc/numerical_sage/ctypes.html " 594 | ] 595 | }, 596 | { 597 | "cell_type": "markdown", 598 | "metadata": {}, 599 | "source": [ 600 | "## 5 Embedding Python in C" 601 | ] 602 | }, 603 | { 604 | "cell_type": "markdown", 605 | "metadata": {}, 606 | "source": [ 607 | "Incorporating Python in C presents a greater complexity and more freedom degrees than incorporating C in Python. It also requires the knowledge of the C APIs in Python.\n", 608 | "The first step for Python embedding is the inclusion of the Python header file 'Python.h' in our C program. 'Py_Initialize()' and 'Py_Finalize()' initialize and close the Python interpreter, respectively.\n", 609 | "There are many possible ways to interface C and Python: \n", 610 | "\n", 611 | "* insert in C static (or dinamically constructed) string of Python code (in a way similar to Weave.inline) and execute them with 'PyRun_SimpleString()'. Since the compilation is a slow process, it is better to precompile them as Python bytecode;\n", 612 | "* call Python objects associating a list of arguments;\n", 613 | "* call and execute entire Python scripts with 'PyRun_SimpleFile()'.\n", 614 | "\n", 615 | "When a Python object is no longer used, the memory can be set free with 'Py_DECREF'.\n", 616 | "The incorporated Python code can be in turn extended with C, using the previously described techniques.\n", 617 | "Considering the difficulties of embedding in C, there are no largely used tools that can simplify the work of integrating of Python in C.\n", 618 | "\n" 619 | ] 620 | }, 621 | { 622 | "cell_type": "markdown", 623 | "metadata": {}, 624 | "source": [ 625 | "### 5.1 References" 626 | ] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "metadata": {}, 631 | "source": [ 632 | "Supercharging C++ Code With Embedded Python – EuroPython 2012 Talk - http://realmike.org/blog/2012/07/05/supercharging-c-code-with-embedded-python/\n", 633 | " \n", 634 | "Embedding Python in Another Application - http://docs.python.org/extending/embedding.html\n", 635 | " \n", 636 | "Python/C API Reference Manual - http://docs.python.org/c-api/index.html#c-api-index " 637 | ] 638 | }, 639 | { 640 | "cell_type": "markdown", 641 | "metadata": {}, 642 | "source": [ 643 | "---\n", 644 | "\n", 645 | "Visit [www.add-for.com]() for more tutorials and updates.\n", 646 | "\n", 647 | "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License." 648 | ] 649 | } 650 | ], 651 | "metadata": { 652 | "kernelspec": { 653 | "display_name": "Python [conda env:addfor_tutorials]", 654 | "language": "python", 655 | "name": "conda-env-addfor_tutorials-py" 656 | }, 657 | "language_info": { 658 | "codemirror_mode": { 659 | "name": "ipython", 660 | "version": 3 661 | }, 662 | "file_extension": ".py", 663 | "mimetype": "text/x-python", 664 | "name": "python", 665 | "nbconvert_exporter": "python", 666 | "pygments_lexer": "ipython3", 667 | "version": "3.6.4" 668 | } 669 | }, 670 | "nbformat": 4, 671 | "nbformat_minor": 1 672 | } 673 | -------------------------------------------------------------------------------- /python-ipython/tmp/my_input2.txt: -------------------------------------------------------------------------------- 1 | First Second 2 | 10 3.243e-01 3 | 20 1.324e+00 4 | 21 7.238e+00 5 | 36 8.330e-01 6 | 56 2.373e+02 7 | -------------------------------------------------------------------------------- /python-ipython/tmp/my_unicode_example.txt: -------------------------------------------------------------------------------- 1 | This is a string: †ℯ✖t -------------------------------------------------------------------------------- /python-ipython/tmp/my_unknow_encoder.txt: -------------------------------------------------------------------------------- 1 | THIS IS AN ENCODING TEST -------------------------------------------------------------------------------- /python-ipython/tmp/my_utf8_example.txt: -------------------------------------------------------------------------------- 1 | This is a string: †ℯ✖t -------------------------------------------------------------------------------- /python-ipython/tmp/out_ascii.pkl: -------------------------------------------------------------------------------- 1 | (lp0 2 | Vone 3 | p1 4 | aVtwo 5 | p2 6 | aVthree 7 | p3 8 | a. -------------------------------------------------------------------------------- /python-ipython/tmp/out_compb.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/tmp/out_compb.pkl -------------------------------------------------------------------------------- /python-ipython/tmp/temporary.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/tmp/temporary.txt -------------------------------------------------------------------------------- /python-ipython/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | from . import my_module 2 | -------------------------------------------------------------------------------- /python-ipython/utilities/integrate.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | def f(x): 4 | return x**4 - 3 * x 5 | 6 | def integrate_f(a, b, N): 7 | """Rectangle integration of a function. 8 | 9 | Parameters 10 | ---------- 11 | a, b : ints 12 | Interval over which to integrate. 13 | N : int 14 | Number of intervals to use in the discretisation. 15 | 16 | """ 17 | s = 0 18 | dx = (b - a) / N 19 | for i in range(N): 20 | s += f(a + i * dx) 21 | return s * dx 22 | 23 | -------------------------------------------------------------------------------- /python-ipython/utilities/integrate.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate.pyd -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_compiled.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate_compiled.so -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_cy.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate_cy.pyd -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_cy.pyx: -------------------------------------------------------------------------------- 1 | # cython: cdivision=True 2 | 3 | # ^^^ Could also use @cython.cdivision(True) decorator 4 | 5 | cdef double f(double x): 6 | return x*x*x*x - 3 * x 7 | 8 | def integrate_f(double a, double b, int N): 9 | cdef double s = 0 10 | cdef double dx = (b - a) / N 11 | cdef int i 12 | for i in range(N): 13 | s += f(a + i * dx) 14 | return s * dx 15 | -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_cy.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate_cy.so -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_hints.py: -------------------------------------------------------------------------------- 1 | # This code still runs under Python! 2 | 3 | # cython: cdivision=True 4 | 5 | from __future__ import division 6 | import cython 7 | 8 | @cython.locals(x=cython.double) 9 | def f(x): 10 | # Note: x**4 translates to slow code in Cython 11 | # Use x*x*x*x to avoid calling 'pow' 12 | return x**4 - 3 * x 13 | 14 | @cython.locals(a=cython.double, b=cython.double, 15 | N=cython.int, s=cython.double, 16 | dx=cython.double, i=cython.int) 17 | def integrate_f(a, b, N): 18 | """Rectangle integration of a function. 19 | 20 | Parameters 21 | ---------- 22 | a, b : ints 23 | Interval over which to integrate. 24 | N : int 25 | Number of intervals to use in the discretisation. 26 | 27 | """ 28 | s = 0 29 | dx = (b - a) / N 30 | for i in range(N): 31 | s += f(a + i * dx) 32 | return s * dx 33 | 34 | -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_hints.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate_hints.pyd -------------------------------------------------------------------------------- /python-ipython/utilities/integrate_hints.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/integrate_hints.so -------------------------------------------------------------------------------- /python-ipython/utilities/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/addfor/tutorials/2e1d136536fcaaf190e1da4ea9fbd60070cfaac2/python-ipython/utilities/logo.png -------------------------------------------------------------------------------- /python-ipython/utilities/my_finance_tools.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import matplotlib.finance as fin 3 | from pandas import Index, DataFrame 4 | from pandas.core.datetools import BMonthEnd 5 | 6 | # MY FINANCE TOOLS ------------------------------------------------------------- 7 | ''' Created by: Addfor S.p.A. 8 | This module provides few example finance functions 9 | ''' 10 | 11 | def getQuotes(symbol, start, end): 12 | '''getQuotes documentation''' 13 | quotes = fin.quotes_historical_yahoo(symbol, start, end) 14 | dates, opn, close, high, low, volume = zip(*quotes) 15 | data = {'open': opn, 'close': close, 'high' : high, 16 | 'low' : low, 'volume': volume} 17 | 18 | dates = Index([datetime.fromordinal(int(d)) for d in dates]) 19 | return DataFrame(data, index=dates) 20 | 21 | 22 | def toMonthly(frame, how): 23 | '''toMonthly documentation''' 24 | offset = BMonthEnd() 25 | return frame.groupby(offset.rollforward).aggregate(how) 26 | -------------------------------------------------------------------------------- /python-ipython/utilities/my_module.py: -------------------------------------------------------------------------------- 1 | # my_module.py ========================================================= 2 | ''' This is the documentation 3 | for my example module''' 4 | 5 | '''Naming convenctions: 6 | Function names must be lowercase separated by underscores '_' 7 | An underscore '_' at the beginning of a functuin name means the function is 8 | intended to be private BUT Python do NOT enforce this rule: 9 | you can break the code if you want! ''' 10 | 11 | # Variables defined here are available for ALL the functions in the module 12 | # When module is imported, these variable can be accessed with: 13 | # module_name.variable_name 14 | module_variable = 1 15 | 16 | # Constants are named in uppercase 17 | MODULE_CONSTANT = 1 18 | 19 | def my_function(name): 20 | ''' This is the documentation for "my_function"''' 21 | # Variables defined inside functions are local 22 | (first, family) = name # Arguments bundled in a Tuple 23 | return _my_private_function(first, family) 24 | 25 | def _my_private_function(first_name, second_name): 26 | ''' This is the documentation for "_my_private_function"''' 27 | full_name = first_name + ' [' + second_name.upper() + ']' 28 | return full_name 29 | 30 | if __name__ == '__main__': 31 | ''' This is a Unit Test: use "run my_module" from Python interpreter''' 32 | print('This is the testing code:') 33 | print(my_function(('Johnn', 'Doe'))) 34 | -------------------------------------------------------------------------------- /python-ipython/utilities/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Distutils import build_ext 4 | 5 | setup( 6 | cmdclass = {'build_ext': build_ext}, 7 | ext_modules = [ 8 | Extension("integrate_compiled", 9 | ["integrate.py"], 10 | ), 11 | Extension("integrate_hints", 12 | ["integrate_hints.py"], 13 | ), 14 | Extension("integrate_cy", 15 | ["integrate_cy.pyx"], 16 | ), 17 | ]) 18 | --------------------------------------------------------------------------------