├── README.md └── notebooks ├── 03-Creating-and-Saving Data in HDF5 Files.ipynb ├── 04-Reading Data from HDF5 Files.ipynb ├── 05-Creating Groups in HDF5 Files.ipynb ├── 06-Reading Groups in HDF5 Files.ipynb ├── 07-HDF5 Compress Data.ipynb ├── 08-HDF5 Attributes.ipynb ├── 09-Create HDF5 Files with Pandas.ipynb └── 10-Read HDF5 Files with Pandas.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # HDF5-with-Python 2 | These are the Jupyter Notebooks that come with [My HDF5 with Python Video Tutorial on Youtube](https://www.youtube.com/watch?v=y4DXr3Y10MM&list=PLea0WJq13cnB_ORdGzEkPlZEN20TSt6Lx) 3 | 4 | ## These videos and notebooks are free for all to use! 5 | 6 | 7 | -------------------------------------------------------------------------------- /notebooks/03-Creating-and-Saving Data in HDF5 Files.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 28, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 29, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "matrix1 = np.random.random(size = (1000,1000))\n", 24 | "matrix2 = np.random.random(size = (10000,100))" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 30, 30 | "metadata": { 31 | "collapsed": false 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "with h5py.File('/home/noureddin/Desktop/hdf5_data.h5', 'w') as hdf:\n", 36 | " hdf.create_dataset('dataset1', data=matrix1)\n", 37 | " hdf.create_dataset('dataset2', data=matrix2)" 38 | ] 39 | } 40 | ], 41 | "metadata": { 42 | "anaconda-cloud": {}, 43 | "kernelspec": { 44 | "display_name": "Python 3", 45 | "language": "python", 46 | "name": "python3" 47 | }, 48 | "language_info": { 49 | "codemirror_mode": { 50 | "name": "ipython", 51 | "version": 3 52 | }, 53 | "file_extension": ".py", 54 | "mimetype": "text/x-python", 55 | "name": "python", 56 | "nbconvert_exporter": "python", 57 | "pygments_lexer": "ipython3", 58 | "version": "3.5.2" 59 | } 60 | }, 61 | "nbformat": 4, 62 | "nbformat_minor": 1 63 | } 64 | -------------------------------------------------------------------------------- /notebooks/04-Reading Data from HDF5 Files.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 40, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 46, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "List of datasets in this file: \n", 27 | " ['dataset1', 'dataset2']\n", 28 | "Shape of dataset1: \n", 29 | " (10000, 100)\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "with h5py.File('/home/noureddin/Desktop/hdf5_data.h5','r') as hdf:\n", 35 | " ls = list(hdf.keys())\n", 36 | " print('List of datasets in this file: \\n', ls)\n", 37 | " data = hdf.get('dataset2')\n", 38 | " dataset1 = np.array(data)\n", 39 | " print('Shape of dataset1: \\n', dataset1.shape)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 48, 45 | "metadata": { 46 | "collapsed": false 47 | }, 48 | "outputs": [ 49 | { 50 | "data": { 51 | "text/plain": [ 52 | "array([[ 0.86222109, 0.81552981, 0.72947893, ..., 0.8151176 ,\n", 53 | " 0.04256893, 0.20956778],\n", 54 | " [ 0.00223081, 0.33763078, 0.16629403, ..., 0.69963576,\n", 55 | " 0.69310324, 0.77839365],\n", 56 | " [ 0.50803133, 0.88046107, 0.49747202, ..., 0.3320066 ,\n", 57 | " 0.75703668, 0.76684165],\n", 58 | " ..., \n", 59 | " [ 0.96940166, 0.28428291, 0.67076274, ..., 0.79801501,\n", 60 | " 0.20872818, 0.28581377],\n", 61 | " [ 0.1356297 , 0.4816769 , 0.08350457, ..., 0.26126381,\n", 62 | " 0.83913621, 0.94482347],\n", 63 | " [ 0.2145087 , 0.21925072, 0.88852823, ..., 0.13799023,\n", 64 | " 0.35711444, 0.97585676]])" 65 | ] 66 | }, 67 | "execution_count": 48, 68 | "metadata": {}, 69 | "output_type": "execute_result" 70 | } 71 | ], 72 | "source": [ 73 | "dataset1" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": 49, 79 | "metadata": { 80 | "collapsed": true 81 | }, 82 | "outputs": [], 83 | "source": [ 84 | "f = h5py.File('/home/noureddin/Desktop/hdf5_data.h5', 'r')\n", 85 | "ls = list(f.keys())\n", 86 | "f.close()" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 50, 92 | "metadata": { 93 | "collapsed": false 94 | }, 95 | "outputs": [ 96 | { 97 | "data": { 98 | "text/plain": [ 99 | "['dataset1', 'dataset2']" 100 | ] 101 | }, 102 | "execution_count": 50, 103 | "metadata": {}, 104 | "output_type": "execute_result" 105 | } 106 | ], 107 | "source": [ 108 | "ls" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": { 115 | "collapsed": false 116 | }, 117 | "outputs": [], 118 | "source": [] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": { 124 | "collapsed": false 125 | }, 126 | "outputs": [], 127 | "source": [] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": { 133 | "collapsed": true 134 | }, 135 | "outputs": [], 136 | "source": [] 137 | } 138 | ], 139 | "metadata": { 140 | "anaconda-cloud": {}, 141 | "kernelspec": { 142 | "display_name": "Python 3", 143 | "language": "python", 144 | "name": "python3" 145 | }, 146 | "language_info": { 147 | "codemirror_mode": { 148 | "name": "ipython", 149 | "version": 3 150 | }, 151 | "file_extension": ".py", 152 | "mimetype": "text/x-python", 153 | "name": "python", 154 | "nbconvert_exporter": "python", 155 | "pygments_lexer": "ipython3", 156 | "version": "3.5.2" 157 | } 158 | }, 159 | "nbformat": 4, 160 | "nbformat_minor": 1 161 | } 162 | -------------------------------------------------------------------------------- /notebooks/05-Creating Groups in HDF5 Files.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 12, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 13, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "matrix1 = np.random.random(size = (1000,1000))\n", 24 | "matrix2 = np.random.random(size = (1000,1000))\n", 25 | "matrix3 = np.random.random(size = (1000,1000))\n", 26 | "matrix4 = np.random.random(size = (1000,1000))" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 14, 32 | "metadata": { 33 | "collapsed": true 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "with h5py.File('/home/noureddin/Desktop/hdf5_groups.h5', 'w') as hdf:\n", 38 | " G1 = hdf.create_group('Group1')\n", 39 | " G1.create_dataset('dataset1', data = matrix1)\n", 40 | " G1.create_dataset('dataset4', data = matrix4)\n", 41 | " \n", 42 | " G21 = hdf.create_group('Group2/SubGroup1')\n", 43 | " G21.create_dataset('dataset3', data = matrix3)\n", 44 | " \n", 45 | " G22 = hdf.create_group('Group2/SubGroup2')\n", 46 | " G22.create_dataset('dataset2', data = matrix2)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": { 53 | "collapsed": false 54 | }, 55 | "outputs": [], 56 | "source": [] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": { 62 | "collapsed": true 63 | }, 64 | "outputs": [], 65 | "source": [] 66 | } 67 | ], 68 | "metadata": { 69 | "anaconda-cloud": {}, 70 | "kernelspec": { 71 | "display_name": "Python 3", 72 | "language": "python", 73 | "name": "python3" 74 | }, 75 | "language_info": { 76 | "codemirror_mode": { 77 | "name": "ipython", 78 | "version": 3 79 | }, 80 | "file_extension": ".py", 81 | "mimetype": "text/x-python", 82 | "name": "python", 83 | "nbconvert_exporter": "python", 84 | "pygments_lexer": "ipython3", 85 | "version": "3.5.2" 86 | } 87 | }, 88 | "nbformat": 4, 89 | "nbformat_minor": 1 90 | } 91 | -------------------------------------------------------------------------------- /notebooks/06-Reading Groups in HDF5 Files.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 13, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 23, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [ 22 | { 23 | "name": "stdout", 24 | "output_type": "stream", 25 | "text": [ 26 | "Items in the base directory: [('Group1', ), ('Group2', )]\n", 27 | "Items in Group2: [('SubGroup1', ), ('SubGroup2', )]\n", 28 | "Items in Group21: [('dataset3', )]\n", 29 | "()\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "with h5py.File('/home/noureddin/Desktop/hdf5_groups.h5','r') as hdf:\n", 35 | " base_items = list(hdf.items())\n", 36 | " print('Items in the base directory:', base_items)\n", 37 | " G2 = hdf.get('Group2')\n", 38 | " G2_items = list(G2.items())\n", 39 | " print('Items in Group2:', G2_items)\n", 40 | " G21 = G2.get('/Group2/SubGroup1')\n", 41 | " G21_items = list(G21.items())\n", 42 | " print('Items in Group21:', G21_items)\n", 43 | " dataset3 = np.array(G21.get('dataset4'))\n", 44 | " print(dataset3.shape)\n" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": { 51 | "collapsed": false 52 | }, 53 | "outputs": [], 54 | "source": [] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": { 60 | "collapsed": true 61 | }, 62 | "outputs": [], 63 | "source": [] 64 | } 65 | ], 66 | "metadata": { 67 | "anaconda-cloud": {}, 68 | "kernelspec": { 69 | "display_name": "Python 3", 70 | "language": "python", 71 | "name": "python3" 72 | }, 73 | "language_info": { 74 | "codemirror_mode": { 75 | "name": "ipython", 76 | "version": 3 77 | }, 78 | "file_extension": ".py", 79 | "mimetype": "text/x-python", 80 | "name": "python", 81 | "nbconvert_exporter": "python", 82 | "pygments_lexer": "ipython3", 83 | "version": "3.5.2" 84 | } 85 | }, 86 | "nbformat": 4, 87 | "nbformat_minor": 1 88 | } 89 | -------------------------------------------------------------------------------- /notebooks/07-HDF5 Compress Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 10, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 11, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "matrix1 = np.random.random(size = (1000,1000))\n", 24 | "matrix2 = np.random.random(size = (1000,1000))\n", 25 | "matrix3 = np.random.random(size = (1000,1000))\n", 26 | "matrix4 = np.random.random(size = (1000,1000))" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 14, 32 | "metadata": { 33 | "collapsed": true 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "with h5py.File('/home/noureddin/Desktop/hdf5_groups_compressed.h5', 'w') as hdf:\n", 38 | " G1 = hdf.create_group('Group1')\n", 39 | " G1.create_dataset('dataset1', data = matrix1, compression=\"gzip\", compression_opts=9)\n", 40 | " G1.create_dataset('dataset4', data = matrix4, compression=\"gzip\", compression_opts=9)\n", 41 | " \n", 42 | " G21 = hdf.create_group('Group2/SubGroup1')\n", 43 | " G21.create_dataset('dataset3', data = matrix3, compression=\"gzip\", compression_opts=9)\n", 44 | " \n", 45 | " G22 = hdf.create_group('Group2/SubGroup2')\n", 46 | " G22.create_dataset('dataset2', data = matrix2, compression=\"gzip\", compression_opts=9)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": { 53 | "collapsed": true 54 | }, 55 | "outputs": [], 56 | "source": [] 57 | } 58 | ], 59 | "metadata": { 60 | "anaconda-cloud": {}, 61 | "kernelspec": { 62 | "display_name": "Python 3", 63 | "language": "python", 64 | "name": "python3" 65 | }, 66 | "language_info": { 67 | "codemirror_mode": { 68 | "name": "ipython", 69 | "version": 3 70 | }, 71 | "file_extension": ".py", 72 | "mimetype": "text/x-python", 73 | "name": "python", 74 | "nbconvert_exporter": "python", 75 | "pygments_lexer": "ipython3", 76 | "version": "3.5.2" 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 1 81 | } 82 | -------------------------------------------------------------------------------- /notebooks/08-HDF5 Attributes.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 33, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import numpy as np\n", 12 | "import h5py" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 34, 18 | "metadata": { 19 | "collapsed": true 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "matrix1 = np.random.random(size = (1000,1000))\n", 24 | "matrix2 = np.random.random(size = (10000,100))" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 35, 30 | "metadata": { 31 | "collapsed": false, 32 | "scrolled": true 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "# Create the HDF5 file\n", 37 | "hdf = h5py.File('/home/noureddin/Desktop/test.h5', 'w')\n", 38 | "\n", 39 | "# Create the datasets\n", 40 | "dataset1 = hdf.create_dataset('dataset1', data=matrix1)\n", 41 | "dataset2 = hdf.create_dataset('dataset2', data=matrix2)\n", 42 | "\n", 43 | "# Set attributes\n", 44 | "dataset1.attrs['CLASS'] = 'DATA MATRIX'\n", 45 | "dataset1.attrs['VERSION'] = '1.1'\n", 46 | "\n", 47 | "hdf.close()" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 36, 53 | "metadata": { 54 | "collapsed": false 55 | }, 56 | "outputs": [ 57 | { 58 | "name": "stdout", 59 | "output_type": "stream", 60 | "text": [ 61 | "List of datasets in this file: \n", 62 | " ['dataset1', 'dataset2']\n", 63 | "Shape of dataset1: \n", 64 | " (1000, 1000)\n", 65 | "CLASS\n", 66 | "DATA MATRIX\n", 67 | "DATA MATRIX\n" 68 | ] 69 | } 70 | ], 71 | "source": [ 72 | "# Read the HDF5 file\n", 73 | "hdf = h5py.File('/home/noureddin/Desktop/test.h5', 'r')\n", 74 | "ls = list(hdf.keys())\n", 75 | "print('List of datasets in this file: \\n', ls)\n", 76 | "data = hdf.get('dataset1')\n", 77 | "dataset1 = np.array(data)\n", 78 | "print('Shape of dataset1: \\n', dataset1.shape)\n", 79 | "#read the attributes\n", 80 | "k = list(data.attrs.keys())\n", 81 | "v = list(data.attrs.values())\n", 82 | "print(k[0])\n", 83 | "print(v[0])\n", 84 | "print(data.attrs[k[0]])\n", 85 | "\n", 86 | "hdf.close()" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": { 93 | "collapsed": false 94 | }, 95 | "outputs": [], 96 | "source": [] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": { 102 | "collapsed": true 103 | }, 104 | "outputs": [], 105 | "source": [] 106 | } 107 | ], 108 | "metadata": { 109 | "anaconda-cloud": {}, 110 | "kernelspec": { 111 | "display_name": "Python 3", 112 | "language": "python", 113 | "name": "python3" 114 | }, 115 | "language_info": { 116 | "codemirror_mode": { 117 | "name": "ipython", 118 | "version": 3 119 | }, 120 | "file_extension": ".py", 121 | "mimetype": "text/x-python", 122 | "name": "python", 123 | "nbconvert_exporter": "python", 124 | "pygments_lexer": "ipython3", 125 | "version": "3.5.2" 126 | } 127 | }, 128 | "nbformat": 4, 129 | "nbformat_minor": 1 130 | } 131 | -------------------------------------------------------------------------------- /notebooks/09-Create HDF5 Files with Pandas.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 68, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 69, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "# creates (or opens in append mode) an hdf5 file\n", 23 | "hdf = pd.HDFStore('/home/noureddin/Desktop/hdf5_pandas.h5')" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 70, 29 | "metadata": { 30 | "collapsed": false 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "df1 = pd.read_csv('/home/noureddin/Downloads/FL_insurance_sample.csv/FL_insurance_sample.csv')# put the dataset in the storage\n", 35 | "hdf.put('DF1', df1, format='table', data_columns=True)" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 71, 41 | "metadata": { 42 | "collapsed": false 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "data = {\n", 47 | " \"city\": [\"Tripoli\", \"Sydney\", \"Tripoli\", \"Rome\", \"Rome\", \"Tripoli\",\"Rome\", \"Sydney\", \"Sydney\"],\n", 48 | " \"rank\": [\"1st\", \"2nd\", \"1st\", \"2nd\", \"1st\", \"2nd\",\"1st\", \"2nd\", \"1st\"], \n", 49 | " \"score1\": [44, 48, 39, 41, 38, 44, 34, 54, 61],\n", 50 | " \"score2\": [67, 63, 55, 70, 64, 77, 45, 66, 72]\n", 51 | " }\n", 52 | " \n", 53 | "df2 = pd.DataFrame(data, columns = ['city', 'rank','score1','score2'])" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 72, 59 | "metadata": { 60 | "collapsed": false 61 | }, 62 | "outputs": [ 63 | { 64 | "data": { 65 | "text/html": [ 66 | "
\n", 67 | "\n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | "
cityrankscore1score2
0Tripoli1st4467
1Sydney2nd4863
2Tripoli1st3955
3Rome2nd4170
4Rome1st3864
5Tripoli2nd4477
6Rome1st3445
7Sydney2nd5466
8Sydney1st6172
\n", 143 | "
" 144 | ], 145 | "text/plain": [ 146 | " city rank score1 score2\n", 147 | "0 Tripoli 1st 44 67\n", 148 | "1 Sydney 2nd 48 63\n", 149 | "2 Tripoli 1st 39 55\n", 150 | "3 Rome 2nd 41 70\n", 151 | "4 Rome 1st 38 64\n", 152 | "5 Tripoli 2nd 44 77\n", 153 | "6 Rome 1st 34 45\n", 154 | "7 Sydney 2nd 54 66\n", 155 | "8 Sydney 1st 61 72" 156 | ] 157 | }, 158 | "execution_count": 72, 159 | "metadata": {}, 160 | "output_type": "execute_result" 161 | } 162 | ], 163 | "source": [ 164 | "df2" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 73, 170 | "metadata": { 171 | "collapsed": false 172 | }, 173 | "outputs": [], 174 | "source": [ 175 | "hdf.put('DF2Key', df2,format='table', data_columns=True)" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 74, 181 | "metadata": { 182 | "collapsed": true 183 | }, 184 | "outputs": [], 185 | "source": [ 186 | "hdf.close() # close the hdf5 file" 187 | ] 188 | }, 189 | { 190 | "cell_type": "code", 191 | "execution_count": null, 192 | "metadata": { 193 | "collapsed": false 194 | }, 195 | "outputs": [], 196 | "source": [] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": null, 201 | "metadata": { 202 | "collapsed": true 203 | }, 204 | "outputs": [], 205 | "source": [] 206 | } 207 | ], 208 | "metadata": { 209 | "anaconda-cloud": {}, 210 | "kernelspec": { 211 | "display_name": "Python 3", 212 | "language": "python", 213 | "name": "python3" 214 | }, 215 | "language_info": { 216 | "codemirror_mode": { 217 | "name": "ipython", 218 | "version": 3 219 | }, 220 | "file_extension": ".py", 221 | "mimetype": "text/x-python", 222 | "name": "python", 223 | "nbconvert_exporter": "python", 224 | "pygments_lexer": "ipython3", 225 | "version": "3.5.2" 226 | } 227 | }, 228 | "nbformat": 4, 229 | "nbformat_minor": 1 230 | } 231 | -------------------------------------------------------------------------------- /notebooks/10-Read HDF5 Files with Pandas.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 41, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import pandas as pd" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 42, 17 | "metadata": { 18 | "collapsed": false 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "# open hdf5 file for reading\n", 23 | "hdf = pd.HDFStore('/home/noureddin/Desktop/hdf5_pandas.h5',mode='r')" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 45, 29 | "metadata": { 30 | "collapsed": false 31 | }, 32 | "outputs": [ 33 | { 34 | "data": { 35 | "text/plain": [ 36 | "['/DF1', '/DF2Key']" 37 | ] 38 | }, 39 | "execution_count": 45, 40 | "metadata": {}, 41 | "output_type": "execute_result" 42 | } 43 | ], 44 | "source": [ 45 | "hdf.keys()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 46, 51 | "metadata": { 52 | "collapsed": false 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "df1 = hdf.get('/DF1')" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": 47, 62 | "metadata": { 63 | "collapsed": false 64 | }, 65 | "outputs": [ 66 | { 67 | "data": { 68 | "text/plain": [ 69 | "pandas.core.frame.DataFrame" 70 | ] 71 | }, 72 | "execution_count": 47, 73 | "metadata": {}, 74 | "output_type": "execute_result" 75 | } 76 | ], 77 | "source": [ 78 | "type(df1)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 49, 84 | "metadata": { 85 | "collapsed": false 86 | }, 87 | "outputs": [ 88 | { 89 | "data": { 90 | "text/html": [ 91 | "
\n", 92 | "\n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | "
policyIDstatecodecountyeq_site_limithu_site_limitfl_site_limitfr_site_limittiv_2011tiv_2012eq_site_deductiblehu_site_deductiblefl_site_deductiblefr_site_deductiblepoint_latitudepoint_longitudelineconstructionpoint_granularity
0119736FLCLAY COUNTY498960.0498960.00498960.0498960.0498960.00792148.900.09979.20.0030.102261-81.711777ResidentialMasonry1
1448094FLCLAY COUNTY1322376.31322376.301322376.31322376.31322376.301438163.570.00.00.0030.063936-81.707664ResidentialMasonry3
2206893FLCLAY COUNTY190724.4190724.40190724.4190724.4190724.40192476.780.00.00.0030.089579-81.700455ResidentialWood1
3333743FLCLAY COUNTY0.079520.760.00.079520.7686854.480.00.00.0030.063236-81.707703ResidentialWood3
4172534FLCLAY COUNTY0.0254281.500.0254281.5254281.50246144.490.00.00.0030.060614-81.702675ResidentialWood1
\n", 224 | "
" 225 | ], 226 | "text/plain": [ 227 | " policyID statecode county eq_site_limit hu_site_limit \\\n", 228 | "0 119736 FL CLAY COUNTY 498960.0 498960.00 \n", 229 | "1 448094 FL CLAY COUNTY 1322376.3 1322376.30 \n", 230 | "2 206893 FL CLAY COUNTY 190724.4 190724.40 \n", 231 | "3 333743 FL CLAY COUNTY 0.0 79520.76 \n", 232 | "4 172534 FL CLAY COUNTY 0.0 254281.50 \n", 233 | "\n", 234 | " fl_site_limit fr_site_limit tiv_2011 tiv_2012 eq_site_deductible \\\n", 235 | "0 498960.0 498960.0 498960.00 792148.90 0.0 \n", 236 | "1 1322376.3 1322376.3 1322376.30 1438163.57 0.0 \n", 237 | "2 190724.4 190724.4 190724.40 192476.78 0.0 \n", 238 | "3 0.0 0.0 79520.76 86854.48 0.0 \n", 239 | "4 0.0 254281.5 254281.50 246144.49 0.0 \n", 240 | "\n", 241 | " hu_site_deductible fl_site_deductible fr_site_deductible point_latitude \\\n", 242 | "0 9979.2 0.0 0 30.102261 \n", 243 | "1 0.0 0.0 0 30.063936 \n", 244 | "2 0.0 0.0 0 30.089579 \n", 245 | "3 0.0 0.0 0 30.063236 \n", 246 | "4 0.0 0.0 0 30.060614 \n", 247 | "\n", 248 | " point_longitude line construction point_granularity \n", 249 | "0 -81.711777 Residential Masonry 1 \n", 250 | "1 -81.707664 Residential Masonry 3 \n", 251 | "2 -81.700455 Residential Wood 1 \n", 252 | "3 -81.707703 Residential Wood 3 \n", 253 | "4 -81.702675 Residential Wood 1 " 254 | ] 255 | }, 256 | "execution_count": 49, 257 | "metadata": {}, 258 | "output_type": "execute_result" 259 | } 260 | ], 261 | "source": [ 262 | "df1.head()" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 50, 268 | "metadata": { 269 | "collapsed": false 270 | }, 271 | "outputs": [], 272 | "source": [ 273 | "hdf.close()" 274 | ] 275 | }, 276 | { 277 | "cell_type": "code", 278 | "execution_count": null, 279 | "metadata": { 280 | "collapsed": true 281 | }, 282 | "outputs": [], 283 | "source": [] 284 | } 285 | ], 286 | "metadata": { 287 | "anaconda-cloud": {}, 288 | "kernelspec": { 289 | "display_name": "Python 3", 290 | "language": "python", 291 | "name": "python3" 292 | }, 293 | "language_info": { 294 | "codemirror_mode": { 295 | "name": "ipython", 296 | "version": 3 297 | }, 298 | "file_extension": ".py", 299 | "mimetype": "text/x-python", 300 | "name": "python", 301 | "nbconvert_exporter": "python", 302 | "pygments_lexer": "ipython3", 303 | "version": "3.5.2" 304 | } 305 | }, 306 | "nbformat": 4, 307 | "nbformat_minor": 1 308 | } 309 | --------------------------------------------------------------------------------