├── .gitignore ├── Makefile ├── README.md ├── airflow ├── code-for-dag │ ├── hive-script.sql │ └── pysparkscript.py └── dags │ ├── __pycache__ │ └── hdfs-hive-spark.cpython-38.pyc │ └── hdfs-hive-spark.py ├── base ├── Dockerfile ├── bde-spark.css ├── entrypoint.sh ├── execute-step.sh ├── finish-step.sh └── wait-for-step.sh ├── codeScript └── pysparkscript.py ├── conf ├── beeline-log4j2.properties ├── hive-env.sh ├── hive-exec-log4j2.properties ├── hive-log4j2.properties ├── hive-site.xml ├── ivysettings.xml └── llap-daemon-log4j2.properties ├── dataForProject ├── breweries.csv ├── dvdrental.rar ├── image │ ├── dvdrental-schema.png │ ├── dvdrental │ │ ├── 3055.dat │ │ ├── 3057.dat │ │ ├── 3059.dat │ │ ├── 3061.dat │ │ ├── 3062.dat │ │ ├── 3063.dat │ │ ├── 3065.dat │ │ ├── 3067.dat │ │ ├── 3069.dat │ │ ├── 3071.dat │ │ ├── 3073.dat │ │ ├── 3075.dat │ │ ├── 3077.dat │ │ ├── 3079.dat │ │ ├── 3081.dat │ │ ├── restore.sql │ │ └── toc.dat │ ├── dvdrentalDW-schema.png │ ├── postgreConf.png │ ├── projectBigdata.jpg │ ├── reportDVD.png │ └── superset.png └── script-init │ ├── createDW-when-db-have-init.sql │ ├── hive-dw-init.txt │ ├── install-jdbc.txt │ └── read-postgres-to-hdfs.txt ├── datanode ├── Dockerfile └── run.sh ├── docker-compose.yaml ├── entrypoint.sh ├── hadoop-hive.env ├── hadoop.env ├── historyserver ├── Dockerfile └── run.sh ├── master ├── Dockerfile ├── README.md └── master.sh ├── namenode ├── Dockerfile └── run.sh ├── nginx ├── Dockerfile ├── bde-hadoop.css ├── default.conf └── materialize.min.css ├── nodemanager ├── Dockerfile └── run.sh ├── postgresql-42.6.0.jar ├── resourcemanager ├── Dockerfile └── run.sh ├── spark_in_action.MD ├── startup.sh ├── superset ├── Dockerfile ├── superset-init.sh └── superset_config.py ├── template ├── java │ ├── Dockerfile │ ├── README.md │ └── template.sh ├── python │ ├── Dockerfile │ ├── README.md │ └── template.sh └── scala │ ├── Dockerfile │ ├── README.md │ ├── build.sbt │ ├── plugins.sbt │ └── template.sh └── worker ├── Dockerfile ├── README.md └── worker.sh /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | airflow/logs 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DOCKER_NETWORK = docker-hadoop_default 2 | ENV_FILE = hadoop.env 3 | current_branch := $(shell git rev-parse --abbrev-ref HEAD) 4 | build: 5 | docker build -t bde2020/hadoop-base:$(current_branch) ./base 6 | docker build -t bde2020/hadoop-namenode:$(current_branch) ./namenode 7 | docker build -t bde2020/hadoop-datanode:$(current_branch) ./datanode 8 | docker build -t bde2020/hadoop-resourcemanager:$(current_branch) ./resourcemanager 9 | docker build -t bde2020/hadoop-nodemanager:$(current_branch) ./nodemanager 10 | docker build -t bde2020/hadoop-historyserver:$(current_branch) ./historyserver 11 | docker build -t bde2020/hadoop-submit:$(current_branch) ./submit 12 | docker build -t bde2020/hive:$(current_branch) ./ 13 | 14 | wordcount: 15 | docker build -t hadoop-wordcount ./submit 16 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ 17 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/ 18 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount 19 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* 20 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output 21 | docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /input 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DvD Rental Pipeline and Analytics Stack with Docker multi-container environment 2 | 3 | This project demonstrates the design and implementation of a data pipeline and analytics stack for processing, storing, and visualizing data. And hands-on Big Data solution with batch processing. 4 | 5 | ### Table of contents 6 | 7 | * 🏛️[Architecture diagram](#architecture-diagram) 8 | * 📖[Overview](#overview) 9 | * ⚙️[How it works](#how-it-works) 10 | * 🛠️[Prerequisites](#prerequisites) 11 | * 🚀[Quick Start](#quick-start) 12 | * [🐘PgAdmin4](#pgadmin4) 13 | * [📁HDFS](#hdfs) 14 | * [✨Spark](#spark) 15 | * [🐝Hive](#hive) 16 | * [📊Superset](#superset) 17 | * ⚙️🔧[Configure Environment Variables](#configure-environment-variables) 18 | * 📚[References](#references) 19 | * 📧[Contact](#contact) 20 | 21 | 22 | ## 🏛️Architecture diagram 23 | 24 | ![](./dataForProject/image/projectBigdata.jpg) 25 | 26 | ## 📖Overview 27 | ### Database Schema 28 | ![](./dataForProject/image/dvdrental-schema.png) 29 | ### Dimensional Modeling with Sales 30 | ![](./dataForProject/image/dvdrentalDW-schema.png) 31 | ### Building report 32 | ![](./dataForProject/image/reportDVD.png) 33 | 34 | ## ⚙️How it works 35 | * Data Sources: 36 | 37 | OLTP PostgreSQL Database: The primary source of transactional data. 38 | 39 | Flat Files: Additional data sources in various formats. 40 | * Data Ingestion: 41 | 42 | Apache Hadoop HDFS: Serves as the data lake for ingesting raw data. 43 | 44 | * Data Transformation: 45 | 46 | Apache Hive: The data warehouse for structured data storage and transformation. 47 | 48 | Apache Spark: Powers ETL (Extract, Transform, Load) processes for data processing. 49 | * Data Modeling: 50 | 51 | Hive QL: Used for defining and applying data models on top of the structured data. 52 | * Data Visualization: 53 | 54 | Apache Superset: Provides a rich and interactive interface for data exploration and visualization. 55 | * Orchestration and Workflow: 56 | 57 | Apache Airflow: Manages and schedules data pipeline workflows. 58 | * Docker: 59 | 60 | Containerizes and isolates components for easy deployment and scalability. 61 | 62 | 63 | ## 🛠️Prerequisites 64 | What you need to run the project: 65 | - [Docker](https://www.docker.com/) - I suggest using Docker 4.22.x or above. 66 | - [Apache Component](https://www.apache.org/) - This project is kind of based on Apache component like HDFS, Hive, Spark,... so I would highly recommend you to take a look each component's docs. 67 | - [OLAP Dimemsional Modeling](https://www.youtube.com/watch?v=lWPiSZf7-uQ) - In this project I try to understand a completely data pipeline from source to visualize, and Modeling DataWwarehouse too, and there a video that cover a lot of things about DW and you can try [it](https://www.youtube.com/watch?v=lWPiSZf7-uQ). 68 | - [PgAdmin4](https://www.pgadmin.org/download/) - This is the most popular and feature rich Open Source administration and development platform for PostgreSQL, we use that to create our OLTP database for ease of using. 69 | - [Airflow Orchestration](https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html) - I haven't done with Airflow services yet, because my laptop can't handle that much containers, but don't worry I will update ASAP. 70 | 71 | ## 🚀Quick Start 72 | 73 | Use git clone to clone this projetct: 74 | ``` 75 | git clone https://github.com/DucAnhNTT/bigdata-ETL-pipeline.git 76 | ``` 77 | 78 | 79 | To deploy an the cluster, run: 80 | ``` 81 | docker-compose up 82 | ``` 83 | 84 | `docker-compose` creates a docker network that can be found by running `docker network list`. 85 | 86 | Run `docker network inspect` on the network to find the IP the hadoop interfaces are published on. Access these interfaces with the following URLs: 87 | 88 | There's up to 19 containers, so it can consume about 6GB RAM or more, that much containers requiring a lot of configuration and reading documents. This project is developing based on Marcel Jan's [project](https://github.com/DucAnhNTT/docker-hadoop-spark) 89 | 90 | The stack includes the following components and technologies: 91 | 92 | * Namenode: http://:9870/dfshealth.html#tab-overview 93 | * History server: http://:8188/applicationhistory 94 | * Datanode: http://:9864/ 95 | * Nodemanager: http://:8042/node 96 | * Resource manager: http://:8088/ 97 | * Spark master: http://:8080/ 98 | * Spark worker: http://:8081/ 99 | * Hive: http://:10001 100 | * Airflow Webserver: http://:8082 101 | * Superset: http://:8088 102 | * Postgres (also metastore for Airflow): http://:5435 103 | 104 | 105 | ### 🐘PgAdmin4: Inital Database 106 | Install and ensure that Postgres container also running, then you can connect to that Postgres using PgAdmin4 interface. 107 | 108 | ![](./dataForProject/image/postgreConf.png) 109 | 1. Then you need to create a database 110 | 2. Create role postgres 111 | 3. Extract the dvdrental.rar 112 | 4. Restore that database with the option Restore from File->"./dataForProject/dvdrental". 113 | 5. Now your own database to play with, and the schema is above. 114 | 115 | Continue, I need you to create the DW based on that database, There's lot of way to do that, but in this project and faster way you can create the DW Dimensional Model inside the database, and then we dump that data to HDFS. 116 | 117 | So, in the Postgres database, you can create the script and run that one I already give you in this path "./dataForProject/script-init/createDW-when-db-have-init.sql" 118 | 119 | Following Demo [here](https://youtu.be/L6IYB6vImZA). 120 | 121 | ### 📁HDFS: Initialize our Datalake folder in HDFS 122 | Go to the bash shell on the namenode with that same Container ID of the namenode. 123 | ``` 124 | docker exec -it namenode bash 125 | ``` 126 | 127 | 128 | Create a HDFS directory /data/staging. 129 | 130 | ``` 131 | hdfs dfs -mkdir -p /data/staging 132 | ``` 133 | Following Demo [here](https://www.youtube.com/watch?v=_f-eyajCySY&list=PLId1IInL1turLaBYUjjWeEM1z3ZCTMY7d). 134 | 135 | ### ✨Spark: ETL data from DBMS to Datalake (HDFS) 136 | 137 | Move on, go to your CMD and make sure your Spark cluster have the Postgres JDBC pre-installed and compatible with each other, I have researched and use it smoothly, all you need to do is copy the postgresql-42.6.0.jar to each Spark workers and Spark master. (TBH, you can run my script "install-jdbc.txt" i give you in the text file and run it in your host CMD) 138 | 139 | 140 | 141 | Go to http://:8080 or http://localhost:8080/ on your Docker host (laptop) to see the status of the Spark master. 142 | 143 | Go to the command line of the Spark master and start spark-shell. 144 | ``` 145 | docker exec -it spark-master bash 146 | 147 | spark/bin/spark-shell --master spark://spark-master:7077 148 | ``` 149 | run the code I provide you in this directory "./dataForProject/script-init/read-postgres-to-hdfs.txt", copy and past to the terminal. REMEMBER TO RESTART CONTAINERS AFTER YOU COPY THE FILE! 150 | 151 | ``` 152 | docker cp postgresql-42.6.0.jar spark-master:spark/jars 153 | docker cp postgresql-42.6.0.jar spark-worker-1:spark/jars 154 | ``` 155 | 156 | 157 | Following Demo [here](https://www.youtube.com/watch?v=_f-eyajCySY&list=PLId1IInL1turLaBYUjjWeEM1z3ZCTMY7d). 158 | ### 🐝Hive: Initialize our Data Warehouse 159 | 160 | Go to the command line of the Hive server and start hiveserver2 161 | 162 | ``` 163 | docker exec -it hive-server bash 164 | 165 | hiveserver2 166 | ``` 167 | 168 | Maybe a little check that something is listening on port 10000 now 169 | ``` 170 | netstat -anp | grep 10000 171 | tcp 0 0 0.0.0.0:10000 0.0.0.0:* LISTEN 446/java 172 | 173 | ``` 174 | 175 | Okay. Beeline is the command line interface with Hive. Let's connect to hiveserver2 now. 176 | 177 | ``` 178 | beeline -u jdbc:hive2://localhost:10000 -n root 179 | 180 | !connect jdbc:hive2://127.0.0.1:10000 scott tiger 181 | ``` 182 | 183 | Didn't expect to encounter scott/tiger again after my Oracle days. But there you have it. Definitely not a good idea to keep that user on production. 184 | 185 | Not a lot of databases here yet. 186 | ``` 187 | show databases; 188 | 189 | +----------------+ 190 | | database_name | 191 | +----------------+ 192 | | default | 193 | +----------------+ 194 | 1 row selected (0.335 seconds) 195 | ``` 196 | 197 | Let's change that. 198 | 199 | ``` 200 | create database dvdrentaldw; 201 | use dvdrentaldw; 202 | ``` 203 | 204 | And let's create a table. 205 | 206 | Now the last thing, run the script "./dataForProject/script-init/hive-dw-init.txt", to create DW in Hive, ATTETION!: In Hive, primary keys and foreign keys are not natively supported, as it prioritizes large-scale data processing and querying over enforcing traditional relational constraints. Hive tables do not enforce primary key uniqueness or foreign key relationships. Data quality and consistency checks are typically handled upstream before data is loaded into Hive. 207 | 208 | Following Demo [here](https://www.youtube.com/watch?v=_f-eyajCySY&list=PLId1IInL1turLaBYUjjWeEM1z3ZCTMY7d). 209 | 210 | ### 📊Superset: Visualize our DW to take insight 211 | Last thing, we want to visualize the report for Business Insight or make report on top of our Data warehouse (Hive), so connect to Hive with Superset at: http://localhost:8088. 212 | 213 | First thing, you want to connect to Hive: 214 | ![Alt text](dataForProject/image/superset.png) 215 | 216 | Now, feel free to unleash your creativity in analyzing and exploring the data. Dive deep into the insights, create compelling visualizations, and generate comprehensive reports. With Superset connected to Hive, you have a powerful tool at your disposal to drive your business forward. 217 | Following Demo [here](https://www.youtube.com/watch?v=_f-eyajCySY&list=PLId1IInL1turLaBYUjjWeEM1z3ZCTMY7d). 218 | ## ⚙️🔧Configure Environment Variables 219 | 220 | The configuration parameters can be specified in the hadoop.env file or as environmental variables for specific services (e.g. namenode, datanode etc.): 221 | ``` 222 | CORE_CONF_fs_defaultFS=hdfs://namenode:9000 223 | ``` 224 | 225 | CORE_CONF corresponds to core-site.xml. fs_defaultFS=hdfs://namenode:9000 will be transformed into: 226 | ``` 227 | fs.defaultFShdfs://namenode:9000 228 | ``` 229 | To define dash inside a configuration parameter, use triple underscore, such as YARN_CONF_yarn_log___aggregation___enable=true (yarn-site.xml): 230 | ``` 231 | yarn.log-aggregation-enabletrue 232 | ``` 233 | 234 | The available configurations are: 235 | * /etc/hadoop/core-site.xml CORE_CONF 236 | * /etc/hadoop/hdfs-site.xml HDFS_CONF 237 | * /etc/hadoop/yarn-site.xml YARN_CONF 238 | * /etc/hadoop/httpfs-site.xml HTTPFS_CONF 239 | * /etc/hadoop/kms-site.xml KMS_CONF 240 | * /etc/hadoop/mapred-site.xml MAPRED_CONF 241 | 242 | If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. 243 | 244 | ## 📚References 245 | Inspired by following codes, articles and videos: 246 | 247 | * [Docker's document](https://www.linkedin.com/pulse/dockerizing-hadoop-hive-spark-sqoop-job-thomas/) 248 | * [Configuration containers](https://www.linkedin.com/pulse/dockerizing-hadoop-hive-spark-sqoop-job-thomas/) 249 | * [Furthermore of docker](https://docs.docker.com/desktop/networking/) 250 | 251 | 252 | ## 📧Contact 253 | Please feel free to contact me if you have any questions. 254 | ducanh0285@gmail.com111 255 | -------------------------------------------------------------------------------- /airflow/code-for-dag/hive-script.sql: -------------------------------------------------------------------------------- 1 | create database openbeer; 2 | use openbeer; 3 | 4 | CREATE EXTERNAL TABLE IF NOT EXISTS breweries( 5 | NUM INT, 6 | NAME CHAR(100), 7 | CITY CHAR(100), 8 | STATE CHAR(100), 9 | ID INT ) 10 | ROW FORMAT DELIMITED 11 | FIELDS TERMINATED BY ',' 12 | STORED AS TEXTFILE 13 | location '/data/openbeer/breweries'; -------------------------------------------------------------------------------- /airflow/code-for-dag/pysparkscript.py: -------------------------------------------------------------------------------- 1 | from pyspark import SparkContext, SparkConf 2 | 3 | # Create a Spark configuration and SparkContext 4 | conf = SparkConf().setAppName("breweries") 5 | sc = SparkContext(conf=conf) 6 | 7 | # Load a text file from HDFS (or your preferred data source) 8 | brewfile = sc.read.csv("hdfs://namenode:9000/data/openbeer/breweries/breweries.csv") 9 | 10 | df = sc.write.csv("hdfs://namenode:9000/data/openbeer/breweries/testbreweries.csv") 11 | 12 | sc.stop() 13 | -------------------------------------------------------------------------------- /airflow/dags/__pycache__/hdfs-hive-spark.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/airflow/dags/__pycache__/hdfs-hive-spark.cpython-38.pyc -------------------------------------------------------------------------------- /airflow/dags/hdfs-hive-spark.py: -------------------------------------------------------------------------------- 1 | from airflow import DAG 2 | from airflow.operators.bash_operator import BashOperator 3 | from airflow.operators.dummy_operator import DummyOperator 4 | from airflow.operators.python_operator import PythonOperator 5 | from airflow.providers.docker.operators.docker import DockerOperator # Import DockerOperator 6 | from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator 7 | from datetime import datetime 8 | import subprocess 9 | 10 | # Define your DAG 11 | dag = DAG( 12 | 'airflow_test_hadoop_cluster', 13 | default_args={ 14 | 'owner': 'your_name', 15 | 'start_date': datetime(2023, 1, 1), 16 | # Add other default arguments as needed 17 | }, 18 | schedule_interval=None, # Set your desired schedule interval 19 | catchup=False # Set to True if you want historical DAG runs to be processed 20 | ) 21 | 22 | 23 | 24 | # Dummy task to start the workflow 25 | start_task = DummyOperator(task_id='start', dag=dag) 26 | 27 | 28 | # SparkSubmitOperator to load breweries.csv from HDFS into Spark 29 | load_into_spark_task = SparkSubmitOperator( 30 | task_id='load_into_spark', 31 | application='/airflow/code-for-dag/pysparkscript.py', # Path to your Spark script 32 | conn_id='http://localhost:8080/', # Specify your Spark connection 33 | conf={ 34 | 'spark.driver.memory': '2g', # Customize Spark configuration as needed 35 | 'spark.executor.memory': '2g', 36 | }, 37 | dag=dag 38 | ) 39 | 40 | # Bash Operator to load breweries.csv from HDFS into Hive 41 | load_into_hive_task = BashOperator( 42 | task_id='load_into_hive', 43 | bash_command='beeline -u jdbc:hive2://localhost:10000 -n root -f /airflow/code-for-dag/hive-script.sql', 44 | dag=dag 45 | ) 46 | 47 | # Dummy task to end the workflow 48 | end_task = DummyOperator(task_id='end', dag=dag) 49 | 50 | # Define the task dependencies 51 | start_task >> load_into_spark_task >> load_into_hive_task >> end_task 52 | -------------------------------------------------------------------------------- /base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:9 2 | 3 | MAINTAINER Ivan Ermilov 4 | MAINTAINER Giannis Mouchakis 5 | 6 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 7 | openjdk-8-jdk \ 8 | net-tools \ 9 | curl \ 10 | netcat \ 11 | gnupg \ 12 | libsnappy-dev \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ 16 | 17 | RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS 18 | 19 | RUN gpg --import KEYS 20 | 21 | ENV HADOOP_VERSION 3.2.1 22 | ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz 23 | 24 | RUN set -x \ 25 | && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \ 26 | && curl -fSL "$HADOOP_URL.asc" -o /tmp/hadoop.tar.gz.asc \ 27 | && gpg --verify /tmp/hadoop.tar.gz.asc \ 28 | && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ 29 | && rm /tmp/hadoop.tar.gz* 30 | 31 | RUN ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop 32 | 33 | RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs 34 | 35 | RUN mkdir /hadoop-data 36 | 37 | ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION 38 | ENV HADOOP_CONF_DIR=/etc/hadoop 39 | ENV MULTIHOMED_NETWORK=1 40 | ENV USER=root 41 | ENV PATH $HADOOP_HOME/bin/:$PATH 42 | 43 | ADD entrypoint.sh /entrypoint.sh 44 | 45 | RUN chmod a+x /entrypoint.sh 46 | 47 | ENTRYPOINT ["/entrypoint.sh"] 48 | -------------------------------------------------------------------------------- /base/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Set some sensible defaults 4 | export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} 5 | 6 | function addProperty() { 7 | local path=$1 8 | local name=$2 9 | local value=$3 10 | 11 | local entry="$name${value}" 12 | local escapedEntry=$(echo $entry | sed 's/\//\\\//g') 13 | sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path 14 | } 15 | 16 | function configure() { 17 | local path=$1 18 | local module=$2 19 | local envPrefix=$3 20 | 21 | local var 22 | local value 23 | 24 | echo "Configuring $module" 25 | for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do 26 | name=`echo ${c} | perl -pe 's/___/-/g; s/__/@/g; s/_/./g; s/@/_/g;'` 27 | var="${envPrefix}_${c}" 28 | value=${!var} 29 | echo " - Setting $name=$value" 30 | addProperty $path $name "$value" 31 | done 32 | } 33 | 34 | configure /etc/hadoop/core-site.xml core CORE_CONF 35 | configure /etc/hadoop/hdfs-site.xml hdfs HDFS_CONF 36 | configure /etc/hadoop/yarn-site.xml yarn YARN_CONF 37 | configure /etc/hadoop/httpfs-site.xml httpfs HTTPFS_CONF 38 | configure /etc/hadoop/kms-site.xml kms KMS_CONF 39 | configure /etc/hadoop/mapred-site.xml mapred MAPRED_CONF 40 | 41 | if [ "$MULTIHOMED_NETWORK" = "1" ]; then 42 | echo "Configuring for multihomed network" 43 | 44 | # HDFS 45 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 46 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.servicerpc-bind-host 0.0.0.0 47 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.http-bind-host 0.0.0.0 48 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.https-bind-host 0.0.0.0 49 | addProperty /etc/hadoop/hdfs-site.xml dfs.client.use.datanode.hostname true 50 | addProperty /etc/hadoop/hdfs-site.xml dfs.datanode.use.datanode.hostname true 51 | 52 | # YARN 53 | addProperty /etc/hadoop/yarn-site.xml yarn.resourcemanager.bind-host 0.0.0.0 54 | addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 55 | addProperty /etc/hadoop/yarn-site.xml yarn.timeline-service.bind-host 0.0.0.0 56 | 57 | # MAPRED 58 | addProperty /etc/hadoop/mapred-site.xml yarn.nodemanager.bind-host 0.0.0.0 59 | fi 60 | 61 | if [ -n "$GANGLIA_HOST" ]; then 62 | mv /etc/hadoop/hadoop-metrics.properties /etc/hadoop/hadoop-metrics.properties.orig 63 | mv /etc/hadoop/hadoop-metrics2.properties /etc/hadoop/hadoop-metrics2.properties.orig 64 | 65 | for module in mapred jvm rpc ugi; do 66 | echo "$module.class=org.apache.hadoop.metrics.ganglia.GangliaContext31" 67 | echo "$module.period=10" 68 | echo "$module.servers=$GANGLIA_HOST:8649" 69 | done > /etc/hadoop/hadoop-metrics.properties 70 | 71 | for module in namenode datanode resourcemanager nodemanager mrappmaster jobhistoryserver; do 72 | echo "$module.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31" 73 | echo "$module.sink.ganglia.period=10" 74 | echo "$module.sink.ganglia.supportsparse=true" 75 | echo "$module.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both" 76 | echo "$module.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40" 77 | echo "$module.sink.ganglia.servers=$GANGLIA_HOST:8649" 78 | done > /etc/hadoop/hadoop-metrics2.properties 79 | fi 80 | 81 | function wait_for_it() 82 | { 83 | local serviceport=$1 84 | local service=${serviceport%%:*} 85 | local port=${serviceport#*:} 86 | local retry_seconds=5 87 | local max_try=100 88 | let i=1 89 | 90 | nc -z $service $port 91 | result=$? 92 | 93 | until [ $result -eq 0 ]; do 94 | echo "[$i/$max_try] check for ${service}:${port}..." 95 | echo "[$i/$max_try] ${service}:${port} is not available yet" 96 | if (( $i == $max_try )); then 97 | echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" 98 | exit 1 99 | fi 100 | 101 | echo "[$i/$max_try] try in ${retry_seconds}s once again ..." 102 | let "i++" 103 | sleep $retry_seconds 104 | 105 | nc -z $service $port 106 | result=$? 107 | done 108 | echo "[$i/$max_try] $service:${port} is available." 109 | } 110 | 111 | for i in ${SERVICE_PRECONDITION[@]} 112 | do 113 | wait_for_it ${i} 114 | done 115 | 116 | exec $@ 117 | -------------------------------------------------------------------------------- /base/execute-step.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $ENABLE_INIT_DAEMON = "true" ] 4 | then 5 | echo "Execute step ${INIT_DAEMON_STEP} in pipeline" 6 | while true; do 7 | sleep 5 8 | echo -n '.' 9 | string=$(curl -sL -w "%{http_code}" -X PUT $INIT_DAEMON_BASE_URI/execute?step=$INIT_DAEMON_STEP -o /dev/null) 10 | [ "$string" = "204" ] && break 11 | done 12 | echo "Notified execution of step ${INIT_DAEMON_STEP}" 13 | fi 14 | 15 | -------------------------------------------------------------------------------- /base/finish-step.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $ENABLE_INIT_DAEMON = "true" ] 4 | then 5 | echo "Finish step ${INIT_DAEMON_STEP} in pipeline" 6 | while true; do 7 | sleep 5 8 | echo -n '.' 9 | string=$(curl -sL -w "%{http_code}" -X PUT $INIT_DAEMON_BASE_URI/finish?step=$INIT_DAEMON_STEP -o /dev/null) 10 | [ "$string" = "204" ] && break 11 | done 12 | echo "Notified finish of step ${INIT_DAEMON_STEP}" 13 | fi 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /base/wait-for-step.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $ENABLE_INIT_DAEMON = "true" ] 4 | then 5 | echo "Validating if step ${INIT_DAEMON_STEP} can start in pipeline" 6 | while true; do 7 | sleep 5 8 | echo -n '.' 9 | string=$(curl -s $INIT_DAEMON_BASE_URI/canStart?step=$INIT_DAEMON_STEP) 10 | [ "$string" = "true" ] && break 11 | done 12 | echo "Can start step ${INIT_DAEMON_STEP}" 13 | fi 14 | -------------------------------------------------------------------------------- /codeScript/pysparkscript.py: -------------------------------------------------------------------------------- 1 | from pyspark import SparkContext, SparkConf 2 | 3 | # Create a Spark configuration and SparkContext 4 | conf = SparkConf().setAppName("breweries") 5 | sc = SparkContext(conf=conf) 6 | 7 | # Load a text file from HDFS (or your preferred data source) 8 | brewfile = sc.read.csv("hdfs://namenode:9000/data/openbeer/breweries/breweries.csv") 9 | 10 | df = sc.write.csv("hdfs://namenode:9000/data/openbeer/breweries/testbreweries.csv") 11 | 12 | sc.stop() 13 | 14 | -------------------------------------------------------------------------------- /conf/beeline-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | status = INFO 18 | name = BeelineLog4j2 19 | packages = org.apache.hadoop.hive.ql.log 20 | 21 | # list of properties 22 | property.hive.log.level = WARN 23 | property.hive.root.logger = console 24 | 25 | # list of all appenders 26 | appenders = console 27 | 28 | # console appender 29 | appender.console.type = Console 30 | appender.console.name = console 31 | appender.console.target = SYSTEM_ERR 32 | appender.console.layout.type = PatternLayout 33 | appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n 34 | 35 | # list of all loggers 36 | loggers = HiveConnection 37 | 38 | # HiveConnection logs useful info for dynamic service discovery 39 | logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection 40 | logger.HiveConnection.level = INFO 41 | 42 | # root logger 43 | rootLogger.level = ${sys:hive.log.level} 44 | rootLogger.appenderRefs = root 45 | rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} 46 | -------------------------------------------------------------------------------- /conf/hive-env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Set Hive and Hadoop environment variables here. These variables can be used 18 | # to control the execution of Hive. It should be used by admins to configure 19 | # the Hive installation (so that users do not have to set environment variables 20 | # or set command line parameters to get correct behavior). 21 | # 22 | # The hive service being invoked (CLI/HWI etc.) is available via the environment 23 | # variable SERVICE 24 | 25 | 26 | # Hive Client memory usage can be an issue if a large number of clients 27 | # are running at the same time. The flags below have been useful in 28 | # reducing memory usage: 29 | # 30 | # if [ "$SERVICE" = "cli" ]; then 31 | # if [ -z "$DEBUG" ]; then 32 | # export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" 33 | # else 34 | # export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" 35 | # fi 36 | # fi 37 | 38 | # The heap size of the jvm stared by hive shell script can be controlled via: 39 | # 40 | # export HADOOP_HEAPSIZE=1024 41 | # 42 | # Larger heap size may be required when running queries over large number of files or partitions. 43 | # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be 44 | # appropriate for hive server (hwi etc). 45 | 46 | 47 | # Set HADOOP_HOME to point to a specific hadoop install directory 48 | # HADOOP_HOME=${bin}/../../hadoop 49 | 50 | # Hive Configuration Directory can be controlled by: 51 | # export HIVE_CONF_DIR= 52 | 53 | # Folder containing extra ibraries required for hive compilation/execution can be controlled by: 54 | # export HIVE_AUX_JARS_PATH= 55 | 56 | 57 | -------------------------------------------------------------------------------- /conf/hive-exec-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | status = INFO 18 | name = HiveExecLog4j2 19 | packages = org.apache.hadoop.hive.ql.log 20 | 21 | # list of properties 22 | property.hive.log.level = INFO 23 | property.hive.root.logger = FA 24 | property.hive.query.id = hadoop 25 | property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} 26 | property.hive.log.file = ${sys:hive.query.id}.log 27 | 28 | # list of all appenders 29 | appenders = console, FA 30 | 31 | # console appender 32 | appender.console.type = Console 33 | appender.console.name = console 34 | appender.console.target = SYSTEM_ERR 35 | appender.console.layout.type = PatternLayout 36 | appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n 37 | 38 | # simple file appender 39 | appender.FA.type = File 40 | appender.FA.name = FA 41 | appender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} 42 | appender.FA.layout.type = PatternLayout 43 | appender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n 44 | 45 | # list of all loggers 46 | loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX 47 | 48 | logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn 49 | logger.NIOServerCnxn.level = WARN 50 | 51 | logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO 52 | logger.ClientCnxnSocketNIO.level = WARN 53 | 54 | logger.DataNucleus.name = DataNucleus 55 | logger.DataNucleus.level = ERROR 56 | 57 | logger.Datastore.name = Datastore 58 | logger.Datastore.level = ERROR 59 | 60 | logger.JPOX.name = JPOX 61 | logger.JPOX.level = ERROR 62 | 63 | # root logger 64 | rootLogger.level = ${sys:hive.log.level} 65 | rootLogger.appenderRefs = root 66 | rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} 67 | -------------------------------------------------------------------------------- /conf/hive-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | status = INFO 18 | name = HiveLog4j2 19 | packages = org.apache.hadoop.hive.ql.log 20 | 21 | # list of properties 22 | property.hive.log.level = INFO 23 | property.hive.root.logger = DRFA 24 | property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} 25 | property.hive.log.file = hive.log 26 | 27 | # list of all appenders 28 | appenders = console, DRFA 29 | 30 | # console appender 31 | appender.console.type = Console 32 | appender.console.name = console 33 | appender.console.target = SYSTEM_ERR 34 | appender.console.layout.type = PatternLayout 35 | appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n 36 | 37 | # daily rolling file appender 38 | appender.DRFA.type = RollingFile 39 | appender.DRFA.name = DRFA 40 | appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} 41 | # Use %pid in the filePattern to append @ to the filename if you want separate log files for different CLI session 42 | appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} 43 | appender.DRFA.layout.type = PatternLayout 44 | appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n 45 | appender.DRFA.policies.type = Policies 46 | appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy 47 | appender.DRFA.policies.time.interval = 1 48 | appender.DRFA.policies.time.modulate = true 49 | appender.DRFA.strategy.type = DefaultRolloverStrategy 50 | appender.DRFA.strategy.max = 30 51 | 52 | # list of all loggers 53 | loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX 54 | 55 | logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn 56 | logger.NIOServerCnxn.level = WARN 57 | 58 | logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO 59 | logger.ClientCnxnSocketNIO.level = WARN 60 | 61 | logger.DataNucleus.name = DataNucleus 62 | logger.DataNucleus.level = ERROR 63 | 64 | logger.Datastore.name = Datastore 65 | logger.Datastore.level = ERROR 66 | 67 | logger.JPOX.name = JPOX 68 | logger.JPOX.level = ERROR 69 | 70 | # root logger 71 | rootLogger.level = ${sys:hive.log.level} 72 | rootLogger.appenderRefs = root 73 | rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} 74 | -------------------------------------------------------------------------------- /conf/hive-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | hive.execution.engine 4 | spark 5 | 6 | 7 | 8 | 24 | 25 | -------------------------------------------------------------------------------- /conf/ivysettings.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /conf/llap-daemon-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | status = INFO 18 | name = LlapDaemonLog4j2 19 | packages = org.apache.hadoop.hive.ql.log 20 | 21 | # list of properties 22 | property.llap.daemon.log.level = INFO 23 | property.llap.daemon.root.logger = console 24 | property.llap.daemon.log.dir = . 25 | property.llap.daemon.log.file = llapdaemon.log 26 | property.llap.daemon.historylog.file = llapdaemon_history.log 27 | property.llap.daemon.log.maxfilesize = 256MB 28 | property.llap.daemon.log.maxbackupindex = 20 29 | 30 | # list of all appenders 31 | appenders = console, RFA, HISTORYAPPENDER 32 | 33 | # console appender 34 | appender.console.type = Console 35 | appender.console.name = console 36 | appender.console.target = SYSTEM_ERR 37 | appender.console.layout.type = PatternLayout 38 | appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t%x] %p %c{2} : %m%n 39 | 40 | # rolling file appender 41 | appender.RFA.type = RollingFile 42 | appender.RFA.name = RFA 43 | appender.RFA.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file} 44 | appender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%i 45 | appender.RFA.layout.type = PatternLayout 46 | appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t%x]: %c{2} (%F:%M(%L)) - %m%n 47 | appender.RFA.policies.type = Policies 48 | appender.RFA.policies.size.type = SizeBasedTriggeringPolicy 49 | appender.RFA.policies.size.size = ${sys:llap.daemon.log.maxfilesize} 50 | appender.RFA.strategy.type = DefaultRolloverStrategy 51 | appender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex} 52 | 53 | # history file appender 54 | appender.HISTORYAPPENDER.type = RollingFile 55 | appender.HISTORYAPPENDER.name = HISTORYAPPENDER 56 | appender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file} 57 | appender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%i 58 | appender.HISTORYAPPENDER.layout.type = PatternLayout 59 | appender.HISTORYAPPENDER.layout.pattern = %m%n 60 | appender.HISTORYAPPENDER.policies.type = Policies 61 | appender.HISTORYAPPENDER.policies.size.type = SizeBasedTriggeringPolicy 62 | appender.HISTORYAPPENDER.policies.size.size = ${sys:llap.daemon.log.maxfilesize} 63 | appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy 64 | appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex} 65 | 66 | # list of all loggers 67 | loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger 68 | 69 | logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn 70 | logger.NIOServerCnxn.level = WARN 71 | 72 | logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO 73 | logger.ClientCnxnSocketNIO.level = WARN 74 | 75 | logger.DataNucleus.name = DataNucleus 76 | logger.DataNucleus.level = ERROR 77 | 78 | logger.Datastore.name = Datastore 79 | logger.Datastore.level = ERROR 80 | 81 | logger.JPOX.name = JPOX 82 | logger.JPOX.level = ERROR 83 | 84 | logger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger 85 | logger.HistoryLogger.level = INFO 86 | logger.HistoryLogger.additivity = false 87 | logger.HistoryLogger.appenderRefs = HistoryAppender 88 | logger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER 89 | 90 | # root logger 91 | rootLogger.level = ${sys:llap.daemon.log.level} 92 | rootLogger.appenderRefs = root 93 | rootLogger.appenderRef.root.ref = ${sys:llap.daemon.root.logger} 94 | -------------------------------------------------------------------------------- /dataForProject/breweries.csv: -------------------------------------------------------------------------------- 1 | ,name,city,state,id 2 | 0,NorthGate Brewing ,Minneapolis, MN,0 3 | 1,Against the Grain Brewery,Louisville, KY,1 4 | 2,Jack's Abby Craft Lagers,Framingham, MA,2 5 | 3,Mike Hess Brewing Company,San Diego, CA,3 6 | 4,Fort Point Beer Company,San Francisco, CA,4 7 | 5,COAST Brewing Company,Charleston, SC,5 8 | 6,Great Divide Brewing Company,Denver, CO,6 9 | 7,Tapistry Brewing,Bridgman, MI,7 10 | 8,Big Lake Brewing,Holland, MI,8 11 | 9,The Mitten Brewing Company,Grand Rapids, MI,9 12 | 10,Brewery Vivant,Grand Rapids, MI,10 13 | 11,Petoskey Brewing,Petoskey, MI,11 14 | 12,Blackrocks Brewery,Marquette, MI,12 15 | 13,Perrin Brewing Company,Comstock Park, MI,13 16 | 14,Witch's Hat Brewing Company,South Lyon, MI,14 17 | 15,Founders Brewing Company,Grand Rapids, MI,15 18 | 16,Flat 12 Bierwerks,Indianapolis, IN,16 19 | 17,Tin Man Brewing Company,Evansville, IN,17 20 | 18,Black Acre Brewing Co.,Indianapolis, IN,18 21 | 19,Brew Link Brewing,Plainfield, IN,19 22 | 20,Bare Hands Brewery,Granger, IN,20 23 | 21,Three Pints Brewing,Martinsville, IN,21 24 | 22,Four Fathers Brewing ,Valparaiso, IN,22 25 | 23,Indiana City Brewing,Indianapolis, IN,23 26 | 24,Burn 'Em Brewing,Michigan City, IN,24 27 | 25,Sun King Brewing Company,Indianapolis, IN,25 28 | 26,Evil Czech Brewery,Mishawaka, IN,26 29 | 27,450 North Brewing Company,Columbus, IN,27 30 | 28,Taxman Brewing Company,Bargersville, IN,28 31 | 29,Cedar Creek Brewery,Seven Points, TX,29 32 | 30,SanTan Brewing Company,Chandler, AZ,30 33 | 31,Boulevard Brewing Company,Kansas City, MO,31 34 | 32,James Page Brewing Company,Stevens Point, WI,32 35 | 33,The Dudes' Brewing Company,Torrance, CA,33 36 | 34,Ballast Point Brewing Company,San Diego, CA,34 37 | 35,Anchor Brewing Company,San Francisco, CA,35 38 | 36,Figueroa Mountain Brewing Company,Buellton, CA,36 39 | 37,Avery Brewing Company,Boulder, CO,37 40 | 38,Twisted X Brewing Company,Dripping Springs, TX,38 41 | 39,Gonzo's BiggDogg Brewing,Kalamazoo, MI,39 42 | 40,Big Muddy Brewing,Murphysboro, IL,40 43 | 41,Lost Nation Brewing,East Fairfield, VT,41 44 | 42,Rising Tide Brewing Company,Portland, ME,42 45 | 43,Rivertowne Brewing Company,Export, PA,43 46 | 44,Revolution Brewing Company,Chicago, IL,44 47 | 45,Tallgrass Brewing Company,Manhattan, KS,45 48 | 46,Sixpoint Craft Ales,Brooklyn, NY,46 49 | 47,White Birch Brewing,Hooksett, NH,47 50 | 48,Firestone Walker Brewing Company,Paso Robles, CA,48 51 | 49,SweetWater Brewing Company,Atlanta, GA,49 52 | 50,Flying Mouse Brewery,Troutville, VA,50 53 | 51,Upslope Brewing Company,Boulder, CO,51 54 | 52,Pipeworks Brewing Company,Chicago, IL,52 55 | 53,Bent Brewstillery,Roseville, MN,53 56 | 54,Flesk Brewing Company,Lombard, IL,54 57 | 55,Pollyanna Brewing Company,Lemont, IL,55 58 | 56,BuckleDown Brewing,Lyons, IL,56 59 | 57,Destihl Brewery,Bloomington, IL,57 60 | 58,Summit Brewing Company,St. Paul, MN,58 61 | 59,Latitude 42 Brewing Company,Portage, MI,59 62 | 60,4 Hands Brewing Company,Saint Louis, MO,60 63 | 61,Surly Brewing Company,Brooklyn Center, MN,61 64 | 62,Against The Grain Brewery,Louisville, KY,62 65 | 63,Crazy Mountain Brewing Company,Edwards, CO,63 66 | 64,SlapShot Brewing Company,Chicago, IL,64 67 | 65,Mikerphone Brewing,Chicago, IL,65 68 | 66,Freetail Brewing Company,San Antonio, TX,66 69 | 67,3 Daughters Brewing,St Petersburg, FL,67 70 | 68,Red Shedman Farm Brewery and Hop...,Mt. Airy, MD,68 71 | 69,Appalachian Mountain Brewery,Boone, NC,69 72 | 70,Birdsong Brewing Company,Charlotte, NC,70 73 | 71,Union Craft Brewing,Baltimore, MD,71 74 | 72,Atwater Brewery,Detroit, MI,72 75 | 73,Ale Asylum,Madison, WI,73 76 | 74,Two Brothers Brewing Company,Warrenville, IL,74 77 | 75,Bent Paddle Brewing Company,Duluth, MN,75 78 | 76,Bell's Brewery,Kalamazoo, MI,76 79 | 77,Blue Owl Brewing,Austin, TX,77 80 | 78,Speakasy Ales & Lagers,San Francisco, CA,78 81 | 79,Black Tooth Brewing Company,Sheridan, WY,79 82 | 80,Hopworks Urban Brewery,Portland, OR,80 83 | 81,Epic Brewing,Denver, CO,81 84 | 82,New Belgium Brewing Company,Fort Collins, CO,82 85 | 83,Sierra Nevada Brewing Company,Chico, CA,83 86 | 84,Keweenaw Brewing Company,Houghton, MI,84 87 | 85,Brewery Terra Firma,Traverse City, MI,85 88 | 86,Grey Sail Brewing Company,Westerly, RI,86 89 | 87,Kirkwood Station Brewing Company,Kirkwood, MO,87 90 | 88,Goose Island Brewing Company,Chicago, IL,88 91 | 89,Broad Brook Brewing LLC,East Windsor, CT,89 92 | 90,The Lion Brewery,Wilkes-Barre, PA,90 93 | 91,Madtree Brewing Company,Cincinnati, OH,91 94 | 92,Jackie O's Pub & Brewery,Athens, OH,92 95 | 93,Rhinegeist Brewery,Cincinnati, OH,93 96 | 94,Warped Wing Brewing Company,Dayton, OH,94 97 | 95,Blackrocks Brewery,Marquette, MA,95 98 | 96,Catawba Valley Brewing Company,Morganton, NC,96 99 | 97,Tröegs Brewing Company,Hershey, PA,97 100 | 98,Mission Brewery,San Diego, CA,98 101 | 99,Christian Moerlein Brewing Company,Cincinnati, OH,99 102 | 100,West Sixth Brewing,Lexington, KY,100 103 | 101,Coastal Extreme Brewing Company,Newport, RI,101 104 | 102,King Street Brewing Company,Anchorage, AK,102 105 | 103,Beer Works Brewery,Lowell, MA,103 106 | 104,Lone Tree Brewing Company,Lone Tree, CO,104 107 | 105,Four String Brewing Company,Columbus, OH,105 108 | 106,Glabrous Brewing Company,Pineland, ME,106 109 | 107,Bonfire Brewing Company,Eagle, CO,107 110 | 108,Thomas Hooker Brewing Company,Bloomfield, CT,108 111 | 109,"Woodstock Inn, Station & Brewery",North Woodstock, NH,109 112 | 110,Renegade Brewing Company,Denver, CO,110 113 | 111,Mother Earth Brew Company,Vista, CA,111 114 | 112,Black Market Brewing Company,Temecula, CA,112 115 | 113,Vault Brewing Company,Yardley, PA,113 116 | 114,Jailbreak Brewing Company,Laurel, MD,114 117 | 115,Smartmouth Brewing Company,Norfolk, VA,115 118 | 116,Base Camp Brewing Co.,Portland, OR,116 119 | 117,Alameda Brewing,Portland, OR,117 120 | 118,Southern Star Brewing Company,Conroe, TX,118 121 | 119,Steamworks Brewing Company,Durango, CO,119 122 | 120,Horny Goat Brew Pub,Milwaukee, WI,120 123 | 121,Cheboygan Brewing Company,Cheboygan, MI,121 124 | 122,Center of the Universe Brewing C...,Ashland, VA,122 125 | 123,Ipswich Ale Brewery,Ipswich, MA,123 126 | 124,Griffin Claw Brewing Company,Birmingham, MI,124 127 | 125,Karbach Brewing Company,Houston, TX,125 128 | 126,Uncle Billy's Brewery and Smokeh...,Austin, TX,126 129 | 127,Deep Ellum Brewing Company,Dallas, TX,127 130 | 128,Real Ale Brewing Company,Blanco, TX,128 131 | 129,Straub Brewery,St Mary's, PA,129 132 | 130,Shebeen Brewing Company,Wolcott, CT,130 133 | 131,Stevens Point Brewery,Stevens Point, WI,131 134 | 132,Weston Brewing Company,Weston, MO,132 135 | 133,Southern Prohibition Brewing Com...,Hattiesburg, MS,133 136 | 134,Minhas Craft Brewery,Monroe, WI,134 137 | 135,Pug Ryan's Brewery,Dillon, CO,135 138 | 136,Hops & Grains Brewing Company,Austin, TX,136 139 | 137,Sietsema Orchards and Cider Mill,Ada, MI,137 140 | 138,Summit Brewing Company,St Paul, MN,138 141 | 139,Core Brewing & Distilling Company,Springdale, AR,139 142 | 140,Independence Brewing Company,Austin, TX,140 143 | 141,Cigar City Brewing Company,Tampa, FL,141 144 | 142,Third Street Brewhouse,Cold Spring, MN,142 145 | 143,Narragansett Brewing Company,Providence, RI,143 146 | 144,Grimm Brothers Brewhouse,Loveland, CO,144 147 | 145,Cisco Brewers,Nantucket, MA,145 148 | 146,Angry Minnow,Hayward, WI,146 149 | 147,Platform Beer Company,Cleveland, OH,147 150 | 148,Odyssey Beerwerks,Arvada, CO,148 151 | 149,Lonerider Brewing Company,Raleigh, NC,149 152 | 150,Oakshire Brewing,Eugene, OR,150 153 | 151,Fort Pitt Brewing Company,Latrobe, PA,151 154 | 152,Tin Roof Brewing Company,Baton Rouge, LA,152 155 | 153,Three Creeks Brewing,Sisters, OR,153 156 | 154,2 Towns Ciderhouse,Corvallis, OR,154 157 | 155,Caldera Brewing Company,Ashland, OR,155 158 | 156,Greenbrier Valley Brewing Company,Lewisburg, WV,156 159 | 157,Phoenix Ale Brewery,Phoenix, AZ,157 160 | 158,Lumberyard Brewing Company,Flagstaff, AZ,158 161 | 159,Uinta Brewing Company,Salt Lake City, UT,159 162 | 160,Four Peaks Brewing Company,Tempe, AZ,160 163 | 161,Martin House Brewing Company,Fort Worth, TX,161 164 | 162,Right Brain Brewery,Traverse City, MI,162 165 | 163,Sly Fox Brewing Company,Phoenixville, PA,163 166 | 164,Round Guys Brewing,Lansdale, PA,164 167 | 165,Great Crescent Brewery,Aurora, IN,165 168 | 166,Oskar Blues Brewery,Longmont, CO,166 169 | 167,Boxcar Brewing Company,West Chester, PA,167 170 | 168,High Hops Brewery,Windsor, CO,168 171 | 169,Crooked Fence Brewing Company,Garden City, ID,169 172 | 170,Everybody's Brewing,White Salmon, WA,170 173 | 171,Anderson Valley Brewing Company,Boonville, CA,171 174 | 172,Fiddlehead Brewing Company,Shelburne, VT,172 175 | 173,Evil Twin Brewing,Brooklyn, NY,173 176 | 174,New Orleans Lager & Ale Brewing ...,New Orleans, LA,174 177 | 175,Spiteful Brewing Company,Chicago, IL,175 178 | 176,Rahr & Sons Brewing Company,Fort Worth, TX,176 179 | 177,18th Street Brewery,Gary, IN,177 180 | 178,Cambridge Brewing Company,Cambridge, MA,178 181 | 179,Carolina Brewery,Pittsboro, NC,179 182 | 180,Frog Level Brewing Company,Waynesville, NC,180 183 | 181,Wild Wolf Brewing Company,Nellysford, VA,181 184 | 182,COOP Ale Works,Oklahoma City, OK,182 185 | 183,Seventh Son Brewing Company,Columbus, OH,183 186 | 184,Oasis Texas Brewing Company,Austin, TX,184 187 | 185,Vander Mill Ciders,Spring Lake, MI,185 188 | 186,St. Julian Winery,Paw Paw, MI,186 189 | 187,Pedernales Brewing Company,Fredericksburg, TX,187 190 | 188,Mother's Brewing,Springfield, MO,188 191 | 189,Modern Monks Brewery,Lincoln, NE,189 192 | 190,Two Beers Brewing Company,Seattle, WA,190 193 | 191,Snake River Brewing Company,Jackson, WY,191 194 | 192,Capital Brewery,Middleton, WI,192 195 | 193,Anthem Brewing Company,Oklahoma City, OK,193 196 | 194,Goodlife Brewing Co.,Bend, OR,194 197 | 195,Breakside Brewery,Portland, OR,195 198 | 196,Goose Island Brewery Company,Chicago, IL,196 199 | 197,Burnside Brewing Co.,Portland, OR,197 200 | 198,Hop Valley Brewing Company,Springfield, OR,198 201 | 199,Worthy Brewing Company,Bend, OR,199 202 | 200,Occidental Brewing Company,Portland, OR,200 203 | 201,Fearless Brewing Company,Estacada, OR,201 204 | 202,Upland Brewing Company,Bloomington, IN,202 205 | 203,Mehana Brewing Co.,Hilo, HI,203 206 | 204,Hawai'i Nui Brewing Co.,Hilo, HI,204 207 | 205,People's Brewing Company,Lafayette, IN,205 208 | 206,Fort George Brewery,Astoria, OR,206 209 | 207,Branchline Brewing Company,San Antonio, TX,207 210 | 208,Kalona Brewing Company,Kalona, IA,208 211 | 209,Modern Times Beer,San Diego, CA,209 212 | 210,Temperance Beer Company,Evanston, IL,210 213 | 211,Wisconsin Brewing Company,Verona, WI,211 214 | 212,Crow Peak Brewing Company,Spearfish, SD,212 215 | 213,Grapevine Craft Brewery,Farmers Branch, TX,213 216 | 214,Buffalo Bayou Brewing Company,Houston, TX,214 217 | 215,Texian Brewing Co.,Richmond, TX,215 218 | 216,Orpheus Brewing,Atlanta, GA,216 219 | 217,Forgotten Boardwalk,Cherry Hill, NJ,217 220 | 218,Laughing Dog Brewing Company,Ponderay, ID,218 221 | 219,Bozeman Brewing Company,Bozeman, MT,219 222 | 220,Big Choice Brewing,Broomfield, CO,220 223 | 221,Big Storm Brewing Company,Odessa, FL,221 224 | 222,Carton Brewing Company,Atlantic Highlands, NJ,222 225 | 223,Midnight Sun Brewing Company,Anchorage, AK,223 226 | 224,Fat Head's Brewery,Middleburg Heights, OH,224 227 | 225,Refuge Brewery,Temecula, CA,225 228 | 226,Chatham Brewing,Chatham, NY,226 229 | 227,DC Brau Brewing Company,Washington, DC,227 230 | 228,Geneva Lake Brewing Company,Lake Geneva, WI,228 231 | 229,Rochester Mills Brewing Company,Rochester, MI,229 232 | 230,Cape Ann Brewing Company,Gloucester, MA,230 233 | 231,Borderlands Brewing Company,Tucson, AZ,231 234 | 232,College Street Brewhouse and Pub,Lake Havasu City, AZ,232 235 | 233,Joseph James Brewing Company,Henderson, NV,233 236 | 234,Harpoon Brewery,Boston, MA,234 237 | 235,Back East Brewing Company,Bloomfield, CT,235 238 | 236,Champion Brewing Company,Charlottesville, VA,236 239 | 237,Devil's Backbone Brewing Company,Lexington, VA,237 240 | 238,Newburgh Brewing Company,Newburgh, NY,238 241 | 239,Wiseacre Brewing Company,Memphis, TN,239 242 | 240,Golden Road Brewing,Los Angeles, CA,240 243 | 241,New Republic Brewing Company,College Station, TX,241 244 | 242,Infamous Brewing Company,Austin, TX,242 245 | 243,Two Henrys Brewing Company,Plant City, FL,243 246 | 244,Lift Bridge Brewing Company,Stillwater, MN,244 247 | 245,Lucky Town Brewing Company,Jackson, MS,245 248 | 246,Quest Brewing Company,Greenville, SC,246 249 | 247,Creature Comforts,Athens, GA,247 250 | 248,Half Full Brewery,Stamford, CT,248 251 | 249,Southampton Publick House,Southampton, NY,249 252 | 250,Chapman's Brewing,Angola, IN,250 253 | 251,Barrio Brewing Company,Tucson, AZ,251 254 | 252,Santa Cruz Mountain Brewing,Santa Cruz, CA,252 255 | 253,Frankenmuth Brewery,Frankenmuth, MI,253 256 | 254,Meckley's Cidery,Somerset Center, MI,254 257 | 255,Stillwater Artisanal Ales,Baltimore, MD,255 258 | 256,Finch's Beer Company,Chicago, IL,256 259 | 257,South Austin Brewery,South Austin, TX,257 260 | 258,Bauhaus Brew Labs,Minneapolis, MN,258 261 | 259,Ozark Beer Company,Rogers, AR,259 262 | 260,Mountain Town Brewing Company ,Mount Pleasant, MI,260 263 | 261,Otter Creek Brewing,Waterbury, VT,261 264 | 262,The Brewer's Art,Baltimore, MD,262 265 | 263,Denver Beer Company,Denver, CO,263 266 | 264,Ska Brewing Company,Durango, CO,264 267 | 265,Tractor Brewing Company,Albuquerque, NM,265 268 | 266,Peak Organic Brewing Company,Portland, ME,266 269 | 267,Cape Cod Beer,Hyannis, MA,267 270 | 268,Long Trail Brewing Company,Bridgewater Corners, VT,268 271 | 269,Great Raft Brewing Company,Shreveport, LA,269 272 | 270,Alaskan Brewing Company,Juneau, AK,270 273 | 271,Notch Brewing Company,Ipswich, MA,271 274 | 272,The Alchemist,Waterbury, VT,272 275 | 273,Three Notch'd Brewing Company,Charlottesville, VA,273 276 | 274,Portside Brewery,Cleveland, OH,274 277 | 275,Otter Creek Brewing,Middlebury, VT,275 278 | 276,Montauk Brewing Company,Montauk, NY,276 279 | 277,Indeed Brewing Company,Minneapolis, MN,277 280 | 278,Berkshire Brewing Company,South Deerfield, MA,278 281 | 279,Foolproof Brewing Company,Pawtucket, RI,279 282 | 280,Headlands Brewing Company,Mill Valley, CA,280 283 | 281,Bolero Snort Brewery,Ridgefield Park, NJ,281 284 | 282,Thunderhead Brewing Company,Kearney, NE,282 285 | 283,Defiance Brewing Company,Hays, KS,283 286 | 284,Milwaukee Brewing Company,Milwaukee, WI,284 287 | 285,Catawba Island Brewing,Port Clinton, OH,285 288 | 286,Back Forty Beer Company,Gadsden, AL,286 289 | 287,Four Corners Brewing Company,Dallas, TX,287 290 | 288,Saint Archer Brewery,San Diego, CA,288 291 | 289,Rogue Ales,Newport, OR,289 292 | 290,Hale's Ales,Seattle, WA,290 293 | 291,Tommyknocker Brewery,Idaho Springs, CO,291 294 | 292,Baxter Brewing Company,Lewiston, ME,292 295 | 293,Northampton Brewery,Northamtpon, MA,293 296 | 294,Black Shirt Brewing Company,Denver, CO,294 297 | 295,Wachusett Brewing Company,Westminster, MA,295 298 | 296,Widmer Brothers Brewing Company,Portland, OR,296 299 | 297,Hop Farm Brewing Company,Pittsburgh, PA,297 300 | 298,Liquid Hero Brewery,York, PA,298 301 | 299,Matt Brewing Company,Utica, NY,299 302 | 300,Boston Beer Company,Boston, MA,300 303 | 301,Old Forge Brewing Company,Danville, PA,301 304 | 302,Utah Brewers Cooperative,Salt Lake City, UT,302 305 | 303,Magic Hat Brewing Company,South Burlington, VT,303 306 | 304,Blue Hills Brewery,Canton, MA,304 307 | 305,Night Shift Brewing,Everett, MA,305 308 | 306,Beach Brewing Company,Virginia Beach, VA,306 309 | 307,Payette Brewing Company,Garden City, ID,307 310 | 308,Brew Bus Brewing,Tampa, FL,308 311 | 309,Sockeye Brewing Company,Boise, ID,309 312 | 310,Pine Street Brewery,San Francisco, CA,310 313 | 311,Dirty Bucket Brewing Company,Woodinville, WA,311 314 | 312,Jackalope Brewing Company,Nashville, TN,312 315 | 313,Slanted Rock Brewing Company,Meridian, ID,313 316 | 314,Piney River Brewing Company,Bucryus, MO,314 317 | 315,Cutters Brewing Company,Avon, IN,315 318 | 316,Iron Hill Brewery & Restaurant,Wilmington, DE,316 319 | 317,Marshall Wharf Brewing Company,Belfast, ME,317 320 | 318,Banner Beer Company,Williamsburg, MA,318 321 | 319,Dick's Brewing Company,Centralia, WA,319 322 | 320,Claremont Craft Ales,Claremont, CA,320 323 | 321,Rivertown Brewing Company,Lockland, OH,321 324 | 322,Voodoo Brewery,Meadville, PA,322 325 | 323,D.L. Geary Brewing Company,Portland, ME,323 326 | 324,Pisgah Brewing Company,Black Mountain, NC,324 327 | 325,Neshaminy Creek Brewing Company,Croydon, PA,325 328 | 326,Morgan Street Brewery,Saint Louis, MO,326 329 | 327,Half Acre Beer Company,Chicago, IL,327 330 | 328,The Just Beer Project,Burlington, VT,328 331 | 329,The Bronx Brewery,Bronx, NY,329 332 | 330,Dead Armadillo Craft Brewing,Tulsa, OK,330 333 | 331,Catawba Brewing Company,Morganton, NC,331 334 | 332,La Cumbre Brewing Company,Albuquerque, NM,332 335 | 333,David's Ale Works,Diamond Springs, CA,333 336 | 334,The Traveler Beer Company,Burlington, VT,334 337 | 335,Fargo Brewing Company,Fargo, ND,335 338 | 336,Big Sky Brewing Company,Missoula, MT,336 339 | 337,Nebraska Brewing Company,Papillion, NE,337 340 | 338,Uncle John's Fruit House Winery,St. John's, MI,338 341 | 339,Wormtown Brewery,Worcester, MA,339 342 | 340,Due South Brewing Company,Boynton Beach, FL,340 343 | 341,Palisade Brewing Company,Palisade, CO,341 344 | 342,KelSo Beer Company,Brooklyn, NY,342 345 | 343,Hardywood Park Craft Brewery,Richmond, VA,343 346 | 344,Wolf Hills Brewing Company,Abingdon, VA,344 347 | 345,Lavery Brewing Company,Erie, PA,345 348 | 346,Manzanita Brewing Company,Santee, CA,346 349 | 347,Fullsteam Brewery,Durham, NC,347 350 | 348,Four Horsemen Brewing Company,South Bend, IN,348 351 | 349,Hinterland Brewery,Green Bay, WI,349 352 | 350,Central Coast Brewing Company,San Luis Obispo, CA,350 353 | 351,Westfield River Brewing Company,Westfield, MA,351 354 | 352,Elevator Brewing Company,Columbus, OH,352 355 | 353,Aslan Brewing Company,Bellingham, WA,353 356 | 354,Kulshan Brewery,Bellingham, WA,354 357 | 355,Pikes Peak Brewing Company,Monument, CO,355 358 | 356,Manayunk Brewing Company,Philadelphia, PA,356 359 | 357,Buckeye Brewing,Cleveland, OH,357 360 | 358,Daredevil Brewing Company,Shelbyville, IN,358 361 | 359,NoDa Brewing Company,Charlotte, NC,359 362 | 360,Aviator Brewing Company,Fuquay-Varina, NC,360 363 | 361,Wild Onion Brewing Company,Lake Barrington, IL,361 364 | 362,Hilliard's Beer,Seattle, WA,362 365 | 363,Mikkeller,Pottstown, PA,363 366 | 364,Bohemian Brewery,Midvale, UT,364 367 | 365,Great River Brewery,Davenport, IA,365 368 | 366,Mustang Brewing Company,Mustang, OK,366 369 | 367,Airways Brewing Company,Kent, WA,367 370 | 368,21st Amendment Brewery,San Francisco, CA,368 371 | 369,Eddyline Brewery & Restaurant,Buena Vista, CO,369 372 | 370,Pizza Port Brewing Company,Carlsbad, CA,370 373 | 371,Sly Fox Brewing Company,Pottstown, PA,371 374 | 372,Spring House Brewing Company,Conestoga, PA,372 375 | 373,7venth Sun,Dunedin, FL,373 376 | 374,Astoria Brewing Company,Astoria, OR,374 377 | 375,Maui Brewing Company,Lahaina, HI,375 378 | 376,RoughTail Brewing Company,Midwest City, OK,376 379 | 377,Lucette Brewing Company,Menominee, WI,377 380 | 378,Bold City Brewery,Jacksonville, FL,378 381 | 379,Grey Sail Brewing of Rhode Island,Westerly, RI,379 382 | 380,Blue Blood Brewing Company,Lincoln, NE,380 383 | 381,Swashbuckler Brewing Company,Manheim, PA,381 384 | 382,Blue Mountain Brewery,Afton, VA,382 385 | 383,Starr Hill Brewery,Crozet, VA,383 386 | 384,Westbrook Brewing Company,Mt. Pleasant, SC,384 387 | 385,Shipyard Brewing Company,Portland, ME,385 388 | 386,Revolution Brewing,Paonia, CO,386 389 | 387,Natian Brewery,Portland, OR,387 390 | 388,Alltech's Lexington Brewing Company,Lexington, KY,388 391 | 389,Oskar Blues Brewery (North Carol...,Brevard, NC,389 392 | 390,Orlison Brewing Company,Airway Heights, WA,390 393 | 391,Breckenridge Brewery,Denver, CO,391 394 | 392,Santa Fe Brewing Company,Santa Fe, NM,392 395 | 393,Miami Brewing Company,Miami, FL,393 396 | 394,Schilling & Company,Seattle, WA,394 397 | 395,Hops & Grain Brewery,Austin, TX,395 398 | 396,White Flame Brewing Company,Hudsonville, MI,396 399 | 397,Ruhstaller Beer Company,Sacramento, CA,397 400 | 398,Saugatuck Brewing Company,Douglas, MI,398 401 | 399,Moab Brewery,Moab, UT,399 402 | 400,Macon Beer Company,Macon, GA,400 403 | 401,Amnesia Brewing Company,Washougal, WA,401 404 | 402,Wolverine State Brewing Company,Ann Arbor, MI,402 405 | 403,Red Tank Cider Company,Bend, OR,403 406 | 404,Cascadia Ciderworks United,Portland, OR,404 407 | 405,Fate Brewing Company,Boulder, CO,405 408 | 406,Lazy Monk Brewing,Eau Claire, WI,406 409 | 407,Bitter Root Brewing,Hamilton, MT,407 410 | 408,10 Barrel Brewing Company,Bend, OR,408 411 | 409,Tamarack Brewing Company,Lakeside, MT,409 412 | 410,New England Brewing Company,Woodbridge, CT,410 413 | 411,Seattle Cider Company,Seattle, WA,411 414 | 412,Straight to Ale,Huntsville, AL,412 415 | 413,Austin Beerworks,Austin, TX,413 416 | 414,Blue Mountain Brewery,Arrington, VA,414 417 | 415,Coastal Empire Beer Company,Savannah, GA,415 418 | 416,Jack's Hard Cider (Hauser Estate...,Biglerville, PA,416 419 | 417,Boulder Beer Company,Boulder, CO,417 420 | 418,Coalition Brewing Company,Portland, OR,418 421 | 419,Sanitas Brewing Company,Boulder, CO,419 422 | 420,Gore Range Brewery,Edwards, CO,420 423 | 421,Redstone Meadery,Boulder, CO,421 424 | 422,Blue Dog Mead,Eugene, OR,422 425 | 423,Hess Brewing Company,San Diego, CA,423 426 | 424,Wynkoop Brewing Company,Denver, CO,424 427 | 425,Ciderboys,Stevens Point, WI,425 428 | 426,Armadillo Ale Works,Denton, TX,426 429 | 427,Roanoke Railhouse Brewery,Roanoke, VA,427 430 | 428,Schlafly Brewing Company,Saint Louis, MO,428 431 | 429,Asher Brewing Company,Boulder, CO,429 432 | 430,Lost Rhino Brewing Company,Ashburn, VA,430 433 | 431,North Country Brewing Company,Slippery Rock, PA,431 434 | 432,Seabright Brewery,Santa Cruz, CA,432 435 | 433,French Broad Brewery,Asheville, NC,433 436 | 434,Angry Orchard Cider Company,Cincinnati, OH,434 437 | 435,Two Roads Brewing Company,Stratford, CT,435 438 | 436,Southern Oregon Brewing Company,Medford, OR,436 439 | 437,Brooklyn Brewery,Brooklyn, NY,437 440 | 438,The Right Brain Brewery,Traverse City, MI,438 441 | 439,Kona Brewing Company,Kona, HI,439 442 | 440,MillKing It Productions,Royal Oak, MI,440 443 | 441,Pateros Creek Brewing Company,Fort Collins, CO,441 444 | 442,O'Fallon Brewery,O'Fallon, MO,442 445 | 443,Marble Brewery,Albuquerque, NM,443 446 | 444,Big Wood Brewery,Vadnais Heights, MN,444 447 | 445,Howard Brewing Company,Lenoir, NC,445 448 | 446,Downeast Cider House,Leominster, MA,446 449 | 447,Swamp Head Brewery,Gainesville, FL,447 450 | 448,Mavericks Beer Company,Half Moon Bay, CA,448 451 | 449,TailGate Beer,San Diego, CA,449 452 | 450,Northwest Brewing Company,Pacific, WA,450 453 | 451,Dad & Dude's Breweria,Aurora, CO,451 454 | 452,Centennial Beer Company,Edwards, CO,452 455 | 453,Denali Brewing Company,Talkeetna, AK,453 456 | 454,Deschutes Brewery,Bend, OR,454 457 | 455,Sunken City Brewing Company,Hardy, VA,455 458 | 456,Lucette Brewing Company,Menominie, WI,456 459 | 457,The Black Tooth Brewing Company,Sheridan, WY,457 460 | 458,Kenai River Brewing Company,Soldotna, AK,458 461 | 459,River North Brewery,Denver, CO,459 462 | 460,Fremont Brewing Company,Seattle, WA,460 463 | 461,Armstrong Brewing Company,South San Francisco, CA,461 464 | 462,AC Golden Brewing Company,Golden, CO,462 465 | 463,Big Bend Brewing Company,Alpine, TX,463 466 | 464,Good Life Brewing Company,Bend, OR,464 467 | 465,Engine 15 Brewing,Jacksonville Beach, FL,465 468 | 466,Green Room Brewing,Jacksonville, FL,466 469 | 467,Brindle Dog Brewing Company,Tampa Bay, FL,467 470 | 468,Peace Tree Brewing Company,Knoxville, IA,468 471 | 469,Terrapin Brewing Company,Athens, GA,469 472 | 470,Pete's Brewing Company,San Antonio, TX,470 473 | 471,Okoboji Brewing Company,Spirit Lake, IA,471 474 | 472,Crystal Springs Brewing Company,Boulder, CO,472 475 | 473,Engine House 9,Tacoma, WA,473 476 | 474,Tonka Beer Company,Minnetonka, MN,474 477 | 475,Red Hare Brewing Company,Marietta, GA,475 478 | 476,Hangar 24 Craft Brewery,Redlands, CA,476 479 | 477,Big Elm Brewing,Sheffield, MA,477 480 | 478,Good People Brewing Company,Birmingham, AL,478 481 | 479,Heavy Seas Beer,Halethorpe, MD,479 482 | 480,Telluride Brewing Company,Telluride, CO,480 483 | 481,7 Seas Brewing Company,Gig Harbor, WA,481 484 | 482,Confluence Brewing Company,Des Moines, IA,482 485 | 483,Bale Breaker Brewing Company,Yakima, WA,483 486 | 484,The Manhattan Brewing Company,New York, NY,484 487 | 485,MacTarnahans Brewing Company,Portland, OR,485 488 | 486,Stillmank Beer Company,Green Bay, WI,486 489 | 487,Redhook Brewery,Woodinville, WA,487 490 | 488,Dock Street Brewery,Philadelphia, PA,488 491 | 489,Blue Point Brewing Company,Patchogue, NY,489 492 | 490,Tampa Bay Brewing Company,Tampa, FL,490 493 | 491,Devil's Canyon Brewery,Belmont, CA,491 494 | 492,Stone Coast Brewing Company,Portland, ME,492 495 | 493,Broken Tooth Brewing Company,Anchorage, AK,493 496 | 494,Seven Brides Brewery,Silverton, OR,494 497 | 495,Newburyport Brewing Company,Newburyport, MA,495 498 | 496,Dry Dock Brewing Company,Aurora, CO,496 499 | 497,Cans Bar and Canteen,Charlotte, NC,497 500 | 498,Sprecher Brewing Company,Glendale, WI,498 501 | 499,Wildwood Brewing Company,Stevensville, MT,499 502 | 500,High Noon Saloon And Brewery,Leavenworth, KS,500 503 | 501,Woodchuck Hard Cider,Middlebury, VT,501 504 | 502,Sea Dog Brewing Company,Portland, ME,502 505 | 503,Oskar Blues Brewery,Lyons, CO,503 506 | 504,Carolina Beer & Beverage,Mooresville, NC,504 507 | 505,Krebs Brewing Company (Pete's Pl...,Krebs, OK,505 508 | 506,Warbird Brewing Company,Fort Wayne, IN,506 509 | 507,Mudshark Brewing Company,Lake Havasu City, AZ,507 510 | 508,Spilker Ales,Cortland, NE,508 511 | 509,Wingman Brewers,Tacoma, WA,509 512 | 510,Kettle House Brewing Company,Missoula, MT,510 513 | 511,Sherwood Forest Brewers,Marlborough, MA,511 514 | 512,Cottrell Brewing,Pawcatuck, CT,512 515 | 513,Arctic Craft Brewery,Colorado Springs, CO,513 516 | 514,Monkey Paw Pub & Brewery,San Diego, CA,514 517 | 515,Crabtree Brewing Company,Greeley, CO,515 518 | 516,Emerald City Beer Company,Seattle, WA,516 519 | 517,Butcher's Brewing,Carlsbad, CA,517 520 | 518,New South Brewing Company,Myrtle Beach, SC,518 521 | 519,Big River Brewing Company,Chattanooga, TN,519 522 | 520,Twisted Pine Brewing Company,Boulder, CO,520 523 | 521,Flying Dog Brewery,Frederick, MD,521 524 | 522,Uncommon Brewers,Santa Cruz, CA,522 525 | 523,Aspen Brewing Company,Aspen, CO,523 526 | 524,Triangle Brewing Company,Durham, NC,524 527 | 525,Bomb Beer Company,New York, NY,525 528 | 526,Churchkey Can Company,Seattle, WA,526 529 | 527,Intuition Ale Works,Jacksonville, FL,527 530 | 528,Asheville Brewing Company,Asheville, NC,528 531 | 529,Northwoods Brewpub,Eau Claire, WI,529 532 | 530,Buckbean Brewing Company,Reno, NV,530 533 | 531,Dolores River Brewery,Dolores, CO,531 534 | 532,Flat Rock Brewing Company,Smithton, PA,532 535 | 533,Abita Brewing Company,Abita Springs, LA,533 536 | 534,Mammoth Brewing Company,Mammoth Lakes, CA,534 537 | 535,Harvest Moon Brewing Company,Belt, MT,535 538 | 536,Grand Canyon Brewing Company,Williams, AZ,536 539 | 537,Lewis and Clark Brewing Company,Helena, MT,537 540 | 538,Dundee Brewing Company,Rochester, NY,538 541 | 539,Twin Lakes Brewing Company,Greenville, DE,539 542 | 540,Mother Earth Brewing Company,Kinston, NC,540 543 | 541,Arcadia Brewing Company,Battle Creek, MI,541 544 | 542,Angry Minnow Brewing Company,Hayward, WI,542 545 | 543,Great Northern Brewing Company,Whitefish, MT,543 546 | 544,Pyramid Breweries,Seattle, WA,544 547 | 545,Lancaster Brewing Company,Lancaster, PA,545 548 | 546,Upstate Brewing Company,Elmira, NY,546 549 | 547,Moat Mountain Smoke House & Brew...,North Conway, NH,547 550 | 548,Prescott Brewing Company,Prescott, AZ,548 551 | 549,Mogollon Brewing Company,Flagstaff, AZ,549 552 | 550,Wind River Brewing Company,Pinedale, WY,550 553 | 551,Silverton Brewery,Silverton, CO,551 554 | 552,Mickey Finn's Brewery,Libertyville, IL,552 555 | 553,Covington Brewhouse,Covington, LA,553 556 | 554,Dave's Brewfarm,Wilson, WI,554 557 | 555,Ukiah Brewing Company,Ukiah, CA,555 558 | 556,Butternuts Beer and Ale,Garrattsville, NY,556 559 | 557,Sleeping Lady Brewing Company,Anchorage, AK,557 560 | -------------------------------------------------------------------------------- /dataForProject/dvdrental.rar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/dvdrental.rar -------------------------------------------------------------------------------- /dataForProject/image/dvdrental-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/dvdrental-schema.png -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3057.dat: -------------------------------------------------------------------------------- 1 | 1 Penelope Guiness 2013-05-26 14:47:57.62 2 | 2 Nick Wahlberg 2013-05-26 14:47:57.62 3 | 3 Ed Chase 2013-05-26 14:47:57.62 4 | 4 Jennifer Davis 2013-05-26 14:47:57.62 5 | 5 Johnny Lollobrigida 2013-05-26 14:47:57.62 6 | 6 Bette Nicholson 2013-05-26 14:47:57.62 7 | 7 Grace Mostel 2013-05-26 14:47:57.62 8 | 8 Matthew Johansson 2013-05-26 14:47:57.62 9 | 9 Joe Swank 2013-05-26 14:47:57.62 10 | 10 Christian Gable 2013-05-26 14:47:57.62 11 | 11 Zero Cage 2013-05-26 14:47:57.62 12 | 12 Karl Berry 2013-05-26 14:47:57.62 13 | 13 Uma Wood 2013-05-26 14:47:57.62 14 | 14 Vivien Bergen 2013-05-26 14:47:57.62 15 | 15 Cuba Olivier 2013-05-26 14:47:57.62 16 | 16 Fred Costner 2013-05-26 14:47:57.62 17 | 17 Helen Voight 2013-05-26 14:47:57.62 18 | 18 Dan Torn 2013-05-26 14:47:57.62 19 | 19 Bob Fawcett 2013-05-26 14:47:57.62 20 | 20 Lucille Tracy 2013-05-26 14:47:57.62 21 | 21 Kirsten Paltrow 2013-05-26 14:47:57.62 22 | 22 Elvis Marx 2013-05-26 14:47:57.62 23 | 23 Sandra Kilmer 2013-05-26 14:47:57.62 24 | 24 Cameron Streep 2013-05-26 14:47:57.62 25 | 25 Kevin Bloom 2013-05-26 14:47:57.62 26 | 26 Rip Crawford 2013-05-26 14:47:57.62 27 | 27 Julia Mcqueen 2013-05-26 14:47:57.62 28 | 28 Woody Hoffman 2013-05-26 14:47:57.62 29 | 29 Alec Wayne 2013-05-26 14:47:57.62 30 | 30 Sandra Peck 2013-05-26 14:47:57.62 31 | 31 Sissy Sobieski 2013-05-26 14:47:57.62 32 | 32 Tim Hackman 2013-05-26 14:47:57.62 33 | 33 Milla Peck 2013-05-26 14:47:57.62 34 | 34 Audrey Olivier 2013-05-26 14:47:57.62 35 | 35 Judy Dean 2013-05-26 14:47:57.62 36 | 36 Burt Dukakis 2013-05-26 14:47:57.62 37 | 37 Val Bolger 2013-05-26 14:47:57.62 38 | 38 Tom Mckellen 2013-05-26 14:47:57.62 39 | 39 Goldie Brody 2013-05-26 14:47:57.62 40 | 40 Johnny Cage 2013-05-26 14:47:57.62 41 | 41 Jodie Degeneres 2013-05-26 14:47:57.62 42 | 42 Tom Miranda 2013-05-26 14:47:57.62 43 | 43 Kirk Jovovich 2013-05-26 14:47:57.62 44 | 44 Nick Stallone 2013-05-26 14:47:57.62 45 | 45 Reese Kilmer 2013-05-26 14:47:57.62 46 | 46 Parker Goldberg 2013-05-26 14:47:57.62 47 | 47 Julia Barrymore 2013-05-26 14:47:57.62 48 | 48 Frances Day-Lewis 2013-05-26 14:47:57.62 49 | 49 Anne Cronyn 2013-05-26 14:47:57.62 50 | 50 Natalie Hopkins 2013-05-26 14:47:57.62 51 | 51 Gary Phoenix 2013-05-26 14:47:57.62 52 | 52 Carmen Hunt 2013-05-26 14:47:57.62 53 | 53 Mena Temple 2013-05-26 14:47:57.62 54 | 54 Penelope Pinkett 2013-05-26 14:47:57.62 55 | 55 Fay Kilmer 2013-05-26 14:47:57.62 56 | 56 Dan Harris 2013-05-26 14:47:57.62 57 | 57 Jude Cruise 2013-05-26 14:47:57.62 58 | 58 Christian Akroyd 2013-05-26 14:47:57.62 59 | 59 Dustin Tautou 2013-05-26 14:47:57.62 60 | 60 Henry Berry 2013-05-26 14:47:57.62 61 | 61 Christian Neeson 2013-05-26 14:47:57.62 62 | 62 Jayne Neeson 2013-05-26 14:47:57.62 63 | 63 Cameron Wray 2013-05-26 14:47:57.62 64 | 64 Ray Johansson 2013-05-26 14:47:57.62 65 | 65 Angela Hudson 2013-05-26 14:47:57.62 66 | 66 Mary Tandy 2013-05-26 14:47:57.62 67 | 67 Jessica Bailey 2013-05-26 14:47:57.62 68 | 68 Rip Winslet 2013-05-26 14:47:57.62 69 | 69 Kenneth Paltrow 2013-05-26 14:47:57.62 70 | 70 Michelle Mcconaughey 2013-05-26 14:47:57.62 71 | 71 Adam Grant 2013-05-26 14:47:57.62 72 | 72 Sean Williams 2013-05-26 14:47:57.62 73 | 73 Gary Penn 2013-05-26 14:47:57.62 74 | 74 Milla Keitel 2013-05-26 14:47:57.62 75 | 75 Burt Posey 2013-05-26 14:47:57.62 76 | 76 Angelina Astaire 2013-05-26 14:47:57.62 77 | 77 Cary Mcconaughey 2013-05-26 14:47:57.62 78 | 78 Groucho Sinatra 2013-05-26 14:47:57.62 79 | 79 Mae Hoffman 2013-05-26 14:47:57.62 80 | 80 Ralph Cruz 2013-05-26 14:47:57.62 81 | 81 Scarlett Damon 2013-05-26 14:47:57.62 82 | 82 Woody Jolie 2013-05-26 14:47:57.62 83 | 83 Ben Willis 2013-05-26 14:47:57.62 84 | 84 James Pitt 2013-05-26 14:47:57.62 85 | 85 Minnie Zellweger 2013-05-26 14:47:57.62 86 | 143 River Dean 2013-05-26 14:47:57.62 87 | 86 Greg Chaplin 2013-05-26 14:47:57.62 88 | 87 Spencer Peck 2013-05-26 14:47:57.62 89 | 88 Kenneth Pesci 2013-05-26 14:47:57.62 90 | 89 Charlize Dench 2013-05-26 14:47:57.62 91 | 90 Sean Guiness 2013-05-26 14:47:57.62 92 | 91 Christopher Berry 2013-05-26 14:47:57.62 93 | 92 Kirsten Akroyd 2013-05-26 14:47:57.62 94 | 93 Ellen Presley 2013-05-26 14:47:57.62 95 | 94 Kenneth Torn 2013-05-26 14:47:57.62 96 | 95 Daryl Wahlberg 2013-05-26 14:47:57.62 97 | 96 Gene Willis 2013-05-26 14:47:57.62 98 | 97 Meg Hawke 2013-05-26 14:47:57.62 99 | 98 Chris Bridges 2013-05-26 14:47:57.62 100 | 99 Jim Mostel 2013-05-26 14:47:57.62 101 | 100 Spencer Depp 2013-05-26 14:47:57.62 102 | 101 Susan Davis 2013-05-26 14:47:57.62 103 | 102 Walter Torn 2013-05-26 14:47:57.62 104 | 103 Matthew Leigh 2013-05-26 14:47:57.62 105 | 104 Penelope Cronyn 2013-05-26 14:47:57.62 106 | 105 Sidney Crowe 2013-05-26 14:47:57.62 107 | 106 Groucho Dunst 2013-05-26 14:47:57.62 108 | 107 Gina Degeneres 2013-05-26 14:47:57.62 109 | 108 Warren Nolte 2013-05-26 14:47:57.62 110 | 109 Sylvester Dern 2013-05-26 14:47:57.62 111 | 110 Susan Davis 2013-05-26 14:47:57.62 112 | 111 Cameron Zellweger 2013-05-26 14:47:57.62 113 | 112 Russell Bacall 2013-05-26 14:47:57.62 114 | 113 Morgan Hopkins 2013-05-26 14:47:57.62 115 | 114 Morgan Mcdormand 2013-05-26 14:47:57.62 116 | 115 Harrison Bale 2013-05-26 14:47:57.62 117 | 116 Dan Streep 2013-05-26 14:47:57.62 118 | 117 Renee Tracy 2013-05-26 14:47:57.62 119 | 118 Cuba Allen 2013-05-26 14:47:57.62 120 | 119 Warren Jackman 2013-05-26 14:47:57.62 121 | 120 Penelope Monroe 2013-05-26 14:47:57.62 122 | 121 Liza Bergman 2013-05-26 14:47:57.62 123 | 122 Salma Nolte 2013-05-26 14:47:57.62 124 | 123 Julianne Dench 2013-05-26 14:47:57.62 125 | 124 Scarlett Bening 2013-05-26 14:47:57.62 126 | 125 Albert Nolte 2013-05-26 14:47:57.62 127 | 126 Frances Tomei 2013-05-26 14:47:57.62 128 | 127 Kevin Garland 2013-05-26 14:47:57.62 129 | 128 Cate Mcqueen 2013-05-26 14:47:57.62 130 | 129 Daryl Crawford 2013-05-26 14:47:57.62 131 | 130 Greta Keitel 2013-05-26 14:47:57.62 132 | 131 Jane Jackman 2013-05-26 14:47:57.62 133 | 132 Adam Hopper 2013-05-26 14:47:57.62 134 | 133 Richard Penn 2013-05-26 14:47:57.62 135 | 134 Gene Hopkins 2013-05-26 14:47:57.62 136 | 135 Rita Reynolds 2013-05-26 14:47:57.62 137 | 136 Ed Mansfield 2013-05-26 14:47:57.62 138 | 137 Morgan Williams 2013-05-26 14:47:57.62 139 | 138 Lucille Dee 2013-05-26 14:47:57.62 140 | 139 Ewan Gooding 2013-05-26 14:47:57.62 141 | 140 Whoopi Hurt 2013-05-26 14:47:57.62 142 | 141 Cate Harris 2013-05-26 14:47:57.62 143 | 142 Jada Ryder 2013-05-26 14:47:57.62 144 | 144 Angela Witherspoon 2013-05-26 14:47:57.62 145 | 145 Kim Allen 2013-05-26 14:47:57.62 146 | 146 Albert Johansson 2013-05-26 14:47:57.62 147 | 147 Fay Winslet 2013-05-26 14:47:57.62 148 | 148 Emily Dee 2013-05-26 14:47:57.62 149 | 149 Russell Temple 2013-05-26 14:47:57.62 150 | 150 Jayne Nolte 2013-05-26 14:47:57.62 151 | 151 Geoffrey Heston 2013-05-26 14:47:57.62 152 | 152 Ben Harris 2013-05-26 14:47:57.62 153 | 153 Minnie Kilmer 2013-05-26 14:47:57.62 154 | 154 Meryl Gibson 2013-05-26 14:47:57.62 155 | 155 Ian Tandy 2013-05-26 14:47:57.62 156 | 156 Fay Wood 2013-05-26 14:47:57.62 157 | 157 Greta Malden 2013-05-26 14:47:57.62 158 | 158 Vivien Basinger 2013-05-26 14:47:57.62 159 | 159 Laura Brody 2013-05-26 14:47:57.62 160 | 160 Chris Depp 2013-05-26 14:47:57.62 161 | 161 Harvey Hope 2013-05-26 14:47:57.62 162 | 162 Oprah Kilmer 2013-05-26 14:47:57.62 163 | 163 Christopher West 2013-05-26 14:47:57.62 164 | 164 Humphrey Willis 2013-05-26 14:47:57.62 165 | 165 Al Garland 2013-05-26 14:47:57.62 166 | 166 Nick Degeneres 2013-05-26 14:47:57.62 167 | 167 Laurence Bullock 2013-05-26 14:47:57.62 168 | 168 Will Wilson 2013-05-26 14:47:57.62 169 | 169 Kenneth Hoffman 2013-05-26 14:47:57.62 170 | 170 Mena Hopper 2013-05-26 14:47:57.62 171 | 171 Olympia Pfeiffer 2013-05-26 14:47:57.62 172 | 172 Groucho Williams 2013-05-26 14:47:57.62 173 | 173 Alan Dreyfuss 2013-05-26 14:47:57.62 174 | 174 Michael Bening 2013-05-26 14:47:57.62 175 | 175 William Hackman 2013-05-26 14:47:57.62 176 | 176 Jon Chase 2013-05-26 14:47:57.62 177 | 177 Gene Mckellen 2013-05-26 14:47:57.62 178 | 178 Lisa Monroe 2013-05-26 14:47:57.62 179 | 179 Ed Guiness 2013-05-26 14:47:57.62 180 | 180 Jeff Silverstone 2013-05-26 14:47:57.62 181 | 181 Matthew Carrey 2013-05-26 14:47:57.62 182 | 182 Debbie Akroyd 2013-05-26 14:47:57.62 183 | 183 Russell Close 2013-05-26 14:47:57.62 184 | 184 Humphrey Garland 2013-05-26 14:47:57.62 185 | 185 Michael Bolger 2013-05-26 14:47:57.62 186 | 186 Julia Zellweger 2013-05-26 14:47:57.62 187 | 187 Renee Ball 2013-05-26 14:47:57.62 188 | 188 Rock Dukakis 2013-05-26 14:47:57.62 189 | 189 Cuba Birch 2013-05-26 14:47:57.62 190 | 190 Audrey Bailey 2013-05-26 14:47:57.62 191 | 191 Gregory Gooding 2013-05-26 14:47:57.62 192 | 192 John Suvari 2013-05-26 14:47:57.62 193 | 193 Burt Temple 2013-05-26 14:47:57.62 194 | 194 Meryl Allen 2013-05-26 14:47:57.62 195 | 195 Jayne Silverstone 2013-05-26 14:47:57.62 196 | 196 Bela Walken 2013-05-26 14:47:57.62 197 | 197 Reese West 2013-05-26 14:47:57.62 198 | 198 Mary Keitel 2013-05-26 14:47:57.62 199 | 199 Julia Fawcett 2013-05-26 14:47:57.62 200 | 200 Thora Temple 2013-05-26 14:47:57.62 201 | \. 202 | 203 | 204 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3059.dat: -------------------------------------------------------------------------------- 1 | 1 Action 2006-02-15 09:46:27 2 | 2 Animation 2006-02-15 09:46:27 3 | 3 Children 2006-02-15 09:46:27 4 | 4 Classics 2006-02-15 09:46:27 5 | 5 Comedy 2006-02-15 09:46:27 6 | 6 Documentary 2006-02-15 09:46:27 7 | 7 Drama 2006-02-15 09:46:27 8 | 8 Family 2006-02-15 09:46:27 9 | 9 Foreign 2006-02-15 09:46:27 10 | 10 Games 2006-02-15 09:46:27 11 | 11 Horror 2006-02-15 09:46:27 12 | 12 Music 2006-02-15 09:46:27 13 | 13 New 2006-02-15 09:46:27 14 | 14 Sci-Fi 2006-02-15 09:46:27 15 | 15 Sports 2006-02-15 09:46:27 16 | 16 Travel 2006-02-15 09:46:27 17 | \. 18 | 19 | 20 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3063.dat: -------------------------------------------------------------------------------- 1 | 1 6 2006-02-15 10:07:09 2 | 2 11 2006-02-15 10:07:09 3 | 3 6 2006-02-15 10:07:09 4 | 4 11 2006-02-15 10:07:09 5 | 5 8 2006-02-15 10:07:09 6 | 6 9 2006-02-15 10:07:09 7 | 7 5 2006-02-15 10:07:09 8 | 8 11 2006-02-15 10:07:09 9 | 9 11 2006-02-15 10:07:09 10 | 10 15 2006-02-15 10:07:09 11 | 11 9 2006-02-15 10:07:09 12 | 12 12 2006-02-15 10:07:09 13 | 13 11 2006-02-15 10:07:09 14 | 14 4 2006-02-15 10:07:09 15 | 15 9 2006-02-15 10:07:09 16 | 16 9 2006-02-15 10:07:09 17 | 17 12 2006-02-15 10:07:09 18 | 18 2 2006-02-15 10:07:09 19 | 19 1 2006-02-15 10:07:09 20 | 20 12 2006-02-15 10:07:09 21 | 21 1 2006-02-15 10:07:09 22 | 22 13 2006-02-15 10:07:09 23 | 23 2 2006-02-15 10:07:09 24 | 24 11 2006-02-15 10:07:09 25 | 25 13 2006-02-15 10:07:09 26 | 26 14 2006-02-15 10:07:09 27 | 27 15 2006-02-15 10:07:09 28 | 28 5 2006-02-15 10:07:09 29 | 29 1 2006-02-15 10:07:09 30 | 30 11 2006-02-15 10:07:09 31 | 31 8 2006-02-15 10:07:09 32 | 32 13 2006-02-15 10:07:09 33 | 33 7 2006-02-15 10:07:09 34 | 34 11 2006-02-15 10:07:09 35 | 35 11 2006-02-15 10:07:09 36 | 36 2 2006-02-15 10:07:09 37 | 37 4 2006-02-15 10:07:09 38 | 38 1 2006-02-15 10:07:09 39 | 39 14 2006-02-15 10:07:09 40 | 40 6 2006-02-15 10:07:09 41 | 41 16 2006-02-15 10:07:09 42 | 42 15 2006-02-15 10:07:09 43 | 43 8 2006-02-15 10:07:09 44 | 44 14 2006-02-15 10:07:09 45 | 45 13 2006-02-15 10:07:09 46 | 46 10 2006-02-15 10:07:09 47 | 47 9 2006-02-15 10:07:09 48 | 48 3 2006-02-15 10:07:09 49 | 49 14 2006-02-15 10:07:09 50 | 50 8 2006-02-15 10:07:09 51 | 51 12 2006-02-15 10:07:09 52 | 52 9 2006-02-15 10:07:09 53 | 53 8 2006-02-15 10:07:09 54 | 54 12 2006-02-15 10:07:09 55 | 55 14 2006-02-15 10:07:09 56 | 56 1 2006-02-15 10:07:09 57 | 57 16 2006-02-15 10:07:09 58 | 58 6 2006-02-15 10:07:09 59 | 59 3 2006-02-15 10:07:09 60 | 60 4 2006-02-15 10:07:09 61 | 61 7 2006-02-15 10:07:09 62 | 62 6 2006-02-15 10:07:09 63 | 63 8 2006-02-15 10:07:09 64 | 64 7 2006-02-15 10:07:09 65 | 65 11 2006-02-15 10:07:09 66 | 66 3 2006-02-15 10:07:09 67 | 67 1 2006-02-15 10:07:09 68 | 68 3 2006-02-15 10:07:09 69 | 69 14 2006-02-15 10:07:09 70 | 70 2 2006-02-15 10:07:09 71 | 71 8 2006-02-15 10:07:09 72 | 72 6 2006-02-15 10:07:09 73 | 73 14 2006-02-15 10:07:09 74 | 74 12 2006-02-15 10:07:09 75 | 75 16 2006-02-15 10:07:09 76 | 76 12 2006-02-15 10:07:09 77 | 77 13 2006-02-15 10:07:09 78 | 78 2 2006-02-15 10:07:09 79 | 79 7 2006-02-15 10:07:09 80 | 80 8 2006-02-15 10:07:09 81 | 81 14 2006-02-15 10:07:09 82 | 82 8 2006-02-15 10:07:09 83 | 83 8 2006-02-15 10:07:09 84 | 84 16 2006-02-15 10:07:09 85 | 85 6 2006-02-15 10:07:09 86 | 86 12 2006-02-15 10:07:09 87 | 87 16 2006-02-15 10:07:09 88 | 88 16 2006-02-15 10:07:09 89 | 89 2 2006-02-15 10:07:09 90 | 90 13 2006-02-15 10:07:09 91 | 91 4 2006-02-15 10:07:09 92 | 92 11 2006-02-15 10:07:09 93 | 93 13 2006-02-15 10:07:09 94 | 94 8 2006-02-15 10:07:09 95 | 95 13 2006-02-15 10:07:09 96 | 96 13 2006-02-15 10:07:09 97 | 97 1 2006-02-15 10:07:09 98 | 98 7 2006-02-15 10:07:09 99 | 99 5 2006-02-15 10:07:09 100 | 100 9 2006-02-15 10:07:09 101 | 101 6 2006-02-15 10:07:09 102 | 102 15 2006-02-15 10:07:09 103 | 103 16 2006-02-15 10:07:09 104 | 104 9 2006-02-15 10:07:09 105 | 105 1 2006-02-15 10:07:09 106 | 106 10 2006-02-15 10:07:09 107 | 107 7 2006-02-15 10:07:09 108 | 108 13 2006-02-15 10:07:09 109 | 109 13 2006-02-15 10:07:09 110 | 110 3 2006-02-15 10:07:09 111 | 111 1 2006-02-15 10:07:09 112 | 112 9 2006-02-15 10:07:09 113 | 113 15 2006-02-15 10:07:09 114 | 114 14 2006-02-15 10:07:09 115 | 115 1 2006-02-15 10:07:09 116 | 116 4 2006-02-15 10:07:09 117 | 117 10 2006-02-15 10:07:09 118 | 118 2 2006-02-15 10:07:09 119 | 119 5 2006-02-15 10:07:09 120 | 120 15 2006-02-15 10:07:09 121 | 121 2 2006-02-15 10:07:09 122 | 122 11 2006-02-15 10:07:09 123 | 123 16 2006-02-15 10:07:09 124 | 124 3 2006-02-15 10:07:09 125 | 125 16 2006-02-15 10:07:09 126 | 126 1 2006-02-15 10:07:09 127 | 127 5 2006-02-15 10:07:09 128 | 128 9 2006-02-15 10:07:09 129 | 129 6 2006-02-15 10:07:09 130 | 130 1 2006-02-15 10:07:09 131 | 131 4 2006-02-15 10:07:09 132 | 132 14 2006-02-15 10:07:09 133 | 133 12 2006-02-15 10:07:09 134 | 134 2 2006-02-15 10:07:09 135 | 135 15 2006-02-15 10:07:09 136 | 136 13 2006-02-15 10:07:09 137 | 137 14 2006-02-15 10:07:09 138 | 138 14 2006-02-15 10:07:09 139 | 139 8 2006-02-15 10:07:09 140 | 140 14 2006-02-15 10:07:09 141 | 141 10 2006-02-15 10:07:09 142 | 142 6 2006-02-15 10:07:09 143 | 143 7 2006-02-15 10:07:09 144 | 144 13 2006-02-15 10:07:09 145 | 145 8 2006-02-15 10:07:09 146 | 146 7 2006-02-15 10:07:09 147 | 147 8 2006-02-15 10:07:09 148 | 148 9 2006-02-15 10:07:09 149 | 149 3 2006-02-15 10:07:09 150 | 150 6 2006-02-15 10:07:09 151 | 151 14 2006-02-15 10:07:09 152 | 152 3 2006-02-15 10:07:09 153 | 153 14 2006-02-15 10:07:09 154 | 154 2 2006-02-15 10:07:09 155 | 155 13 2006-02-15 10:07:09 156 | 156 6 2006-02-15 10:07:09 157 | 157 3 2006-02-15 10:07:09 158 | 158 12 2006-02-15 10:07:09 159 | 159 5 2006-02-15 10:07:09 160 | 160 2 2006-02-15 10:07:09 161 | 161 12 2006-02-15 10:07:09 162 | 162 1 2006-02-15 10:07:09 163 | 163 13 2006-02-15 10:07:09 164 | 164 6 2006-02-15 10:07:09 165 | 165 14 2006-02-15 10:07:09 166 | 166 4 2006-02-15 10:07:09 167 | 167 16 2006-02-15 10:07:09 168 | 168 3 2006-02-15 10:07:09 169 | 169 16 2006-02-15 10:07:09 170 | 170 9 2006-02-15 10:07:09 171 | 171 11 2006-02-15 10:07:09 172 | 172 7 2006-02-15 10:07:09 173 | 173 7 2006-02-15 10:07:09 174 | 174 12 2006-02-15 10:07:09 175 | 175 8 2006-02-15 10:07:09 176 | 176 15 2006-02-15 10:07:09 177 | 177 14 2006-02-15 10:07:09 178 | 178 5 2006-02-15 10:07:09 179 | 179 7 2006-02-15 10:07:09 180 | 180 4 2006-02-15 10:07:09 181 | 181 16 2006-02-15 10:07:09 182 | 182 5 2006-02-15 10:07:09 183 | 183 8 2006-02-15 10:07:09 184 | 184 4 2006-02-15 10:07:09 185 | 185 9 2006-02-15 10:07:09 186 | 186 7 2006-02-15 10:07:09 187 | 187 15 2006-02-15 10:07:09 188 | 188 5 2006-02-15 10:07:09 189 | 189 10 2006-02-15 10:07:09 190 | 190 4 2006-02-15 10:07:09 191 | 191 3 2006-02-15 10:07:09 192 | 192 9 2006-02-15 10:07:09 193 | 193 2 2006-02-15 10:07:09 194 | 194 1 2006-02-15 10:07:09 195 | 195 14 2006-02-15 10:07:09 196 | 196 4 2006-02-15 10:07:09 197 | 197 15 2006-02-15 10:07:09 198 | 198 9 2006-02-15 10:07:09 199 | 199 6 2006-02-15 10:07:09 200 | 200 10 2006-02-15 10:07:09 201 | 201 9 2006-02-15 10:07:09 202 | 202 5 2006-02-15 10:07:09 203 | 203 14 2006-02-15 10:07:09 204 | 204 7 2006-02-15 10:07:09 205 | 205 1 2006-02-15 10:07:09 206 | 206 6 2006-02-15 10:07:09 207 | 207 9 2006-02-15 10:07:09 208 | 208 2 2006-02-15 10:07:09 209 | 209 7 2006-02-15 10:07:09 210 | 210 1 2006-02-15 10:07:09 211 | 211 10 2006-02-15 10:07:09 212 | 212 1 2006-02-15 10:07:09 213 | 213 8 2006-02-15 10:07:09 214 | 214 3 2006-02-15 10:07:09 215 | 215 10 2006-02-15 10:07:09 216 | 216 13 2006-02-15 10:07:09 217 | 217 10 2006-02-15 10:07:09 218 | 218 7 2006-02-15 10:07:09 219 | 219 6 2006-02-15 10:07:09 220 | 220 12 2006-02-15 10:07:09 221 | 221 6 2006-02-15 10:07:09 222 | 222 11 2006-02-15 10:07:09 223 | 223 2 2006-02-15 10:07:09 224 | 224 16 2006-02-15 10:07:09 225 | 225 7 2006-02-15 10:07:09 226 | 226 13 2006-02-15 10:07:09 227 | 227 10 2006-02-15 10:07:09 228 | 228 4 2006-02-15 10:07:09 229 | 229 1 2006-02-15 10:07:09 230 | 230 7 2006-02-15 10:07:09 231 | 231 8 2006-02-15 10:07:09 232 | 232 10 2006-02-15 10:07:09 233 | 233 16 2006-02-15 10:07:09 234 | 234 14 2006-02-15 10:07:09 235 | 235 14 2006-02-15 10:07:09 236 | 236 10 2006-02-15 10:07:09 237 | 237 15 2006-02-15 10:07:09 238 | 238 3 2006-02-15 10:07:09 239 | 239 2 2006-02-15 10:07:09 240 | 240 14 2006-02-15 10:07:09 241 | 241 2 2006-02-15 10:07:09 242 | 242 5 2006-02-15 10:07:09 243 | 243 2 2006-02-15 10:07:09 244 | 244 12 2006-02-15 10:07:09 245 | 245 2 2006-02-15 10:07:09 246 | 246 9 2006-02-15 10:07:09 247 | 247 5 2006-02-15 10:07:09 248 | 248 6 2006-02-15 10:07:09 249 | 249 4 2006-02-15 10:07:09 250 | 250 1 2006-02-15 10:07:09 251 | 251 13 2006-02-15 10:07:09 252 | 252 1 2006-02-15 10:07:09 253 | 253 1 2006-02-15 10:07:09 254 | 254 15 2006-02-15 10:07:09 255 | 255 12 2006-02-15 10:07:09 256 | 256 15 2006-02-15 10:07:09 257 | 257 16 2006-02-15 10:07:09 258 | 258 11 2006-02-15 10:07:09 259 | 259 2 2006-02-15 10:07:09 260 | 260 15 2006-02-15 10:07:09 261 | 261 6 2006-02-15 10:07:09 262 | 262 8 2006-02-15 10:07:09 263 | 263 15 2006-02-15 10:07:09 264 | 264 10 2006-02-15 10:07:09 265 | 265 5 2006-02-15 10:07:09 266 | 266 4 2006-02-15 10:07:09 267 | 267 13 2006-02-15 10:07:09 268 | 268 2 2006-02-15 10:07:09 269 | 269 8 2006-02-15 10:07:09 270 | 270 13 2006-02-15 10:07:09 271 | 271 1 2006-02-15 10:07:09 272 | 272 7 2006-02-15 10:07:09 273 | 273 8 2006-02-15 10:07:09 274 | 274 6 2006-02-15 10:07:09 275 | 275 11 2006-02-15 10:07:09 276 | 276 5 2006-02-15 10:07:09 277 | 277 11 2006-02-15 10:07:09 278 | 278 12 2006-02-15 10:07:09 279 | 279 15 2006-02-15 10:07:09 280 | 280 3 2006-02-15 10:07:09 281 | 281 10 2006-02-15 10:07:09 282 | 282 7 2006-02-15 10:07:09 283 | 283 13 2006-02-15 10:07:09 284 | 284 12 2006-02-15 10:07:09 285 | 285 14 2006-02-15 10:07:09 286 | 286 16 2006-02-15 10:07:09 287 | 287 1 2006-02-15 10:07:09 288 | 288 16 2006-02-15 10:07:09 289 | 289 13 2006-02-15 10:07:09 290 | 290 9 2006-02-15 10:07:09 291 | 291 15 2006-02-15 10:07:09 292 | 292 1 2006-02-15 10:07:09 293 | 293 15 2006-02-15 10:07:09 294 | 294 16 2006-02-15 10:07:09 295 | 295 6 2006-02-15 10:07:09 296 | 296 14 2006-02-15 10:07:09 297 | 297 4 2006-02-15 10:07:09 298 | 298 14 2006-02-15 10:07:09 299 | 299 16 2006-02-15 10:07:09 300 | 300 2 2006-02-15 10:07:09 301 | 301 11 2006-02-15 10:07:09 302 | 302 10 2006-02-15 10:07:09 303 | 303 1 2006-02-15 10:07:09 304 | 304 3 2006-02-15 10:07:09 305 | 305 13 2006-02-15 10:07:09 306 | 306 10 2006-02-15 10:07:09 307 | 307 16 2006-02-15 10:07:09 308 | 308 5 2006-02-15 10:07:09 309 | 309 8 2006-02-15 10:07:09 310 | 310 10 2006-02-15 10:07:09 311 | 311 9 2006-02-15 10:07:09 312 | 312 14 2006-02-15 10:07:09 313 | 313 11 2006-02-15 10:07:09 314 | 314 2 2006-02-15 10:07:09 315 | 315 8 2006-02-15 10:07:09 316 | 316 10 2006-02-15 10:07:09 317 | 317 5 2006-02-15 10:07:09 318 | 318 1 2006-02-15 10:07:09 319 | 319 14 2006-02-15 10:07:09 320 | 320 13 2006-02-15 10:07:09 321 | 321 13 2006-02-15 10:07:09 322 | 322 15 2006-02-15 10:07:09 323 | 323 15 2006-02-15 10:07:09 324 | 324 5 2006-02-15 10:07:09 325 | 325 2 2006-02-15 10:07:09 326 | 326 2 2006-02-15 10:07:09 327 | 327 1 2006-02-15 10:07:09 328 | 328 3 2006-02-15 10:07:09 329 | 329 1 2006-02-15 10:07:09 330 | 330 2 2006-02-15 10:07:09 331 | 331 10 2006-02-15 10:07:09 332 | 332 5 2006-02-15 10:07:09 333 | 333 12 2006-02-15 10:07:09 334 | 334 11 2006-02-15 10:07:09 335 | 335 5 2006-02-15 10:07:09 336 | 336 6 2006-02-15 10:07:09 337 | 337 9 2006-02-15 10:07:09 338 | 338 14 2006-02-15 10:07:09 339 | 339 16 2006-02-15 10:07:09 340 | 340 13 2006-02-15 10:07:09 341 | 341 4 2006-02-15 10:07:09 342 | 342 16 2006-02-15 10:07:09 343 | 343 3 2006-02-15 10:07:09 344 | 344 3 2006-02-15 10:07:09 345 | 345 8 2006-02-15 10:07:09 346 | 346 4 2006-02-15 10:07:09 347 | 347 16 2006-02-15 10:07:09 348 | 348 8 2006-02-15 10:07:09 349 | 349 2 2006-02-15 10:07:09 350 | 350 14 2006-02-15 10:07:09 351 | 351 11 2006-02-15 10:07:09 352 | 352 10 2006-02-15 10:07:09 353 | 353 9 2006-02-15 10:07:09 354 | 354 3 2006-02-15 10:07:09 355 | 355 2 2006-02-15 10:07:09 356 | 356 3 2006-02-15 10:07:09 357 | 357 4 2006-02-15 10:07:09 358 | 358 4 2006-02-15 10:07:09 359 | 359 8 2006-02-15 10:07:09 360 | 360 1 2006-02-15 10:07:09 361 | 361 15 2006-02-15 10:07:09 362 | 362 10 2006-02-15 10:07:09 363 | 363 12 2006-02-15 10:07:09 364 | 364 13 2006-02-15 10:07:09 365 | 365 5 2006-02-15 10:07:09 366 | 366 7 2006-02-15 10:07:09 367 | 367 14 2006-02-15 10:07:09 368 | 368 7 2006-02-15 10:07:09 369 | 369 14 2006-02-15 10:07:09 370 | 370 3 2006-02-15 10:07:09 371 | 371 1 2006-02-15 10:07:09 372 | 372 15 2006-02-15 10:07:09 373 | 373 3 2006-02-15 10:07:09 374 | 374 14 2006-02-15 10:07:09 375 | 375 1 2006-02-15 10:07:09 376 | 376 9 2006-02-15 10:07:09 377 | 377 8 2006-02-15 10:07:09 378 | 378 12 2006-02-15 10:07:09 379 | 379 7 2006-02-15 10:07:09 380 | 380 9 2006-02-15 10:07:09 381 | 381 10 2006-02-15 10:07:09 382 | 382 10 2006-02-15 10:07:09 383 | 383 15 2006-02-15 10:07:09 384 | 384 12 2006-02-15 10:07:09 385 | 385 5 2006-02-15 10:07:09 386 | 386 16 2006-02-15 10:07:09 387 | 387 10 2006-02-15 10:07:09 388 | 388 5 2006-02-15 10:07:09 389 | 389 15 2006-02-15 10:07:09 390 | 390 14 2006-02-15 10:07:09 391 | 391 8 2006-02-15 10:07:09 392 | 392 3 2006-02-15 10:07:09 393 | 393 6 2006-02-15 10:07:09 394 | 394 14 2006-02-15 10:07:09 395 | 395 1 2006-02-15 10:07:09 396 | 396 7 2006-02-15 10:07:09 397 | 397 14 2006-02-15 10:07:09 398 | 398 12 2006-02-15 10:07:09 399 | 399 9 2006-02-15 10:07:09 400 | 400 6 2006-02-15 10:07:09 401 | 401 7 2006-02-15 10:07:09 402 | 402 2 2006-02-15 10:07:09 403 | 403 7 2006-02-15 10:07:09 404 | 404 5 2006-02-15 10:07:09 405 | 405 16 2006-02-15 10:07:09 406 | 406 10 2006-02-15 10:07:09 407 | 407 6 2006-02-15 10:07:09 408 | 408 10 2006-02-15 10:07:09 409 | 409 3 2006-02-15 10:07:09 410 | 410 5 2006-02-15 10:07:09 411 | 411 12 2006-02-15 10:07:09 412 | 412 6 2006-02-15 10:07:09 413 | 413 5 2006-02-15 10:07:09 414 | 414 9 2006-02-15 10:07:09 415 | 415 11 2006-02-15 10:07:09 416 | 416 9 2006-02-15 10:07:09 417 | 417 1 2006-02-15 10:07:09 418 | 418 7 2006-02-15 10:07:09 419 | 419 8 2006-02-15 10:07:09 420 | 420 15 2006-02-15 10:07:09 421 | 421 9 2006-02-15 10:07:09 422 | 422 14 2006-02-15 10:07:09 423 | 423 3 2006-02-15 10:07:09 424 | 424 3 2006-02-15 10:07:09 425 | 425 4 2006-02-15 10:07:09 426 | 426 12 2006-02-15 10:07:09 427 | 427 6 2006-02-15 10:07:09 428 | 428 8 2006-02-15 10:07:09 429 | 429 15 2006-02-15 10:07:09 430 | 430 2 2006-02-15 10:07:09 431 | 431 9 2006-02-15 10:07:09 432 | 432 4 2006-02-15 10:07:09 433 | 433 2 2006-02-15 10:07:09 434 | 434 16 2006-02-15 10:07:09 435 | 435 9 2006-02-15 10:07:09 436 | 436 13 2006-02-15 10:07:09 437 | 437 8 2006-02-15 10:07:09 438 | 438 10 2006-02-15 10:07:09 439 | 439 7 2006-02-15 10:07:09 440 | 440 9 2006-02-15 10:07:09 441 | 441 6 2006-02-15 10:07:09 442 | 442 8 2006-02-15 10:07:09 443 | 443 5 2006-02-15 10:07:09 444 | 444 5 2006-02-15 10:07:09 445 | 445 4 2006-02-15 10:07:09 446 | 446 15 2006-02-15 10:07:09 447 | 447 10 2006-02-15 10:07:09 448 | 448 13 2006-02-15 10:07:09 449 | 449 14 2006-02-15 10:07:09 450 | 450 3 2006-02-15 10:07:09 451 | 451 16 2006-02-15 10:07:09 452 | 452 9 2006-02-15 10:07:09 453 | 453 15 2006-02-15 10:07:09 454 | 454 12 2006-02-15 10:07:09 455 | 455 9 2006-02-15 10:07:09 456 | 456 2 2006-02-15 10:07:09 457 | 457 6 2006-02-15 10:07:09 458 | 458 8 2006-02-15 10:07:09 459 | 459 9 2006-02-15 10:07:09 460 | 460 9 2006-02-15 10:07:09 461 | 461 2 2006-02-15 10:07:09 462 | 462 12 2006-02-15 10:07:09 463 | 463 15 2006-02-15 10:07:09 464 | 464 2 2006-02-15 10:07:09 465 | 465 13 2006-02-15 10:07:09 466 | 466 6 2006-02-15 10:07:09 467 | 467 9 2006-02-15 10:07:09 468 | 468 3 2006-02-15 10:07:09 469 | 469 4 2006-02-15 10:07:09 470 | 470 2 2006-02-15 10:07:09 471 | 471 4 2006-02-15 10:07:09 472 | 472 16 2006-02-15 10:07:09 473 | 473 7 2006-02-15 10:07:09 474 | 474 15 2006-02-15 10:07:09 475 | 475 11 2006-02-15 10:07:09 476 | 476 8 2006-02-15 10:07:09 477 | 477 12 2006-02-15 10:07:09 478 | 478 5 2006-02-15 10:07:09 479 | 479 8 2006-02-15 10:07:09 480 | 480 4 2006-02-15 10:07:09 481 | 481 13 2006-02-15 10:07:09 482 | 482 4 2006-02-15 10:07:09 483 | 483 10 2006-02-15 10:07:09 484 | 484 4 2006-02-15 10:07:09 485 | 485 3 2006-02-15 10:07:09 486 | 486 9 2006-02-15 10:07:09 487 | 487 4 2006-02-15 10:07:09 488 | 488 15 2006-02-15 10:07:09 489 | 489 2 2006-02-15 10:07:09 490 | 490 13 2006-02-15 10:07:09 491 | 491 3 2006-02-15 10:07:09 492 | 492 13 2006-02-15 10:07:09 493 | 493 9 2006-02-15 10:07:09 494 | 494 11 2006-02-15 10:07:09 495 | 495 11 2006-02-15 10:07:09 496 | 496 16 2006-02-15 10:07:09 497 | 497 6 2006-02-15 10:07:09 498 | 498 8 2006-02-15 10:07:09 499 | 499 8 2006-02-15 10:07:09 500 | 500 9 2006-02-15 10:07:09 501 | 501 1 2006-02-15 10:07:09 502 | 502 5 2006-02-15 10:07:09 503 | 503 15 2006-02-15 10:07:09 504 | 504 7 2006-02-15 10:07:09 505 | 505 3 2006-02-15 10:07:09 506 | 506 11 2006-02-15 10:07:09 507 | 507 10 2006-02-15 10:07:09 508 | 508 10 2006-02-15 10:07:09 509 | 509 3 2006-02-15 10:07:09 510 | 510 2 2006-02-15 10:07:09 511 | 511 1 2006-02-15 10:07:09 512 | 512 4 2006-02-15 10:07:09 513 | 513 16 2006-02-15 10:07:09 514 | 514 7 2006-02-15 10:07:09 515 | 515 3 2006-02-15 10:07:09 516 | 516 12 2006-02-15 10:07:09 517 | 517 15 2006-02-15 10:07:09 518 | 518 16 2006-02-15 10:07:09 519 | 519 15 2006-02-15 10:07:09 520 | 520 14 2006-02-15 10:07:09 521 | 521 7 2006-02-15 10:07:09 522 | 522 5 2006-02-15 10:07:09 523 | 523 4 2006-02-15 10:07:09 524 | 524 5 2006-02-15 10:07:09 525 | 525 4 2006-02-15 10:07:09 526 | 526 16 2006-02-15 10:07:09 527 | 527 11 2006-02-15 10:07:09 528 | 528 8 2006-02-15 10:07:09 529 | 529 5 2006-02-15 10:07:09 530 | 530 1 2006-02-15 10:07:09 531 | 531 9 2006-02-15 10:07:09 532 | 532 15 2006-02-15 10:07:09 533 | 533 9 2006-02-15 10:07:09 534 | 534 8 2006-02-15 10:07:09 535 | 535 11 2006-02-15 10:07:09 536 | 536 4 2006-02-15 10:07:09 537 | 537 4 2006-02-15 10:07:09 538 | 538 13 2006-02-15 10:07:09 539 | 539 7 2006-02-15 10:07:09 540 | 540 12 2006-02-15 10:07:09 541 | 541 2 2006-02-15 10:07:09 542 | 542 1 2006-02-15 10:07:09 543 | 543 16 2006-02-15 10:07:09 544 | 544 6 2006-02-15 10:07:09 545 | 545 9 2006-02-15 10:07:09 546 | 546 10 2006-02-15 10:07:09 547 | 547 3 2006-02-15 10:07:09 548 | 548 4 2006-02-15 10:07:09 549 | 549 1 2006-02-15 10:07:09 550 | 550 8 2006-02-15 10:07:09 551 | 551 13 2006-02-15 10:07:09 552 | 552 6 2006-02-15 10:07:09 553 | 553 3 2006-02-15 10:07:09 554 | 554 4 2006-02-15 10:07:09 555 | 555 5 2006-02-15 10:07:09 556 | 556 10 2006-02-15 10:07:09 557 | 557 8 2006-02-15 10:07:09 558 | 558 13 2006-02-15 10:07:09 559 | 559 14 2006-02-15 10:07:09 560 | 560 10 2006-02-15 10:07:09 561 | 561 13 2006-02-15 10:07:09 562 | 562 12 2006-02-15 10:07:09 563 | 563 10 2006-02-15 10:07:09 564 | 564 2 2006-02-15 10:07:09 565 | 565 9 2006-02-15 10:07:09 566 | 566 9 2006-02-15 10:07:09 567 | 567 9 2006-02-15 10:07:09 568 | 568 5 2006-02-15 10:07:09 569 | 569 2 2006-02-15 10:07:09 570 | 570 15 2006-02-15 10:07:09 571 | 571 6 2006-02-15 10:07:09 572 | 572 14 2006-02-15 10:07:09 573 | 573 3 2006-02-15 10:07:09 574 | 574 1 2006-02-15 10:07:09 575 | 575 6 2006-02-15 10:07:09 576 | 576 6 2006-02-15 10:07:09 577 | 577 15 2006-02-15 10:07:09 578 | 578 4 2006-02-15 10:07:09 579 | 579 1 2006-02-15 10:07:09 580 | 580 13 2006-02-15 10:07:09 581 | 581 12 2006-02-15 10:07:09 582 | 582 2 2006-02-15 10:07:09 583 | 583 2 2006-02-15 10:07:09 584 | 584 9 2006-02-15 10:07:09 585 | 585 7 2006-02-15 10:07:09 586 | 586 1 2006-02-15 10:07:09 587 | 587 6 2006-02-15 10:07:09 588 | 588 3 2006-02-15 10:07:09 589 | 589 6 2006-02-15 10:07:09 590 | 590 13 2006-02-15 10:07:09 591 | 591 10 2006-02-15 10:07:09 592 | 592 12 2006-02-15 10:07:09 593 | 593 11 2006-02-15 10:07:09 594 | 594 1 2006-02-15 10:07:09 595 | 595 9 2006-02-15 10:07:09 596 | 596 10 2006-02-15 10:07:09 597 | 597 10 2006-02-15 10:07:09 598 | 598 15 2006-02-15 10:07:09 599 | 599 15 2006-02-15 10:07:09 600 | 600 11 2006-02-15 10:07:09 601 | 601 16 2006-02-15 10:07:09 602 | 602 14 2006-02-15 10:07:09 603 | 603 8 2006-02-15 10:07:09 604 | 604 5 2006-02-15 10:07:09 605 | 605 9 2006-02-15 10:07:09 606 | 606 15 2006-02-15 10:07:09 607 | 607 9 2006-02-15 10:07:09 608 | 608 3 2006-02-15 10:07:09 609 | 609 16 2006-02-15 10:07:09 610 | 610 8 2006-02-15 10:07:09 611 | 611 4 2006-02-15 10:07:09 612 | 612 15 2006-02-15 10:07:09 613 | 613 5 2006-02-15 10:07:09 614 | 614 10 2006-02-15 10:07:09 615 | 615 2 2006-02-15 10:07:09 616 | 616 6 2006-02-15 10:07:09 617 | 617 8 2006-02-15 10:07:09 618 | 618 7 2006-02-15 10:07:09 619 | 619 15 2006-02-15 10:07:09 620 | 620 14 2006-02-15 10:07:09 621 | 621 8 2006-02-15 10:07:09 622 | 622 6 2006-02-15 10:07:09 623 | 623 9 2006-02-15 10:07:09 624 | 624 10 2006-02-15 10:07:09 625 | 625 14 2006-02-15 10:07:09 626 | 626 3 2006-02-15 10:07:09 627 | 627 6 2006-02-15 10:07:09 628 | 628 15 2006-02-15 10:07:09 629 | 629 6 2006-02-15 10:07:09 630 | 630 7 2006-02-15 10:07:09 631 | 631 15 2006-02-15 10:07:09 632 | 632 13 2006-02-15 10:07:09 633 | 633 4 2006-02-15 10:07:09 634 | 634 8 2006-02-15 10:07:09 635 | 635 13 2006-02-15 10:07:09 636 | 636 12 2006-02-15 10:07:09 637 | 637 14 2006-02-15 10:07:09 638 | 638 5 2006-02-15 10:07:09 639 | 639 8 2006-02-15 10:07:09 640 | 640 9 2006-02-15 10:07:09 641 | 641 9 2006-02-15 10:07:09 642 | 642 16 2006-02-15 10:07:09 643 | 643 7 2006-02-15 10:07:09 644 | 644 2 2006-02-15 10:07:09 645 | 645 16 2006-02-15 10:07:09 646 | 646 10 2006-02-15 10:07:09 647 | 647 12 2006-02-15 10:07:09 648 | 648 16 2006-02-15 10:07:09 649 | 649 2 2006-02-15 10:07:09 650 | 650 6 2006-02-15 10:07:09 651 | 651 2 2006-02-15 10:07:09 652 | 652 4 2006-02-15 10:07:09 653 | 653 11 2006-02-15 10:07:09 654 | 654 10 2006-02-15 10:07:09 655 | 655 14 2006-02-15 10:07:09 656 | 656 16 2006-02-15 10:07:09 657 | 657 5 2006-02-15 10:07:09 658 | 658 11 2006-02-15 10:07:09 659 | 659 1 2006-02-15 10:07:09 660 | 660 5 2006-02-15 10:07:09 661 | 661 9 2006-02-15 10:07:09 662 | 662 7 2006-02-15 10:07:09 663 | 663 4 2006-02-15 10:07:09 664 | 664 1 2006-02-15 10:07:09 665 | 665 11 2006-02-15 10:07:09 666 | 666 7 2006-02-15 10:07:09 667 | 667 15 2006-02-15 10:07:09 668 | 668 15 2006-02-15 10:07:09 669 | 669 9 2006-02-15 10:07:09 670 | 670 6 2006-02-15 10:07:09 671 | 671 15 2006-02-15 10:07:09 672 | 672 5 2006-02-15 10:07:09 673 | 673 12 2006-02-15 10:07:09 674 | 674 9 2006-02-15 10:07:09 675 | 675 13 2006-02-15 10:07:09 676 | 676 15 2006-02-15 10:07:09 677 | 677 13 2006-02-15 10:07:09 678 | 678 15 2006-02-15 10:07:09 679 | 679 8 2006-02-15 10:07:09 680 | 680 5 2006-02-15 10:07:09 681 | 681 15 2006-02-15 10:07:09 682 | 682 8 2006-02-15 10:07:09 683 | 683 7 2006-02-15 10:07:09 684 | 684 10 2006-02-15 10:07:09 685 | 685 13 2006-02-15 10:07:09 686 | 686 13 2006-02-15 10:07:09 687 | 687 6 2006-02-15 10:07:09 688 | 688 3 2006-02-15 10:07:09 689 | 689 9 2006-02-15 10:07:09 690 | 690 2 2006-02-15 10:07:09 691 | 691 15 2006-02-15 10:07:09 692 | 692 2 2006-02-15 10:07:09 693 | 693 2 2006-02-15 10:07:09 694 | 694 4 2006-02-15 10:07:09 695 | 695 8 2006-02-15 10:07:09 696 | 696 2 2006-02-15 10:07:09 697 | 697 1 2006-02-15 10:07:09 698 | 698 6 2006-02-15 10:07:09 699 | 699 10 2006-02-15 10:07:09 700 | 700 8 2006-02-15 10:07:09 701 | 701 10 2006-02-15 10:07:09 702 | 702 11 2006-02-15 10:07:09 703 | 703 2 2006-02-15 10:07:09 704 | 704 5 2006-02-15 10:07:09 705 | 705 9 2006-02-15 10:07:09 706 | 706 7 2006-02-15 10:07:09 707 | 707 1 2006-02-15 10:07:09 708 | 708 6 2006-02-15 10:07:09 709 | 709 7 2006-02-15 10:07:09 710 | 710 8 2006-02-15 10:07:09 711 | 711 14 2006-02-15 10:07:09 712 | 712 6 2006-02-15 10:07:09 713 | 713 6 2006-02-15 10:07:09 714 | 714 14 2006-02-15 10:07:09 715 | 715 8 2006-02-15 10:07:09 716 | 716 11 2006-02-15 10:07:09 717 | 717 1 2006-02-15 10:07:09 718 | 718 12 2006-02-15 10:07:09 719 | 719 15 2006-02-15 10:07:09 720 | 720 13 2006-02-15 10:07:09 721 | 721 12 2006-02-15 10:07:09 722 | 722 11 2006-02-15 10:07:09 723 | 723 14 2006-02-15 10:07:09 724 | 724 8 2006-02-15 10:07:09 725 | 725 4 2006-02-15 10:07:09 726 | 726 9 2006-02-15 10:07:09 727 | 727 8 2006-02-15 10:07:09 728 | 728 7 2006-02-15 10:07:09 729 | 729 15 2006-02-15 10:07:09 730 | 730 13 2006-02-15 10:07:09 731 | 731 4 2006-02-15 10:07:09 732 | 732 1 2006-02-15 10:07:09 733 | 733 15 2006-02-15 10:07:09 734 | 734 6 2006-02-15 10:07:09 735 | 735 3 2006-02-15 10:07:09 736 | 736 8 2006-02-15 10:07:09 737 | 737 11 2006-02-15 10:07:09 738 | 738 9 2006-02-15 10:07:09 739 | 739 7 2006-02-15 10:07:09 740 | 740 11 2006-02-15 10:07:09 741 | 741 12 2006-02-15 10:07:09 742 | 742 10 2006-02-15 10:07:09 743 | 743 2 2006-02-15 10:07:09 744 | 744 4 2006-02-15 10:07:09 745 | 745 15 2006-02-15 10:07:09 746 | 746 10 2006-02-15 10:07:09 747 | 747 10 2006-02-15 10:07:09 748 | 748 1 2006-02-15 10:07:09 749 | 749 11 2006-02-15 10:07:09 750 | 750 13 2006-02-15 10:07:09 751 | 751 13 2006-02-15 10:07:09 752 | 752 12 2006-02-15 10:07:09 753 | 753 8 2006-02-15 10:07:09 754 | 754 5 2006-02-15 10:07:09 755 | 755 3 2006-02-15 10:07:09 756 | 756 5 2006-02-15 10:07:09 757 | 757 6 2006-02-15 10:07:09 758 | 758 7 2006-02-15 10:07:09 759 | 759 13 2006-02-15 10:07:09 760 | 760 13 2006-02-15 10:07:09 761 | 761 3 2006-02-15 10:07:09 762 | 762 10 2006-02-15 10:07:09 763 | 763 15 2006-02-15 10:07:09 764 | 764 15 2006-02-15 10:07:09 765 | 765 5 2006-02-15 10:07:09 766 | 766 7 2006-02-15 10:07:09 767 | 767 12 2006-02-15 10:07:09 768 | 768 3 2006-02-15 10:07:09 769 | 769 9 2006-02-15 10:07:09 770 | 770 9 2006-02-15 10:07:09 771 | 771 7 2006-02-15 10:07:09 772 | 772 7 2006-02-15 10:07:09 773 | 773 15 2006-02-15 10:07:09 774 | 774 5 2006-02-15 10:07:09 775 | 775 7 2006-02-15 10:07:09 776 | 776 6 2006-02-15 10:07:09 777 | 777 15 2006-02-15 10:07:09 778 | 778 8 2006-02-15 10:07:09 779 | 779 15 2006-02-15 10:07:09 780 | 780 8 2006-02-15 10:07:09 781 | 781 10 2006-02-15 10:07:09 782 | 782 15 2006-02-15 10:07:09 783 | 783 16 2006-02-15 10:07:09 784 | 784 16 2006-02-15 10:07:09 785 | 785 16 2006-02-15 10:07:09 786 | 786 3 2006-02-15 10:07:09 787 | 787 16 2006-02-15 10:07:09 788 | 788 6 2006-02-15 10:07:09 789 | 789 9 2006-02-15 10:07:09 790 | 790 7 2006-02-15 10:07:09 791 | 791 6 2006-02-15 10:07:09 792 | 792 9 2006-02-15 10:07:09 793 | 793 1 2006-02-15 10:07:09 794 | 794 1 2006-02-15 10:07:09 795 | 795 8 2006-02-15 10:07:09 796 | 796 15 2006-02-15 10:07:09 797 | 797 12 2006-02-15 10:07:09 798 | 798 14 2006-02-15 10:07:09 799 | 799 11 2006-02-15 10:07:09 800 | 800 11 2006-02-15 10:07:09 801 | 801 3 2006-02-15 10:07:09 802 | 802 1 2006-02-15 10:07:09 803 | 803 7 2006-02-15 10:07:09 804 | 804 11 2006-02-15 10:07:09 805 | 805 2 2006-02-15 10:07:09 806 | 806 13 2006-02-15 10:07:09 807 | 807 10 2006-02-15 10:07:09 808 | 808 4 2006-02-15 10:07:09 809 | 809 15 2006-02-15 10:07:09 810 | 810 8 2006-02-15 10:07:09 811 | 811 16 2006-02-15 10:07:09 812 | 812 6 2006-02-15 10:07:09 813 | 813 15 2006-02-15 10:07:09 814 | 814 5 2006-02-15 10:07:09 815 | 815 4 2006-02-15 10:07:09 816 | 816 2 2006-02-15 10:07:09 817 | 817 14 2006-02-15 10:07:09 818 | 818 7 2006-02-15 10:07:09 819 | 819 12 2006-02-15 10:07:09 820 | 820 2 2006-02-15 10:07:09 821 | 821 9 2006-02-15 10:07:09 822 | 822 8 2006-02-15 10:07:09 823 | 823 1 2006-02-15 10:07:09 824 | 824 8 2006-02-15 10:07:09 825 | 825 1 2006-02-15 10:07:09 826 | 826 16 2006-02-15 10:07:09 827 | 827 7 2006-02-15 10:07:09 828 | 828 4 2006-02-15 10:07:09 829 | 829 8 2006-02-15 10:07:09 830 | 830 11 2006-02-15 10:07:09 831 | 831 14 2006-02-15 10:07:09 832 | 832 8 2006-02-15 10:07:09 833 | 833 3 2006-02-15 10:07:09 834 | 834 6 2006-02-15 10:07:09 835 | 835 10 2006-02-15 10:07:09 836 | 836 15 2006-02-15 10:07:09 837 | 837 5 2006-02-15 10:07:09 838 | 838 1 2006-02-15 10:07:09 839 | 839 14 2006-02-15 10:07:09 840 | 840 10 2006-02-15 10:07:09 841 | 841 15 2006-02-15 10:07:09 842 | 842 10 2006-02-15 10:07:09 843 | 843 4 2006-02-15 10:07:09 844 | 844 15 2006-02-15 10:07:09 845 | 845 9 2006-02-15 10:07:09 846 | 846 13 2006-02-15 10:07:09 847 | 847 13 2006-02-15 10:07:09 848 | 848 16 2006-02-15 10:07:09 849 | 849 2 2006-02-15 10:07:09 850 | 850 1 2006-02-15 10:07:09 851 | 851 15 2006-02-15 10:07:09 852 | 852 3 2006-02-15 10:07:09 853 | 853 3 2006-02-15 10:07:09 854 | 854 11 2006-02-15 10:07:09 855 | 855 6 2006-02-15 10:07:09 856 | 856 11 2006-02-15 10:07:09 857 | 857 5 2006-02-15 10:07:09 858 | 858 5 2006-02-15 10:07:09 859 | 859 2 2006-02-15 10:07:09 860 | 860 14 2006-02-15 10:07:09 861 | 861 10 2006-02-15 10:07:09 862 | 862 4 2006-02-15 10:07:09 863 | 863 14 2006-02-15 10:07:09 864 | 864 3 2006-02-15 10:07:09 865 | 865 2 2006-02-15 10:07:09 866 | 866 8 2006-02-15 10:07:09 867 | 867 8 2006-02-15 10:07:09 868 | 868 16 2006-02-15 10:07:09 869 | 869 1 2006-02-15 10:07:09 870 | 870 11 2006-02-15 10:07:09 871 | 871 5 2006-02-15 10:07:09 872 | 872 16 2006-02-15 10:07:09 873 | 873 3 2006-02-15 10:07:09 874 | 874 4 2006-02-15 10:07:09 875 | 875 15 2006-02-15 10:07:09 876 | 876 11 2006-02-15 10:07:09 877 | 877 12 2006-02-15 10:07:09 878 | 878 16 2006-02-15 10:07:09 879 | 879 12 2006-02-15 10:07:09 880 | 880 2 2006-02-15 10:07:09 881 | 881 11 2006-02-15 10:07:09 882 | 882 7 2006-02-15 10:07:09 883 | 883 3 2006-02-15 10:07:09 884 | 884 12 2006-02-15 10:07:09 885 | 885 11 2006-02-15 10:07:09 886 | 886 2 2006-02-15 10:07:09 887 | 887 2 2006-02-15 10:07:09 888 | 888 6 2006-02-15 10:07:09 889 | 889 3 2006-02-15 10:07:09 890 | 890 15 2006-02-15 10:07:09 891 | 891 4 2006-02-15 10:07:09 892 | 892 2 2006-02-15 10:07:09 893 | 893 14 2006-02-15 10:07:09 894 | 894 16 2006-02-15 10:07:09 895 | 895 4 2006-02-15 10:07:09 896 | 896 3 2006-02-15 10:07:09 897 | 897 7 2006-02-15 10:07:09 898 | 898 15 2006-02-15 10:07:09 899 | 899 4 2006-02-15 10:07:09 900 | 900 9 2006-02-15 10:07:09 901 | 901 2 2006-02-15 10:07:09 902 | 902 15 2006-02-15 10:07:09 903 | 903 16 2006-02-15 10:07:09 904 | 904 11 2006-02-15 10:07:09 905 | 905 5 2006-02-15 10:07:09 906 | 906 5 2006-02-15 10:07:09 907 | 907 7 2006-02-15 10:07:09 908 | 908 9 2006-02-15 10:07:09 909 | 909 11 2006-02-15 10:07:09 910 | 910 7 2006-02-15 10:07:09 911 | 911 1 2006-02-15 10:07:09 912 | 912 14 2006-02-15 10:07:09 913 | 913 13 2006-02-15 10:07:09 914 | 914 16 2006-02-15 10:07:09 915 | 915 1 2006-02-15 10:07:09 916 | 916 2 2006-02-15 10:07:09 917 | 917 15 2006-02-15 10:07:09 918 | 918 3 2006-02-15 10:07:09 919 | 919 10 2006-02-15 10:07:09 920 | 920 13 2006-02-15 10:07:09 921 | 921 12 2006-02-15 10:07:09 922 | 922 11 2006-02-15 10:07:09 923 | 923 7 2006-02-15 10:07:09 924 | 924 14 2006-02-15 10:07:09 925 | 925 6 2006-02-15 10:07:09 926 | 926 6 2006-02-15 10:07:09 927 | 927 1 2006-02-15 10:07:09 928 | 928 3 2006-02-15 10:07:09 929 | 929 9 2006-02-15 10:07:09 930 | 930 14 2006-02-15 10:07:09 931 | 931 16 2006-02-15 10:07:09 932 | 932 5 2006-02-15 10:07:09 933 | 933 13 2006-02-15 10:07:09 934 | 934 10 2006-02-15 10:07:09 935 | 935 13 2006-02-15 10:07:09 936 | 936 12 2006-02-15 10:07:09 937 | 937 13 2006-02-15 10:07:09 938 | 938 5 2006-02-15 10:07:09 939 | 939 5 2006-02-15 10:07:09 940 | 940 15 2006-02-15 10:07:09 941 | 941 10 2006-02-15 10:07:09 942 | 942 7 2006-02-15 10:07:09 943 | 943 6 2006-02-15 10:07:09 944 | 944 7 2006-02-15 10:07:09 945 | 945 6 2006-02-15 10:07:09 946 | 946 8 2006-02-15 10:07:09 947 | 947 9 2006-02-15 10:07:09 948 | 948 13 2006-02-15 10:07:09 949 | 949 10 2006-02-15 10:07:09 950 | 950 4 2006-02-15 10:07:09 951 | 951 4 2006-02-15 10:07:09 952 | 952 6 2006-02-15 10:07:09 953 | 953 2 2006-02-15 10:07:09 954 | 954 13 2006-02-15 10:07:09 955 | 955 3 2006-02-15 10:07:09 956 | 956 10 2006-02-15 10:07:09 957 | 957 9 2006-02-15 10:07:09 958 | 958 7 2006-02-15 10:07:09 959 | 959 3 2006-02-15 10:07:09 960 | 960 6 2006-02-15 10:07:09 961 | 961 9 2006-02-15 10:07:09 962 | 962 4 2006-02-15 10:07:09 963 | 963 2 2006-02-15 10:07:09 964 | 964 1 2006-02-15 10:07:09 965 | 965 11 2006-02-15 10:07:09 966 | 966 6 2006-02-15 10:07:09 967 | 967 14 2006-02-15 10:07:09 968 | 968 1 2006-02-15 10:07:09 969 | 969 7 2006-02-15 10:07:09 970 | 970 4 2006-02-15 10:07:09 971 | 971 9 2006-02-15 10:07:09 972 | 972 14 2006-02-15 10:07:09 973 | 973 6 2006-02-15 10:07:09 974 | 974 13 2006-02-15 10:07:09 975 | 975 8 2006-02-15 10:07:09 976 | 976 10 2006-02-15 10:07:09 977 | 977 16 2006-02-15 10:07:09 978 | 978 5 2006-02-15 10:07:09 979 | 979 7 2006-02-15 10:07:09 980 | 980 12 2006-02-15 10:07:09 981 | 981 16 2006-02-15 10:07:09 982 | 982 1 2006-02-15 10:07:09 983 | 983 12 2006-02-15 10:07:09 984 | 984 9 2006-02-15 10:07:09 985 | 985 14 2006-02-15 10:07:09 986 | 986 2 2006-02-15 10:07:09 987 | 987 12 2006-02-15 10:07:09 988 | 988 16 2006-02-15 10:07:09 989 | 989 16 2006-02-15 10:07:09 990 | 990 11 2006-02-15 10:07:09 991 | 991 1 2006-02-15 10:07:09 992 | 992 6 2006-02-15 10:07:09 993 | 993 3 2006-02-15 10:07:09 994 | 994 13 2006-02-15 10:07:09 995 | 995 11 2006-02-15 10:07:09 996 | 996 6 2006-02-15 10:07:09 997 | 997 12 2006-02-15 10:07:09 998 | 998 11 2006-02-15 10:07:09 999 | 999 3 2006-02-15 10:07:09 1000 | 1000 5 2006-02-15 10:07:09 1001 | \. 1002 | 1003 | 1004 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3067.dat: -------------------------------------------------------------------------------- 1 | 1 A Corua (La Corua) 87 2006-02-15 09:45:25 2 | 2 Abha 82 2006-02-15 09:45:25 3 | 3 Abu Dhabi 101 2006-02-15 09:45:25 4 | 4 Acua 60 2006-02-15 09:45:25 5 | 5 Adana 97 2006-02-15 09:45:25 6 | 6 Addis Abeba 31 2006-02-15 09:45:25 7 | 7 Aden 107 2006-02-15 09:45:25 8 | 8 Adoni 44 2006-02-15 09:45:25 9 | 9 Ahmadnagar 44 2006-02-15 09:45:25 10 | 10 Akishima 50 2006-02-15 09:45:25 11 | 11 Akron 103 2006-02-15 09:45:25 12 | 12 al-Ayn 101 2006-02-15 09:45:25 13 | 13 al-Hawiya 82 2006-02-15 09:45:25 14 | 14 al-Manama 11 2006-02-15 09:45:25 15 | 15 al-Qadarif 89 2006-02-15 09:45:25 16 | 16 al-Qatif 82 2006-02-15 09:45:25 17 | 17 Alessandria 49 2006-02-15 09:45:25 18 | 18 Allappuzha (Alleppey) 44 2006-02-15 09:45:25 19 | 19 Allende 60 2006-02-15 09:45:25 20 | 20 Almirante Brown 6 2006-02-15 09:45:25 21 | 21 Alvorada 15 2006-02-15 09:45:25 22 | 22 Ambattur 44 2006-02-15 09:45:25 23 | 23 Amersfoort 67 2006-02-15 09:45:25 24 | 24 Amroha 44 2006-02-15 09:45:25 25 | 25 Angra dos Reis 15 2006-02-15 09:45:25 26 | 26 Anpolis 15 2006-02-15 09:45:25 27 | 27 Antofagasta 22 2006-02-15 09:45:25 28 | 28 Aparecida de Goinia 15 2006-02-15 09:45:25 29 | 29 Apeldoorn 67 2006-02-15 09:45:25 30 | 30 Araatuba 15 2006-02-15 09:45:25 31 | 31 Arak 46 2006-02-15 09:45:25 32 | 32 Arecibo 77 2006-02-15 09:45:25 33 | 33 Arlington 103 2006-02-15 09:45:25 34 | 34 Ashdod 48 2006-02-15 09:45:25 35 | 35 Ashgabat 98 2006-02-15 09:45:25 36 | 36 Ashqelon 48 2006-02-15 09:45:25 37 | 37 Asuncin 73 2006-02-15 09:45:25 38 | 38 Athenai 39 2006-02-15 09:45:25 39 | 39 Atinsk 80 2006-02-15 09:45:25 40 | 40 Atlixco 60 2006-02-15 09:45:25 41 | 41 Augusta-Richmond County 103 2006-02-15 09:45:25 42 | 42 Aurora 103 2006-02-15 09:45:25 43 | 43 Avellaneda 6 2006-02-15 09:45:25 44 | 44 Bag 15 2006-02-15 09:45:25 45 | 45 Baha Blanca 6 2006-02-15 09:45:25 46 | 46 Baicheng 23 2006-02-15 09:45:25 47 | 47 Baiyin 23 2006-02-15 09:45:25 48 | 48 Baku 10 2006-02-15 09:45:25 49 | 49 Balaiha 80 2006-02-15 09:45:25 50 | 50 Balikesir 97 2006-02-15 09:45:25 51 | 51 Balurghat 44 2006-02-15 09:45:25 52 | 52 Bamenda 19 2006-02-15 09:45:25 53 | 53 Bandar Seri Begawan 16 2006-02-15 09:45:25 54 | 54 Banjul 37 2006-02-15 09:45:25 55 | 55 Barcelona 104 2006-02-15 09:45:25 56 | 56 Basel 91 2006-02-15 09:45:25 57 | 57 Bat Yam 48 2006-02-15 09:45:25 58 | 58 Batman 97 2006-02-15 09:45:25 59 | 59 Batna 2 2006-02-15 09:45:25 60 | 60 Battambang 18 2006-02-15 09:45:25 61 | 61 Baybay 75 2006-02-15 09:45:25 62 | 62 Bayugan 75 2006-02-15 09:45:25 63 | 63 Bchar 2 2006-02-15 09:45:25 64 | 64 Beira 63 2006-02-15 09:45:25 65 | 65 Bellevue 103 2006-02-15 09:45:25 66 | 66 Belm 15 2006-02-15 09:45:25 67 | 67 Benguela 4 2006-02-15 09:45:25 68 | 68 Beni-Mellal 62 2006-02-15 09:45:25 69 | 69 Benin City 69 2006-02-15 09:45:25 70 | 70 Bergamo 49 2006-02-15 09:45:25 71 | 71 Berhampore (Baharampur) 44 2006-02-15 09:45:25 72 | 72 Bern 91 2006-02-15 09:45:25 73 | 73 Bhavnagar 44 2006-02-15 09:45:25 74 | 74 Bhilwara 44 2006-02-15 09:45:25 75 | 75 Bhimavaram 44 2006-02-15 09:45:25 76 | 76 Bhopal 44 2006-02-15 09:45:25 77 | 77 Bhusawal 44 2006-02-15 09:45:25 78 | 78 Bijapur 44 2006-02-15 09:45:25 79 | 79 Bilbays 29 2006-02-15 09:45:25 80 | 80 Binzhou 23 2006-02-15 09:45:25 81 | 81 Birgunj 66 2006-02-15 09:45:25 82 | 82 Bislig 75 2006-02-15 09:45:25 83 | 83 Blumenau 15 2006-02-15 09:45:25 84 | 84 Boa Vista 15 2006-02-15 09:45:25 85 | 85 Boksburg 85 2006-02-15 09:45:25 86 | 86 Botosani 78 2006-02-15 09:45:25 87 | 87 Botshabelo 85 2006-02-15 09:45:25 88 | 88 Bradford 102 2006-02-15 09:45:25 89 | 89 Braslia 15 2006-02-15 09:45:25 90 | 90 Bratislava 84 2006-02-15 09:45:25 91 | 91 Brescia 49 2006-02-15 09:45:25 92 | 92 Brest 34 2006-02-15 09:45:25 93 | 93 Brindisi 49 2006-02-15 09:45:25 94 | 94 Brockton 103 2006-02-15 09:45:25 95 | 95 Bucuresti 78 2006-02-15 09:45:25 96 | 96 Buenaventura 24 2006-02-15 09:45:25 97 | 97 Bydgoszcz 76 2006-02-15 09:45:25 98 | 98 Cabuyao 75 2006-02-15 09:45:25 99 | 99 Callao 74 2006-02-15 09:45:25 100 | 100 Cam Ranh 105 2006-02-15 09:45:25 101 | 101 Cape Coral 103 2006-02-15 09:45:25 102 | 102 Caracas 104 2006-02-15 09:45:25 103 | 103 Carmen 60 2006-02-15 09:45:25 104 | 104 Cavite 75 2006-02-15 09:45:25 105 | 105 Cayenne 35 2006-02-15 09:45:25 106 | 106 Celaya 60 2006-02-15 09:45:25 107 | 107 Chandrapur 44 2006-02-15 09:45:25 108 | 108 Changhwa 92 2006-02-15 09:45:25 109 | 109 Changzhou 23 2006-02-15 09:45:25 110 | 110 Chapra 44 2006-02-15 09:45:25 111 | 111 Charlotte Amalie 106 2006-02-15 09:45:25 112 | 112 Chatsworth 85 2006-02-15 09:45:25 113 | 113 Cheju 86 2006-02-15 09:45:25 114 | 114 Chiayi 92 2006-02-15 09:45:25 115 | 115 Chisinau 61 2006-02-15 09:45:25 116 | 116 Chungho 92 2006-02-15 09:45:25 117 | 117 Cianjur 45 2006-02-15 09:45:25 118 | 118 Ciomas 45 2006-02-15 09:45:25 119 | 119 Ciparay 45 2006-02-15 09:45:25 120 | 120 Citrus Heights 103 2006-02-15 09:45:25 121 | 121 Citt del Vaticano 41 2006-02-15 09:45:25 122 | 122 Ciudad del Este 73 2006-02-15 09:45:25 123 | 123 Clarksville 103 2006-02-15 09:45:25 124 | 124 Coacalco de Berriozbal 60 2006-02-15 09:45:25 125 | 125 Coatzacoalcos 60 2006-02-15 09:45:25 126 | 126 Compton 103 2006-02-15 09:45:25 127 | 127 Coquimbo 22 2006-02-15 09:45:25 128 | 128 Crdoba 6 2006-02-15 09:45:25 129 | 129 Cuauhtmoc 60 2006-02-15 09:45:25 130 | 130 Cuautla 60 2006-02-15 09:45:25 131 | 131 Cuernavaca 60 2006-02-15 09:45:25 132 | 132 Cuman 104 2006-02-15 09:45:25 133 | 133 Czestochowa 76 2006-02-15 09:45:25 134 | 134 Dadu 72 2006-02-15 09:45:25 135 | 135 Dallas 103 2006-02-15 09:45:25 136 | 136 Datong 23 2006-02-15 09:45:25 137 | 137 Daugavpils 54 2006-02-15 09:45:25 138 | 138 Davao 75 2006-02-15 09:45:25 139 | 139 Daxian 23 2006-02-15 09:45:25 140 | 140 Dayton 103 2006-02-15 09:45:25 141 | 141 Deba Habe 69 2006-02-15 09:45:25 142 | 142 Denizli 97 2006-02-15 09:45:25 143 | 143 Dhaka 12 2006-02-15 09:45:25 144 | 144 Dhule (Dhulia) 44 2006-02-15 09:45:25 145 | 145 Dongying 23 2006-02-15 09:45:25 146 | 146 Donostia-San Sebastin 87 2006-02-15 09:45:25 147 | 147 Dos Quebradas 24 2006-02-15 09:45:25 148 | 148 Duisburg 38 2006-02-15 09:45:25 149 | 149 Dundee 102 2006-02-15 09:45:25 150 | 150 Dzerzinsk 80 2006-02-15 09:45:25 151 | 151 Ede 67 2006-02-15 09:45:25 152 | 152 Effon-Alaiye 69 2006-02-15 09:45:25 153 | 153 El Alto 14 2006-02-15 09:45:25 154 | 154 El Fuerte 60 2006-02-15 09:45:25 155 | 155 El Monte 103 2006-02-15 09:45:25 156 | 156 Elista 80 2006-02-15 09:45:25 157 | 157 Emeishan 23 2006-02-15 09:45:25 158 | 158 Emmen 67 2006-02-15 09:45:25 159 | 159 Enshi 23 2006-02-15 09:45:25 160 | 160 Erlangen 38 2006-02-15 09:45:25 161 | 161 Escobar 6 2006-02-15 09:45:25 162 | 162 Esfahan 46 2006-02-15 09:45:25 163 | 163 Eskisehir 97 2006-02-15 09:45:25 164 | 164 Etawah 44 2006-02-15 09:45:25 165 | 165 Ezeiza 6 2006-02-15 09:45:25 166 | 166 Ezhou 23 2006-02-15 09:45:25 167 | 167 Faaa 36 2006-02-15 09:45:25 168 | 168 Fengshan 92 2006-02-15 09:45:25 169 | 169 Firozabad 44 2006-02-15 09:45:25 170 | 170 Florencia 24 2006-02-15 09:45:25 171 | 171 Fontana 103 2006-02-15 09:45:25 172 | 172 Fukuyama 50 2006-02-15 09:45:25 173 | 173 Funafuti 99 2006-02-15 09:45:25 174 | 174 Fuyu 23 2006-02-15 09:45:25 175 | 175 Fuzhou 23 2006-02-15 09:45:25 176 | 176 Gandhinagar 44 2006-02-15 09:45:25 177 | 177 Garden Grove 103 2006-02-15 09:45:25 178 | 178 Garland 103 2006-02-15 09:45:25 179 | 179 Gatineau 20 2006-02-15 09:45:25 180 | 180 Gaziantep 97 2006-02-15 09:45:25 181 | 181 Gijn 87 2006-02-15 09:45:25 182 | 182 Gingoog 75 2006-02-15 09:45:25 183 | 183 Goinia 15 2006-02-15 09:45:25 184 | 184 Gorontalo 45 2006-02-15 09:45:25 185 | 185 Grand Prairie 103 2006-02-15 09:45:25 186 | 186 Graz 9 2006-02-15 09:45:25 187 | 187 Greensboro 103 2006-02-15 09:45:25 188 | 188 Guadalajara 60 2006-02-15 09:45:25 189 | 189 Guaruj 15 2006-02-15 09:45:25 190 | 190 guas Lindas de Gois 15 2006-02-15 09:45:25 191 | 191 Gulbarga 44 2006-02-15 09:45:25 192 | 192 Hagonoy 75 2006-02-15 09:45:25 193 | 193 Haining 23 2006-02-15 09:45:25 194 | 194 Haiphong 105 2006-02-15 09:45:25 195 | 195 Haldia 44 2006-02-15 09:45:25 196 | 196 Halifax 20 2006-02-15 09:45:25 197 | 197 Halisahar 44 2006-02-15 09:45:25 198 | 198 Halle/Saale 38 2006-02-15 09:45:25 199 | 199 Hami 23 2006-02-15 09:45:25 200 | 200 Hamilton 68 2006-02-15 09:45:25 201 | 201 Hanoi 105 2006-02-15 09:45:25 202 | 202 Hidalgo 60 2006-02-15 09:45:25 203 | 203 Higashiosaka 50 2006-02-15 09:45:25 204 | 204 Hino 50 2006-02-15 09:45:25 205 | 205 Hiroshima 50 2006-02-15 09:45:25 206 | 206 Hodeida 107 2006-02-15 09:45:25 207 | 207 Hohhot 23 2006-02-15 09:45:25 208 | 208 Hoshiarpur 44 2006-02-15 09:45:25 209 | 209 Hsichuh 92 2006-02-15 09:45:25 210 | 210 Huaian 23 2006-02-15 09:45:25 211 | 211 Hubli-Dharwad 44 2006-02-15 09:45:25 212 | 212 Huejutla de Reyes 60 2006-02-15 09:45:25 213 | 213 Huixquilucan 60 2006-02-15 09:45:25 214 | 214 Hunuco 74 2006-02-15 09:45:25 215 | 215 Ibirit 15 2006-02-15 09:45:25 216 | 216 Idfu 29 2006-02-15 09:45:25 217 | 217 Ife 69 2006-02-15 09:45:25 218 | 218 Ikerre 69 2006-02-15 09:45:25 219 | 219 Iligan 75 2006-02-15 09:45:25 220 | 220 Ilorin 69 2006-02-15 09:45:25 221 | 221 Imus 75 2006-02-15 09:45:25 222 | 222 Inegl 97 2006-02-15 09:45:25 223 | 223 Ipoh 59 2006-02-15 09:45:25 224 | 224 Isesaki 50 2006-02-15 09:45:25 225 | 225 Ivanovo 80 2006-02-15 09:45:25 226 | 226 Iwaki 50 2006-02-15 09:45:25 227 | 227 Iwakuni 50 2006-02-15 09:45:25 228 | 228 Iwatsuki 50 2006-02-15 09:45:25 229 | 229 Izumisano 50 2006-02-15 09:45:25 230 | 230 Jaffna 88 2006-02-15 09:45:25 231 | 231 Jaipur 44 2006-02-15 09:45:25 232 | 232 Jakarta 45 2006-02-15 09:45:25 233 | 233 Jalib al-Shuyukh 53 2006-02-15 09:45:25 234 | 234 Jamalpur 12 2006-02-15 09:45:25 235 | 235 Jaroslavl 80 2006-02-15 09:45:25 236 | 236 Jastrzebie-Zdrj 76 2006-02-15 09:45:25 237 | 237 Jedda 82 2006-02-15 09:45:25 238 | 238 Jelets 80 2006-02-15 09:45:25 239 | 239 Jhansi 44 2006-02-15 09:45:25 240 | 240 Jinchang 23 2006-02-15 09:45:25 241 | 241 Jining 23 2006-02-15 09:45:25 242 | 242 Jinzhou 23 2006-02-15 09:45:25 243 | 243 Jodhpur 44 2006-02-15 09:45:25 244 | 244 Johannesburg 85 2006-02-15 09:45:25 245 | 245 Joliet 103 2006-02-15 09:45:25 246 | 246 Jos Azueta 60 2006-02-15 09:45:25 247 | 247 Juazeiro do Norte 15 2006-02-15 09:45:25 248 | 248 Juiz de Fora 15 2006-02-15 09:45:25 249 | 249 Junan 23 2006-02-15 09:45:25 250 | 250 Jurez 60 2006-02-15 09:45:25 251 | 251 Kabul 1 2006-02-15 09:45:25 252 | 252 Kaduna 69 2006-02-15 09:45:25 253 | 253 Kakamigahara 50 2006-02-15 09:45:25 254 | 254 Kaliningrad 80 2006-02-15 09:45:25 255 | 255 Kalisz 76 2006-02-15 09:45:25 256 | 256 Kamakura 50 2006-02-15 09:45:25 257 | 257 Kamarhati 44 2006-02-15 09:45:25 258 | 258 Kamjanets-Podilskyi 100 2006-02-15 09:45:25 259 | 259 Kamyin 80 2006-02-15 09:45:25 260 | 260 Kanazawa 50 2006-02-15 09:45:25 261 | 261 Kanchrapara 44 2006-02-15 09:45:25 262 | 262 Kansas City 103 2006-02-15 09:45:25 263 | 263 Karnal 44 2006-02-15 09:45:25 264 | 264 Katihar 44 2006-02-15 09:45:25 265 | 265 Kermanshah 46 2006-02-15 09:45:25 266 | 266 Kilis 97 2006-02-15 09:45:25 267 | 267 Kimberley 85 2006-02-15 09:45:25 268 | 268 Kimchon 86 2006-02-15 09:45:25 269 | 269 Kingstown 81 2006-02-15 09:45:25 270 | 270 Kirovo-Tepetsk 80 2006-02-15 09:45:25 271 | 271 Kisumu 52 2006-02-15 09:45:25 272 | 272 Kitwe 109 2006-02-15 09:45:25 273 | 273 Klerksdorp 85 2006-02-15 09:45:25 274 | 274 Kolpino 80 2006-02-15 09:45:25 275 | 275 Konotop 100 2006-02-15 09:45:25 276 | 276 Koriyama 50 2006-02-15 09:45:25 277 | 277 Korla 23 2006-02-15 09:45:25 278 | 278 Korolev 80 2006-02-15 09:45:25 279 | 279 Kowloon and New Kowloon 42 2006-02-15 09:45:25 280 | 280 Kragujevac 108 2006-02-15 09:45:25 281 | 281 Ktahya 97 2006-02-15 09:45:25 282 | 282 Kuching 59 2006-02-15 09:45:25 283 | 283 Kumbakonam 44 2006-02-15 09:45:25 284 | 284 Kurashiki 50 2006-02-15 09:45:25 285 | 285 Kurgan 80 2006-02-15 09:45:25 286 | 286 Kursk 80 2006-02-15 09:45:25 287 | 287 Kuwana 50 2006-02-15 09:45:25 288 | 288 La Paz 60 2006-02-15 09:45:25 289 | 289 La Plata 6 2006-02-15 09:45:25 290 | 290 La Romana 27 2006-02-15 09:45:25 291 | 291 Laiwu 23 2006-02-15 09:45:25 292 | 292 Lancaster 103 2006-02-15 09:45:25 293 | 293 Laohekou 23 2006-02-15 09:45:25 294 | 294 Lapu-Lapu 75 2006-02-15 09:45:25 295 | 295 Laredo 103 2006-02-15 09:45:25 296 | 296 Lausanne 91 2006-02-15 09:45:25 297 | 297 Le Mans 34 2006-02-15 09:45:25 298 | 298 Lengshuijiang 23 2006-02-15 09:45:25 299 | 299 Leshan 23 2006-02-15 09:45:25 300 | 300 Lethbridge 20 2006-02-15 09:45:25 301 | 301 Lhokseumawe 45 2006-02-15 09:45:25 302 | 302 Liaocheng 23 2006-02-15 09:45:25 303 | 303 Liepaja 54 2006-02-15 09:45:25 304 | 304 Lilongwe 58 2006-02-15 09:45:25 305 | 305 Lima 74 2006-02-15 09:45:25 306 | 306 Lincoln 103 2006-02-15 09:45:25 307 | 307 Linz 9 2006-02-15 09:45:25 308 | 308 Lipetsk 80 2006-02-15 09:45:25 309 | 309 Livorno 49 2006-02-15 09:45:25 310 | 310 Ljubertsy 80 2006-02-15 09:45:25 311 | 311 Loja 28 2006-02-15 09:45:25 312 | 312 London 102 2006-02-15 09:45:25 313 | 313 London 20 2006-02-15 09:45:25 314 | 314 Lublin 76 2006-02-15 09:45:25 315 | 315 Lubumbashi 25 2006-02-15 09:45:25 316 | 316 Lungtan 92 2006-02-15 09:45:25 317 | 317 Luzinia 15 2006-02-15 09:45:25 318 | 318 Madiun 45 2006-02-15 09:45:25 319 | 319 Mahajanga 57 2006-02-15 09:45:25 320 | 320 Maikop 80 2006-02-15 09:45:25 321 | 321 Malm 90 2006-02-15 09:45:25 322 | 322 Manchester 103 2006-02-15 09:45:25 323 | 323 Mandaluyong 75 2006-02-15 09:45:25 324 | 324 Mandi Bahauddin 72 2006-02-15 09:45:25 325 | 325 Mannheim 38 2006-02-15 09:45:25 326 | 326 Maracabo 104 2006-02-15 09:45:25 327 | 327 Mardan 72 2006-02-15 09:45:25 328 | 328 Maring 15 2006-02-15 09:45:25 329 | 329 Masqat 71 2006-02-15 09:45:25 330 | 330 Matamoros 60 2006-02-15 09:45:25 331 | 331 Matsue 50 2006-02-15 09:45:25 332 | 332 Meixian 23 2006-02-15 09:45:25 333 | 333 Memphis 103 2006-02-15 09:45:25 334 | 334 Merlo 6 2006-02-15 09:45:25 335 | 335 Mexicali 60 2006-02-15 09:45:25 336 | 336 Miraj 44 2006-02-15 09:45:25 337 | 337 Mit Ghamr 29 2006-02-15 09:45:25 338 | 338 Miyakonojo 50 2006-02-15 09:45:25 339 | 339 Mogiljov 13 2006-02-15 09:45:25 340 | 340 Molodetno 13 2006-02-15 09:45:25 341 | 341 Monclova 60 2006-02-15 09:45:25 342 | 342 Monywa 64 2006-02-15 09:45:25 343 | 343 Moscow 80 2006-02-15 09:45:25 344 | 344 Mosul 47 2006-02-15 09:45:25 345 | 345 Mukateve 100 2006-02-15 09:45:25 346 | 346 Munger (Monghyr) 44 2006-02-15 09:45:25 347 | 347 Mwanza 93 2006-02-15 09:45:25 348 | 348 Mwene-Ditu 25 2006-02-15 09:45:25 349 | 349 Myingyan 64 2006-02-15 09:45:25 350 | 350 Mysore 44 2006-02-15 09:45:25 351 | 351 Naala-Porto 63 2006-02-15 09:45:25 352 | 352 Nabereznyje Telny 80 2006-02-15 09:45:25 353 | 353 Nador 62 2006-02-15 09:45:25 354 | 354 Nagaon 44 2006-02-15 09:45:25 355 | 355 Nagareyama 50 2006-02-15 09:45:25 356 | 356 Najafabad 46 2006-02-15 09:45:25 357 | 357 Naju 86 2006-02-15 09:45:25 358 | 358 Nakhon Sawan 94 2006-02-15 09:45:25 359 | 359 Nam Dinh 105 2006-02-15 09:45:25 360 | 360 Namibe 4 2006-02-15 09:45:25 361 | 361 Nantou 92 2006-02-15 09:45:25 362 | 362 Nanyang 23 2006-02-15 09:45:25 363 | 363 NDjamna 21 2006-02-15 09:45:25 364 | 364 Newcastle 85 2006-02-15 09:45:25 365 | 365 Nezahualcyotl 60 2006-02-15 09:45:25 366 | 366 Nha Trang 105 2006-02-15 09:45:25 367 | 367 Niznekamsk 80 2006-02-15 09:45:25 368 | 368 Novi Sad 108 2006-02-15 09:45:25 369 | 369 Novoterkassk 80 2006-02-15 09:45:25 370 | 370 Nukualofa 95 2006-02-15 09:45:25 371 | 371 Nuuk 40 2006-02-15 09:45:25 372 | 372 Nyeri 52 2006-02-15 09:45:25 373 | 373 Ocumare del Tuy 104 2006-02-15 09:45:25 374 | 374 Ogbomosho 69 2006-02-15 09:45:25 375 | 375 Okara 72 2006-02-15 09:45:25 376 | 376 Okayama 50 2006-02-15 09:45:25 377 | 377 Okinawa 50 2006-02-15 09:45:25 378 | 378 Olomouc 26 2006-02-15 09:45:25 379 | 379 Omdurman 89 2006-02-15 09:45:25 380 | 380 Omiya 50 2006-02-15 09:45:25 381 | 381 Ondo 69 2006-02-15 09:45:25 382 | 382 Onomichi 50 2006-02-15 09:45:25 383 | 383 Oshawa 20 2006-02-15 09:45:25 384 | 384 Osmaniye 97 2006-02-15 09:45:25 385 | 385 ostka 100 2006-02-15 09:45:25 386 | 386 Otsu 50 2006-02-15 09:45:25 387 | 387 Oulu 33 2006-02-15 09:45:25 388 | 388 Ourense (Orense) 87 2006-02-15 09:45:25 389 | 389 Owo 69 2006-02-15 09:45:25 390 | 390 Oyo 69 2006-02-15 09:45:25 391 | 391 Ozamis 75 2006-02-15 09:45:25 392 | 392 Paarl 85 2006-02-15 09:45:25 393 | 393 Pachuca de Soto 60 2006-02-15 09:45:25 394 | 394 Pak Kret 94 2006-02-15 09:45:25 395 | 395 Palghat (Palakkad) 44 2006-02-15 09:45:25 396 | 396 Pangkal Pinang 45 2006-02-15 09:45:25 397 | 397 Papeete 36 2006-02-15 09:45:25 398 | 398 Parbhani 44 2006-02-15 09:45:25 399 | 399 Pathankot 44 2006-02-15 09:45:25 400 | 400 Patiala 44 2006-02-15 09:45:25 401 | 401 Patras 39 2006-02-15 09:45:25 402 | 402 Pavlodar 51 2006-02-15 09:45:25 403 | 403 Pemalang 45 2006-02-15 09:45:25 404 | 404 Peoria 103 2006-02-15 09:45:25 405 | 405 Pereira 24 2006-02-15 09:45:25 406 | 406 Phnom Penh 18 2006-02-15 09:45:25 407 | 407 Pingxiang 23 2006-02-15 09:45:25 408 | 408 Pjatigorsk 80 2006-02-15 09:45:25 409 | 409 Plock 76 2006-02-15 09:45:25 410 | 410 Po 15 2006-02-15 09:45:25 411 | 411 Ponce 77 2006-02-15 09:45:25 412 | 412 Pontianak 45 2006-02-15 09:45:25 413 | 413 Poos de Caldas 15 2006-02-15 09:45:25 414 | 414 Portoviejo 28 2006-02-15 09:45:25 415 | 415 Probolinggo 45 2006-02-15 09:45:25 416 | 416 Pudukkottai 44 2006-02-15 09:45:25 417 | 417 Pune 44 2006-02-15 09:45:25 418 | 418 Purnea (Purnia) 44 2006-02-15 09:45:25 419 | 419 Purwakarta 45 2006-02-15 09:45:25 420 | 420 Pyongyang 70 2006-02-15 09:45:25 421 | 421 Qalyub 29 2006-02-15 09:45:25 422 | 422 Qinhuangdao 23 2006-02-15 09:45:25 423 | 423 Qomsheh 46 2006-02-15 09:45:25 424 | 424 Quilmes 6 2006-02-15 09:45:25 425 | 425 Rae Bareli 44 2006-02-15 09:45:25 426 | 426 Rajkot 44 2006-02-15 09:45:25 427 | 427 Rampur 44 2006-02-15 09:45:25 428 | 428 Rancagua 22 2006-02-15 09:45:25 429 | 429 Ranchi 44 2006-02-15 09:45:25 430 | 430 Richmond Hill 20 2006-02-15 09:45:25 431 | 431 Rio Claro 15 2006-02-15 09:45:25 432 | 432 Rizhao 23 2006-02-15 09:45:25 433 | 433 Roanoke 103 2006-02-15 09:45:25 434 | 434 Robamba 28 2006-02-15 09:45:25 435 | 435 Rockford 103 2006-02-15 09:45:25 436 | 436 Ruse 17 2006-02-15 09:45:25 437 | 437 Rustenburg 85 2006-02-15 09:45:25 438 | 438 s-Hertogenbosch 67 2006-02-15 09:45:25 439 | 439 Saarbrcken 38 2006-02-15 09:45:25 440 | 440 Sagamihara 50 2006-02-15 09:45:25 441 | 441 Saint Louis 103 2006-02-15 09:45:25 442 | 442 Saint-Denis 79 2006-02-15 09:45:25 443 | 443 Sal 62 2006-02-15 09:45:25 444 | 444 Salala 71 2006-02-15 09:45:25 445 | 445 Salamanca 60 2006-02-15 09:45:25 446 | 446 Salinas 103 2006-02-15 09:45:25 447 | 447 Salzburg 9 2006-02-15 09:45:25 448 | 448 Sambhal 44 2006-02-15 09:45:25 449 | 449 San Bernardino 103 2006-02-15 09:45:25 450 | 450 San Felipe de Puerto Plata 27 2006-02-15 09:45:25 451 | 451 San Felipe del Progreso 60 2006-02-15 09:45:25 452 | 452 San Juan Bautista Tuxtepec 60 2006-02-15 09:45:25 453 | 453 San Lorenzo 73 2006-02-15 09:45:25 454 | 454 San Miguel de Tucumn 6 2006-02-15 09:45:25 455 | 455 Sanaa 107 2006-02-15 09:45:25 456 | 456 Santa Brbara dOeste 15 2006-02-15 09:45:25 457 | 457 Santa F 6 2006-02-15 09:45:25 458 | 458 Santa Rosa 75 2006-02-15 09:45:25 459 | 459 Santiago de Compostela 87 2006-02-15 09:45:25 460 | 460 Santiago de los Caballeros 27 2006-02-15 09:45:25 461 | 461 Santo Andr 15 2006-02-15 09:45:25 462 | 462 Sanya 23 2006-02-15 09:45:25 463 | 463 Sasebo 50 2006-02-15 09:45:25 464 | 464 Satna 44 2006-02-15 09:45:25 465 | 465 Sawhaj 29 2006-02-15 09:45:25 466 | 466 Serpuhov 80 2006-02-15 09:45:25 467 | 467 Shahr-e Kord 46 2006-02-15 09:45:25 468 | 468 Shanwei 23 2006-02-15 09:45:25 469 | 469 Shaoguan 23 2006-02-15 09:45:25 470 | 470 Sharja 101 2006-02-15 09:45:25 471 | 471 Shenzhen 23 2006-02-15 09:45:25 472 | 472 Shikarpur 72 2006-02-15 09:45:25 473 | 473 Shimoga 44 2006-02-15 09:45:25 474 | 474 Shimonoseki 50 2006-02-15 09:45:25 475 | 475 Shivapuri 44 2006-02-15 09:45:25 476 | 476 Shubra al-Khayma 29 2006-02-15 09:45:25 477 | 477 Siegen 38 2006-02-15 09:45:25 478 | 478 Siliguri (Shiliguri) 44 2006-02-15 09:45:25 479 | 479 Simferopol 100 2006-02-15 09:45:25 480 | 480 Sincelejo 24 2006-02-15 09:45:25 481 | 481 Sirjan 46 2006-02-15 09:45:25 482 | 482 Sivas 97 2006-02-15 09:45:25 483 | 483 Skikda 2 2006-02-15 09:45:25 484 | 484 Smolensk 80 2006-02-15 09:45:25 485 | 485 So Bernardo do Campo 15 2006-02-15 09:45:25 486 | 486 So Leopoldo 15 2006-02-15 09:45:25 487 | 487 Sogamoso 24 2006-02-15 09:45:25 488 | 488 Sokoto 69 2006-02-15 09:45:25 489 | 489 Songkhla 94 2006-02-15 09:45:25 490 | 490 Sorocaba 15 2006-02-15 09:45:25 491 | 491 Soshanguve 85 2006-02-15 09:45:25 492 | 492 Sousse 96 2006-02-15 09:45:25 493 | 493 South Hill 5 2006-02-15 09:45:25 494 | 494 Southampton 102 2006-02-15 09:45:25 495 | 495 Southend-on-Sea 102 2006-02-15 09:45:25 496 | 496 Southport 102 2006-02-15 09:45:25 497 | 497 Springs 85 2006-02-15 09:45:25 498 | 498 Stara Zagora 17 2006-02-15 09:45:25 499 | 499 Sterling Heights 103 2006-02-15 09:45:25 500 | 500 Stockport 102 2006-02-15 09:45:25 501 | 501 Sucre 14 2006-02-15 09:45:25 502 | 502 Suihua 23 2006-02-15 09:45:25 503 | 503 Sullana 74 2006-02-15 09:45:25 504 | 504 Sultanbeyli 97 2006-02-15 09:45:25 505 | 505 Sumqayit 10 2006-02-15 09:45:25 506 | 506 Sumy 100 2006-02-15 09:45:25 507 | 507 Sungai Petani 59 2006-02-15 09:45:25 508 | 508 Sunnyvale 103 2006-02-15 09:45:25 509 | 509 Surakarta 45 2006-02-15 09:45:25 510 | 510 Syktyvkar 80 2006-02-15 09:45:25 511 | 511 Syrakusa 49 2006-02-15 09:45:25 512 | 512 Szkesfehrvr 43 2006-02-15 09:45:25 513 | 513 Tabora 93 2006-02-15 09:45:25 514 | 514 Tabriz 46 2006-02-15 09:45:25 515 | 515 Tabuk 82 2006-02-15 09:45:25 516 | 516 Tafuna 3 2006-02-15 09:45:25 517 | 517 Taguig 75 2006-02-15 09:45:25 518 | 518 Taizz 107 2006-02-15 09:45:25 519 | 519 Talavera 75 2006-02-15 09:45:25 520 | 520 Tallahassee 103 2006-02-15 09:45:25 521 | 521 Tama 50 2006-02-15 09:45:25 522 | 522 Tambaram 44 2006-02-15 09:45:25 523 | 523 Tanauan 75 2006-02-15 09:45:25 524 | 524 Tandil 6 2006-02-15 09:45:25 525 | 525 Tangail 12 2006-02-15 09:45:25 526 | 526 Tanshui 92 2006-02-15 09:45:25 527 | 527 Tanza 75 2006-02-15 09:45:25 528 | 528 Tarlac 75 2006-02-15 09:45:25 529 | 529 Tarsus 97 2006-02-15 09:45:25 530 | 530 Tartu 30 2006-02-15 09:45:25 531 | 531 Teboksary 80 2006-02-15 09:45:25 532 | 532 Tegal 45 2006-02-15 09:45:25 533 | 533 Tel Aviv-Jaffa 48 2006-02-15 09:45:25 534 | 534 Tete 63 2006-02-15 09:45:25 535 | 535 Tianjin 23 2006-02-15 09:45:25 536 | 536 Tiefa 23 2006-02-15 09:45:25 537 | 537 Tieli 23 2006-02-15 09:45:25 538 | 538 Tokat 97 2006-02-15 09:45:25 539 | 539 Tonghae 86 2006-02-15 09:45:25 540 | 540 Tongliao 23 2006-02-15 09:45:25 541 | 541 Torren 60 2006-02-15 09:45:25 542 | 542 Touliu 92 2006-02-15 09:45:25 543 | 543 Toulon 34 2006-02-15 09:45:25 544 | 544 Toulouse 34 2006-02-15 09:45:25 545 | 545 Trshavn 32 2006-02-15 09:45:25 546 | 546 Tsaotun 92 2006-02-15 09:45:25 547 | 547 Tsuyama 50 2006-02-15 09:45:25 548 | 548 Tuguegarao 75 2006-02-15 09:45:25 549 | 549 Tychy 76 2006-02-15 09:45:25 550 | 550 Udaipur 44 2006-02-15 09:45:25 551 | 551 Udine 49 2006-02-15 09:45:25 552 | 552 Ueda 50 2006-02-15 09:45:25 553 | 553 Uijongbu 86 2006-02-15 09:45:25 554 | 554 Uluberia 44 2006-02-15 09:45:25 555 | 555 Urawa 50 2006-02-15 09:45:25 556 | 556 Uruapan 60 2006-02-15 09:45:25 557 | 557 Usak 97 2006-02-15 09:45:25 558 | 558 Usolje-Sibirskoje 80 2006-02-15 09:45:25 559 | 559 Uttarpara-Kotrung 44 2006-02-15 09:45:25 560 | 560 Vaduz 55 2006-02-15 09:45:25 561 | 561 Valencia 104 2006-02-15 09:45:25 562 | 562 Valle de la Pascua 104 2006-02-15 09:45:25 563 | 563 Valle de Santiago 60 2006-02-15 09:45:25 564 | 564 Valparai 44 2006-02-15 09:45:25 565 | 565 Vancouver 20 2006-02-15 09:45:25 566 | 566 Varanasi (Benares) 44 2006-02-15 09:45:25 567 | 567 Vicente Lpez 6 2006-02-15 09:45:25 568 | 568 Vijayawada 44 2006-02-15 09:45:25 569 | 569 Vila Velha 15 2006-02-15 09:45:25 570 | 570 Vilnius 56 2006-02-15 09:45:25 571 | 571 Vinh 105 2006-02-15 09:45:25 572 | 572 Vitria de Santo Anto 15 2006-02-15 09:45:25 573 | 573 Warren 103 2006-02-15 09:45:25 574 | 574 Weifang 23 2006-02-15 09:45:25 575 | 575 Witten 38 2006-02-15 09:45:25 576 | 576 Woodridge 8 2006-02-15 09:45:25 577 | 577 Wroclaw 76 2006-02-15 09:45:25 578 | 578 Xiangfan 23 2006-02-15 09:45:25 579 | 579 Xiangtan 23 2006-02-15 09:45:25 580 | 580 Xintai 23 2006-02-15 09:45:25 581 | 581 Xinxiang 23 2006-02-15 09:45:25 582 | 582 Yamuna Nagar 44 2006-02-15 09:45:25 583 | 583 Yangor 65 2006-02-15 09:45:25 584 | 584 Yantai 23 2006-02-15 09:45:25 585 | 585 Yaound 19 2006-02-15 09:45:25 586 | 586 Yerevan 7 2006-02-15 09:45:25 587 | 587 Yinchuan 23 2006-02-15 09:45:25 588 | 588 Yingkou 23 2006-02-15 09:45:25 589 | 589 York 102 2006-02-15 09:45:25 590 | 590 Yuncheng 23 2006-02-15 09:45:25 591 | 591 Yuzhou 23 2006-02-15 09:45:25 592 | 592 Zalantun 23 2006-02-15 09:45:25 593 | 593 Zanzibar 93 2006-02-15 09:45:25 594 | 594 Zaoyang 23 2006-02-15 09:45:25 595 | 595 Zapopan 60 2006-02-15 09:45:25 596 | 596 Zaria 69 2006-02-15 09:45:25 597 | 597 Zeleznogorsk 80 2006-02-15 09:45:25 598 | 598 Zhezqazghan 51 2006-02-15 09:45:25 599 | 599 Zhoushan 23 2006-02-15 09:45:25 600 | 600 Ziguinchor 83 2006-02-15 09:45:25 601 | \. 602 | 603 | 604 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3069.dat: -------------------------------------------------------------------------------- 1 | 1 Afghanistan 2006-02-15 09:44:00 2 | 2 Algeria 2006-02-15 09:44:00 3 | 3 American Samoa 2006-02-15 09:44:00 4 | 4 Angola 2006-02-15 09:44:00 5 | 5 Anguilla 2006-02-15 09:44:00 6 | 6 Argentina 2006-02-15 09:44:00 7 | 7 Armenia 2006-02-15 09:44:00 8 | 8 Australia 2006-02-15 09:44:00 9 | 9 Austria 2006-02-15 09:44:00 10 | 10 Azerbaijan 2006-02-15 09:44:00 11 | 11 Bahrain 2006-02-15 09:44:00 12 | 12 Bangladesh 2006-02-15 09:44:00 13 | 13 Belarus 2006-02-15 09:44:00 14 | 14 Bolivia 2006-02-15 09:44:00 15 | 15 Brazil 2006-02-15 09:44:00 16 | 16 Brunei 2006-02-15 09:44:00 17 | 17 Bulgaria 2006-02-15 09:44:00 18 | 18 Cambodia 2006-02-15 09:44:00 19 | 19 Cameroon 2006-02-15 09:44:00 20 | 20 Canada 2006-02-15 09:44:00 21 | 21 Chad 2006-02-15 09:44:00 22 | 22 Chile 2006-02-15 09:44:00 23 | 23 China 2006-02-15 09:44:00 24 | 24 Colombia 2006-02-15 09:44:00 25 | 25 Congo, The Democratic Republic of the 2006-02-15 09:44:00 26 | 26 Czech Republic 2006-02-15 09:44:00 27 | 27 Dominican Republic 2006-02-15 09:44:00 28 | 28 Ecuador 2006-02-15 09:44:00 29 | 29 Egypt 2006-02-15 09:44:00 30 | 30 Estonia 2006-02-15 09:44:00 31 | 31 Ethiopia 2006-02-15 09:44:00 32 | 32 Faroe Islands 2006-02-15 09:44:00 33 | 33 Finland 2006-02-15 09:44:00 34 | 34 France 2006-02-15 09:44:00 35 | 35 French Guiana 2006-02-15 09:44:00 36 | 36 French Polynesia 2006-02-15 09:44:00 37 | 37 Gambia 2006-02-15 09:44:00 38 | 38 Germany 2006-02-15 09:44:00 39 | 39 Greece 2006-02-15 09:44:00 40 | 40 Greenland 2006-02-15 09:44:00 41 | 41 Holy See (Vatican City State) 2006-02-15 09:44:00 42 | 42 Hong Kong 2006-02-15 09:44:00 43 | 43 Hungary 2006-02-15 09:44:00 44 | 44 India 2006-02-15 09:44:00 45 | 45 Indonesia 2006-02-15 09:44:00 46 | 46 Iran 2006-02-15 09:44:00 47 | 47 Iraq 2006-02-15 09:44:00 48 | 48 Israel 2006-02-15 09:44:00 49 | 49 Italy 2006-02-15 09:44:00 50 | 50 Japan 2006-02-15 09:44:00 51 | 51 Kazakstan 2006-02-15 09:44:00 52 | 52 Kenya 2006-02-15 09:44:00 53 | 53 Kuwait 2006-02-15 09:44:00 54 | 54 Latvia 2006-02-15 09:44:00 55 | 55 Liechtenstein 2006-02-15 09:44:00 56 | 56 Lithuania 2006-02-15 09:44:00 57 | 57 Madagascar 2006-02-15 09:44:00 58 | 58 Malawi 2006-02-15 09:44:00 59 | 59 Malaysia 2006-02-15 09:44:00 60 | 60 Mexico 2006-02-15 09:44:00 61 | 61 Moldova 2006-02-15 09:44:00 62 | 62 Morocco 2006-02-15 09:44:00 63 | 63 Mozambique 2006-02-15 09:44:00 64 | 64 Myanmar 2006-02-15 09:44:00 65 | 65 Nauru 2006-02-15 09:44:00 66 | 66 Nepal 2006-02-15 09:44:00 67 | 67 Netherlands 2006-02-15 09:44:00 68 | 68 New Zealand 2006-02-15 09:44:00 69 | 69 Nigeria 2006-02-15 09:44:00 70 | 70 North Korea 2006-02-15 09:44:00 71 | 71 Oman 2006-02-15 09:44:00 72 | 72 Pakistan 2006-02-15 09:44:00 73 | 73 Paraguay 2006-02-15 09:44:00 74 | 74 Peru 2006-02-15 09:44:00 75 | 75 Philippines 2006-02-15 09:44:00 76 | 76 Poland 2006-02-15 09:44:00 77 | 77 Puerto Rico 2006-02-15 09:44:00 78 | 78 Romania 2006-02-15 09:44:00 79 | 79 Runion 2006-02-15 09:44:00 80 | 80 Russian Federation 2006-02-15 09:44:00 81 | 81 Saint Vincent and the Grenadines 2006-02-15 09:44:00 82 | 82 Saudi Arabia 2006-02-15 09:44:00 83 | 83 Senegal 2006-02-15 09:44:00 84 | 84 Slovakia 2006-02-15 09:44:00 85 | 85 South Africa 2006-02-15 09:44:00 86 | 86 South Korea 2006-02-15 09:44:00 87 | 87 Spain 2006-02-15 09:44:00 88 | 88 Sri Lanka 2006-02-15 09:44:00 89 | 89 Sudan 2006-02-15 09:44:00 90 | 90 Sweden 2006-02-15 09:44:00 91 | 91 Switzerland 2006-02-15 09:44:00 92 | 92 Taiwan 2006-02-15 09:44:00 93 | 93 Tanzania 2006-02-15 09:44:00 94 | 94 Thailand 2006-02-15 09:44:00 95 | 95 Tonga 2006-02-15 09:44:00 96 | 96 Tunisia 2006-02-15 09:44:00 97 | 97 Turkey 2006-02-15 09:44:00 98 | 98 Turkmenistan 2006-02-15 09:44:00 99 | 99 Tuvalu 2006-02-15 09:44:00 100 | 100 Ukraine 2006-02-15 09:44:00 101 | 101 United Arab Emirates 2006-02-15 09:44:00 102 | 102 United Kingdom 2006-02-15 09:44:00 103 | 103 United States 2006-02-15 09:44:00 104 | 104 Venezuela 2006-02-15 09:44:00 105 | 105 Vietnam 2006-02-15 09:44:00 106 | 106 Virgin Islands, U.S. 2006-02-15 09:44:00 107 | 107 Yemen 2006-02-15 09:44:00 108 | 108 Yugoslavia 2006-02-15 09:44:00 109 | 109 Zambia 2006-02-15 09:44:00 110 | \. 111 | 112 | 113 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3073.dat: -------------------------------------------------------------------------------- 1 | 1 English 2006-02-15 10:02:19 2 | 2 Italian 2006-02-15 10:02:19 3 | 3 Japanese 2006-02-15 10:02:19 4 | 4 Mandarin 2006-02-15 10:02:19 5 | 5 French 2006-02-15 10:02:19 6 | 6 German 2006-02-15 10:02:19 7 | \. 8 | 9 | 10 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3079.dat: -------------------------------------------------------------------------------- 1 | 1 Mike Hillyer 3 Mike.Hillyer@sakilastaff.com 1 t Mike 8cb2237d0679ca88db6464eac60da96345513964 2006-05-16 16:13:11.79328 \\x89504e470d0a5a0a 2 | 2 Jon Stephens 4 Jon.Stephens@sakilastaff.com 2 t Jon 8cb2237d0679ca88db6464eac60da96345513964 2006-05-16 16:13:11.79328 \N 3 | \. 4 | 5 | 6 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/3081.dat: -------------------------------------------------------------------------------- 1 | 1 1 1 2006-02-15 09:57:12 2 | 2 2 2 2006-02-15 09:57:12 3 | \. 4 | 5 | 6 | -------------------------------------------------------------------------------- /dataForProject/image/dvdrental/toc.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/dvdrental/toc.dat -------------------------------------------------------------------------------- /dataForProject/image/dvdrentalDW-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/dvdrentalDW-schema.png -------------------------------------------------------------------------------- /dataForProject/image/postgreConf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/postgreConf.png -------------------------------------------------------------------------------- /dataForProject/image/projectBigdata.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/projectBigdata.jpg -------------------------------------------------------------------------------- /dataForProject/image/reportDVD.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/reportDVD.png -------------------------------------------------------------------------------- /dataForProject/image/superset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/dataForProject/image/superset.png -------------------------------------------------------------------------------- /dataForProject/script-init/createDW-when-db-have-init.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE if exists d_date; 2 | 3 | CREATE TABLE d_date 4 | ( 5 | date_dim_id INT NOT NULL, 6 | date_actual DATE NOT NULL, 7 | epoch BIGINT NOT NULL, 8 | day_suffix VARCHAR(4) NOT NULL, 9 | day_name VARCHAR(9) NOT NULL, 10 | day_of_week INT NOT NULL, 11 | day_of_month INT NOT NULL, 12 | day_of_quarter INT NOT NULL, 13 | day_of_year INT NOT NULL, 14 | week_of_month INT NOT NULL, 15 | week_of_year INT NOT NULL, 16 | week_of_year_iso CHAR(10) NOT NULL, 17 | month_actual INT NOT NULL, 18 | month_name VARCHAR(9) NOT NULL, 19 | month_name_abbreviated CHAR(3) NOT NULL, 20 | quarter_actual INT NOT NULL, 21 | quarter_name VARCHAR(9) NOT NULL, 22 | year_actual INT NOT NULL, 23 | first_day_of_week DATE NOT NULL, 24 | last_day_of_week DATE NOT NULL, 25 | first_day_of_month DATE NOT NULL, 26 | last_day_of_month DATE NOT NULL, 27 | first_day_of_quarter DATE NOT NULL, 28 | last_day_of_quarter DATE NOT NULL, 29 | first_day_of_year DATE NOT NULL, 30 | last_day_of_year DATE NOT NULL, 31 | mmyyyy CHAR(6) NOT NULL, 32 | mmddyyyy CHAR(10) NOT NULL, 33 | weekend_indr BOOLEAN NOT NULL 34 | ); 35 | 36 | ALTER TABLE public.d_date ADD CONSTRAINT d_date_date_dim_id_pk PRIMARY KEY (date_dim_id); 37 | 38 | CREATE INDEX d_date_date_actual_idx 39 | ON d_date(date_actual); 40 | 41 | COMMIT; 42 | 43 | INSERT INTO d_date 44 | SELECT TO_CHAR(datum, 'yyyymmdd')::INT AS date_dim_id, 45 | datum AS date_actual, 46 | EXTRACT(EPOCH FROM datum) AS epoch, 47 | TO_CHAR(datum, 'fmDDth') AS day_suffix, 48 | TO_CHAR(datum, 'TMDay') AS day_name, 49 | EXTRACT(ISODOW FROM datum) AS day_of_week, 50 | EXTRACT(DAY FROM datum) AS day_of_month, 51 | datum - DATE_TRUNC('quarter', datum)::DATE + 1 AS day_of_quarter, 52 | EXTRACT(DOY FROM datum) AS day_of_year, 53 | TO_CHAR(datum, 'W')::INT AS week_of_month, 54 | EXTRACT(WEEK FROM datum) AS week_of_year, 55 | EXTRACT(ISOYEAR FROM datum) || TO_CHAR(datum, '"-W"IW-') || EXTRACT(ISODOW FROM datum) AS week_of_year_iso, 56 | EXTRACT(MONTH FROM datum) AS month_actual, 57 | TO_CHAR(datum, 'TMMonth') AS month_name, 58 | TO_CHAR(datum, 'Mon') AS month_name_abbreviated, 59 | EXTRACT(QUARTER FROM datum) AS quarter_actual, 60 | CASE 61 | WHEN EXTRACT(QUARTER FROM datum) = 1 THEN 'First' 62 | WHEN EXTRACT(QUARTER FROM datum) = 2 THEN 'Second' 63 | WHEN EXTRACT(QUARTER FROM datum) = 3 THEN 'Third' 64 | WHEN EXTRACT(QUARTER FROM datum) = 4 THEN 'Fourth' 65 | END AS quarter_name, 66 | EXTRACT(YEAR FROM datum) AS year_actual, 67 | datum + (1 - EXTRACT(ISODOW FROM datum))::INT AS first_day_of_week, 68 | datum + (7 - EXTRACT(ISODOW FROM datum))::INT AS last_day_of_week, 69 | datum + (1 - EXTRACT(DAY FROM datum))::INT AS first_day_of_month, 70 | (DATE_TRUNC('MONTH', datum) + INTERVAL '1 MONTH - 1 day')::DATE AS last_day_of_month, 71 | DATE_TRUNC('quarter', datum)::DATE AS first_day_of_quarter, 72 | (DATE_TRUNC('quarter', datum) + INTERVAL '3 MONTH - 1 day')::DATE AS last_day_of_quarter, 73 | TO_DATE(EXTRACT(YEAR FROM datum) || '-01-01', 'YYYY-MM-DD') AS first_day_of_year, 74 | TO_DATE(EXTRACT(YEAR FROM datum) || '-12-31', 'YYYY-MM-DD') AS last_day_of_year, 75 | TO_CHAR(datum, 'mmyyyy') AS mmyyyy, 76 | TO_CHAR(datum, 'mmddyyyy') AS mmddyyyy, 77 | CASE 78 | WHEN EXTRACT(ISODOW FROM datum) IN (6, 7) THEN TRUE 79 | ELSE FALSE 80 | END AS weekend_indr 81 | FROM (SELECT '1970-01-01'::DATE + SEQUENCE.DAY AS datum 82 | FROM GENERATE_SERIES(0, 29219) AS SEQUENCE (DAY) 83 | GROUP BY SEQUENCE.DAY) DQ 84 | ORDER BY 1; 85 | 86 | COMMIT; 87 | 88 | 89 | 90 | 91 | -- Create the staff_dimension table 92 | CREATE TABLE IF NOT EXISTS staff_dimension ( 93 | staff_id smallint PRIMARY KEY, 94 | first_name varchar(45), 95 | last_name varchar(45), 96 | email varchar(50), 97 | username varchar(16) 98 | ); 99 | 100 | -- Insert data into the staff_dimension table 101 | INSERT INTO staff_dimension (staff_id, first_name, last_name, email, username) 102 | SELECT DISTINCT 103 | s.staff_id, 104 | s.first_name, 105 | s.last_name, 106 | s.email, 107 | s.username 108 | FROM 109 | staff s; 110 | 111 | -- Create the customer_dimension table if it doesn't exist 112 | CREATE TABLE IF NOT EXISTS customer_dimension ( 113 | customer_id smallint PRIMARY KEY, 114 | first_name varchar(45), 115 | last_name varchar(45), 116 | email varchar(50), 117 | address varchar(50), 118 | city varchar(50), 119 | country varchar(50), 120 | create_date date, 121 | last_update timestamp 122 | ); 123 | CREATE TABLE IF NOT EXISTS film_dimension ( 124 | film_id smallint PRIMARY KEY, 125 | title varchar(255), 126 | description text, 127 | release_year smallint, 128 | language varchar(20), 129 | rental_duration smallint, 130 | rental_rate numeric(4,2), 131 | length smallint, 132 | replacement_cost numeric(5,2), 133 | rating varchar(10), 134 | special_features varchar(255), 135 | last_update timestamp 136 | ); 137 | CREATE TABLE IF NOT EXISTS store_dimension ( 138 | store_id INT PRIMARY KEY, 139 | manager_staff_id INT, 140 | address VARCHAR(50), 141 | city VARCHAR(50), 142 | country VARCHAR(50) 143 | ); 144 | 145 | INSERT INTO store_dimension (store_id, manager_staff_id, address, city, country) 146 | SELECT distinct 147 | s.store_id, 148 | s.manager_staff_id, 149 | a.address, 150 | city.city, 151 | country.country 152 | FROM 153 | public.store AS s 154 | JOIN public.address AS a ON s.address_id = a.address_id 155 | JOIN public.city AS city ON a.city_id = city.city_id 156 | JOIN public.country AS country ON city.country_id = country.country_id; 157 | 158 | INSERT INTO film_dimension (film_id, title, description, release_year, language, rental_duration, rental_rate, length, replacement_cost, rating, special_features, last_update) 159 | SELECT distinct 160 | f.film_id, 161 | f.title, 162 | f.description, 163 | f.release_year, 164 | l.name AS language, 165 | f.rental_duration, 166 | f.rental_rate, 167 | f.length, 168 | f.replacement_cost, 169 | f.rating, 170 | f.special_features, 171 | f.last_update 172 | FROM 173 | film f 174 | JOIN 175 | language l ON f.language_id = l.language_id; 176 | 177 | -- Insert data into the customer_dimension table 178 | INSERT INTO customer_dimension (customer_id, first_name, last_name, email, address, city, country, create_date, last_update) 179 | SELECT DISTINCT 180 | c.customer_id, 181 | c.first_name, 182 | c.last_name, 183 | c.email, 184 | a.address, 185 | ci.city, 186 | co.country, 187 | c.create_date::date, -- Ensure create_date is of type date 188 | c.last_update 189 | FROM 190 | customer c 191 | JOIN 192 | address a ON c.address_id = a.address_id 193 | JOIN 194 | city ci ON a.city_id = ci.city_id 195 | JOIN 196 | country co ON ci.country_id = co.country_id; 197 | 198 | 199 | 200 | CREATE TABLE IF NOT EXISTS sales_fact ( 201 | sales_id serial PRIMARY KEY, 202 | date_dim_id int NOT NULL, 203 | customer_id smallint NOT NULL, 204 | store_id smallint NOT NULL, 205 | staff_id smallint NOT NULL, 206 | film_id smallint NOT NULL, 207 | quantity smallint NOT NULL, 208 | amount numeric(5, 2), 209 | CONSTRAINT sales_fact_date_dim_id_fkey FOREIGN KEY (date_dim_id) REFERENCES d_date (date_dim_id), 210 | CONSTRAINT sales_fact_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer_dimension (customer_id), 211 | CONSTRAINT sales_fact_store_id_fkey FOREIGN KEY (store_id) REFERENCES store_dimension (store_id), 212 | CONSTRAINT sales_fact_film_id_fkey FOREIGN KEY (film_id) REFERENCES film_dimension (film_id), 213 | CONSTRAINT sales_fact_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff_dimension (staff_id) 214 | ); 215 | 216 | 217 | INSERT INTO public.sales_fact( 218 | date_dim_id, customer_id, store_id, staff_id, film_id, quantity, amount) 219 | SELECT 220 | TO_CHAR(r.rental_date, 'YYYYMMDD')::int as date_dim_id, 221 | p.customer_id, 222 | st.store_id, 223 | s.staff_id, 224 | i.film_id, 225 | COUNT(r.rental_id) AS quantity, 226 | SUM(f.rental_rate) AS amount 227 | FROM 228 | payment p 229 | JOIN rental r ON p.rental_id = r.rental_id 230 | JOIN staff s ON r.staff_id = s.staff_id 231 | JOIN store st ON s.store_id = st.store_id 232 | JOIN inventory i ON r.inventory_id = i.inventory_id 233 | JOIN film_dimension f ON i.film_id = f.film_id 234 | GROUP BY 235 | date_dim_id, 236 | p.customer_id, 237 | st.store_id, 238 | s.staff_id, 239 | i.film_id; 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | -- If it show this line or same, congrats! let's move on 248 | -- NOTICE: table "d_date" does not exist, skipping 249 | -- WARNING: there is no transaction in progress 250 | -- WARNING: there is no transaction in progress 251 | -- INSERT 0 14594 252 | 253 | -- Query returned successfully in 4 secs 126 msec. -------------------------------------------------------------------------------- /dataForProject/script-init/hive-dw-init.txt: -------------------------------------------------------------------------------- 1 | CREATE EXTERNAL TABLE IF NOT EXISTS staff_dimension ( 2 | staff_id SMALLINT, 3 | first_name STRING, 4 | last_name STRING, 5 | email STRING, 6 | username STRING 7 | ) 8 | ROW FORMAT DELIMITED 9 | FIELDS TERMINATED BY ',' 10 | STORED AS TEXTFILE 11 | LOCATION '/data/staging/staff_dimension.csv'; 12 | 13 | 14 | 15 | CREATE EXTERNAL TABLE IF NOT EXISTS film_dimension ( 16 | film_id SMALLINT, 17 | title STRING, 18 | description STRING, 19 | release_year SMALLINT, 20 | language STRING, 21 | rental_duration SMALLINT, 22 | rental_rate DECIMAL(4, 2), 23 | length SMALLINT, 24 | replacement_cost DECIMAL(5, 2), 25 | rating STRING, 26 | special_features STRING, 27 | last_update TIMESTAMP 28 | ) 29 | ROW FORMAT DELIMITED 30 | FIELDS TERMINATED BY ',' 31 | STORED AS TEXTFILE 32 | LOCATION '/data/staging/film_dimension.csv'; 33 | 34 | 35 | 36 | CREATE EXTERNAL TABLE IF NOT EXISTS store_dimension ( 37 | store_id INT, 38 | manager_staff_id INT, 39 | address STRING, 40 | city STRING, 41 | country STRING 42 | ) 43 | ROW FORMAT DELIMITED 44 | FIELDS TERMINATED BY ',' 45 | STORED AS TEXTFILE 46 | LOCATION '/data/staging/store_dimension.csv'; 47 | 48 | CREATE EXTERNAL TABLE IF NOT EXISTS customer_dimension ( 49 | customer_id SMALLINT, 50 | first_name STRING, 51 | last_name STRING, 52 | email STRING, 53 | address STRING, 54 | city STRING, 55 | country STRING, 56 | create_date DATE, 57 | last_update TIMESTAMP 58 | ) 59 | ROW FORMAT DELIMITED 60 | FIELDS TERMINATED BY ',' 61 | STORED AS TEXTFILE 62 | LOCATION '/data/staging/customer_dimension.csv'; 63 | 64 | -- Create an external table in Hive based on the PostgreSQL table structure 65 | CREATE EXTERNAL TABLE IF NOT EXISTS d_date ( 66 | date_dim_id INT, 67 | date_actual DATE, 68 | epoch BIGINT, 69 | day_suffix STRING, 70 | day_name STRING, 71 | day_of_week INT, 72 | day_of_month INT, 73 | day_of_quarter INT, 74 | day_of_year INT, 75 | week_of_month INT, 76 | week_of_year INT, 77 | week_of_year_iso STRING, 78 | month_actual INT, 79 | month_name STRING, 80 | month_name_abbreviated STRING, 81 | quarter_actual INT, 82 | quarter_name STRING, 83 | year_actual INT, 84 | first_day_of_week DATE, 85 | last_day_of_week DATE, 86 | first_day_of_month DATE, 87 | last_day_of_month DATE, 88 | first_day_of_quarter DATE, 89 | last_day_of_quarter DATE, 90 | first_day_of_year DATE, 91 | last_day_of_year DATE, 92 | mmyyyy STRING, 93 | mmddyyyy STRING, 94 | weekend_indr BOOLEAN 95 | ) 96 | ROW FORMAT DELIMITED 97 | FIELDS TERMINATED BY ',' 98 | STORED AS TEXTFILE 99 | LOCATION '/data/staging/d_date.csv'; 100 | 101 | 102 | CREATE EXTERNAL TABLE IF NOT EXISTS sales_fact ( 103 | sales_id INT, 104 | date_dim_id INT, 105 | customer_id SMALLINT, 106 | store_id SMALLINT, 107 | staff_id SMALLINT, 108 | film_id SMALLINT, 109 | quantity SMALLINT, 110 | amount DECIMAL(5, 2) 111 | ) 112 | ROW FORMAT DELIMITED 113 | FIELDS TERMINATED BY ',' 114 | STORED AS TEXTFILE 115 | LOCATION '/data/staging/sales_fact.csv'; 116 | -------------------------------------------------------------------------------- /dataForProject/script-init/install-jdbc.txt: -------------------------------------------------------------------------------- 1 | docker cp postgresql-42.6.0.jar spark-master:spark/jars 2 | docker cp postgresql-42.6.0.jar spark-worker-1:spark/jars -------------------------------------------------------------------------------- /dataForProject/script-init/read-postgres-to-hdfs.txt: -------------------------------------------------------------------------------- 1 | val jdbcURL = "jdbc:postgresql://host.docker.internal:5435/dvdrental" 2 | val connectionProperties = new java.util.Properties() 3 | connectionProperties.setProperty("user", "airflow") 4 | connectionProperties.setProperty("password", "airflow") 5 | val table_name = "sales_fact" 6 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 7 | df.write.format("csv").mode("overwrite").save("hdfs://namenode:9000/data/staging/sales_fact.csv") 8 | val table_name = "staff_dimension" 9 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 10 | df.write.format("csv").save("hdfs://namenode:9000/data/staging/staff_dimension.csv") 11 | val table_name = "store_dimension" 12 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 13 | df.write.format("csv").save("hdfs://namenode:9000/data/staging/store_dimension.csv") 14 | val table_name = "customer_dimension" 15 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 16 | df.write.format("csv").save("hdfs://namenode:9000/data/staging/customer_dimension.csv") 17 | val table_name = "film_dimension" 18 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 19 | df.write.format("csv").save("hdfs://namenode:9000/data/staging/film_dimension.csv") 20 | val table_name = "d_date" 21 | val df = spark.read.jdbc(jdbcURL, table_name, connectionProperties) 22 | df.write.format("csv").save("hdfs://namenode:9000/data/staging/d_date.csv") 23 | 24 | 25 | -------------------------------------------------------------------------------- /datanode/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 2 | 3 | MAINTAINER Ivan Ermilov 4 | 5 | HEALTHCHECK CMD curl -f http://localhost:9864/ || exit 1 6 | 7 | ENV HDFS_CONF_dfs_datanode_data_dir=file:///hadoop/dfs/data 8 | RUN mkdir -p /hadoop/dfs/data 9 | VOLUME /hadoop/dfs/data 10 | 11 | ADD run.sh /run.sh 12 | RUN chmod a+x /run.sh 13 | 14 | EXPOSE 9864 15 | 16 | CMD ["/run.sh"] 17 | -------------------------------------------------------------------------------- /datanode/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'` 4 | if [ ! -d $datadir ]; then 5 | echo "Datanode data directory not found: $datadir" 6 | exit 2 7 | fi 8 | 9 | $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode 10 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | 2 | # AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow. 3 | # Default: apache/airflow:2.7.1 4 | # AIRFLOW_UID - User ID in Airflow containers 5 | # Default: 50000 6 | # AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. 7 | # Default: . 8 | # Those configurations are useful mostly in case of standalone testing/running Airflow in test/try-out mode 9 | # 10 | # _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account (if requested). 11 | # Default: airflow 12 | # _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account (if requested). 13 | # Default: airflow 14 | # _PIP_ADDITIONAL_REQUIREMENTS - Additional PIP requirements to add when starting all containers. 15 | # Use this option ONLY for quick checks. Installing requirements at container 16 | # startup is done EVERY TIME the service is started. 17 | # A better way is to build a custom image or extend the official image 18 | # as described in https://airflow.apache.org/docs/docker-stack/build.html. 19 | # Default: '' 20 | 21 | version: '3' 22 | # x-airflow-common: 23 | # &airflow-common 24 | # # In order to add custom dependencies or upgrade provider packages you can use your extended image. 25 | # # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml 26 | # # and uncomment the "build" line below, Then run `docker-compose build` to build the images. 27 | # image: apache/airflow:2.7.1 28 | # # build: . 29 | # environment: 30 | # &airflow-common-env 31 | # AIRFLOW__CORE__EXECUTOR: CeleryExecutor 32 | # AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow 33 | # # For backward compatibility, with Airflow <2.3 34 | # AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow 35 | # AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow 36 | # AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 37 | # AIRFLOW__CORE__FERNET_KEY: '' 38 | # AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' 39 | # AIRFLOW__CORE__LOAD_EXAMPLES: 'false' 40 | # AIRFLOW__CORE__TEST_CONNECTION: 'enabled' 41 | # AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session' 42 | # # yamllint disable rule:line-length 43 | # # Use simple http server on scheduler for health checks 44 | # # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server 45 | # # yamllint enable rule:line-length 46 | # AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' 47 | # # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks 48 | # # for other purpose (development, test and especially production usage) build/extend Airflow image. 49 | # _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} 50 | # volumes: 51 | # - ./airflow/dags:/opt/airflow/dags 52 | # - ./airflow/logs:/opt/airflow/logs 53 | # - ./airflow/config:/opt/airflow/config 54 | # - ./airflow/plugins:/opt/airflow/plugins 55 | # user: "${AIRFLOW_UID:-50000}:0" 56 | # depends_on: 57 | # &airflow-common-depends-on 58 | # # redis: 59 | # # condition: service_healthy 60 | # postgres: 61 | # condition: service_healthy 62 | 63 | services: 64 | postgres: 65 | image: postgres:13 66 | container_name: postgres13 67 | environment: 68 | POSTGRES_USER: airflow 69 | POSTGRES_PASSWORD: airflow 70 | POSTGRES_DB: airflow 71 | volumes: 72 | - ./airflow/postgres-db-volume:/var/lib/postgresql/data 73 | ports: 74 | - "5435:5432" 75 | healthcheck: 76 | test: ["CMD", "pg_isready", "-U", "airflow"] 77 | interval: 10s 78 | retries: 5 79 | start_period: 5s 80 | restart: always 81 | 82 | #if you want to play with airflow, uncomment from redis image till the flower image 83 | 84 | # redis: 85 | # image: redis:latest 86 | # container_name: redis 87 | # expose: 88 | # - 6379 89 | # healthcheck: 90 | # test: ["CMD", "redis-cli", "ping"] 91 | # interval: 10s 92 | # timeout: 30s 93 | # retries: 50 94 | # start_period: 30s 95 | # restart: always 96 | 97 | # airflow-webserver: 98 | # container_name: airflow-webserver 99 | # <<: *airflow-common 100 | # command: webserver 101 | # ports: 102 | # - "8082:8080" 103 | # healthcheck: 104 | # test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] 105 | # interval: 30s 106 | # timeout: 10s 107 | # retries: 5 108 | # start_period: 30s 109 | # restart: always 110 | # depends_on: 111 | # <<: *airflow-common-depends-on 112 | # airflow-init: 113 | # condition: service_completed_successfully 114 | 115 | # airflow-scheduler: 116 | # container_name: airflow-scheduler 117 | # <<: *airflow-common 118 | # command: scheduler 119 | # healthcheck: 120 | # test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] 121 | # interval: 30s 122 | # timeout: 10s 123 | # retries: 5 124 | # start_period: 30s 125 | # restart: always 126 | # depends_on: 127 | # <<: *airflow-common-depends-on 128 | # airflow-init: 129 | # condition: service_completed_successfully 130 | 131 | # airflow-worker: 132 | # container_name: airflow-worker 133 | # <<: *airflow-common 134 | # command: celery worker 135 | # healthcheck: 136 | # # yamllint disable rule:line-length 137 | # test: 138 | # - "CMD-SHELL" 139 | # - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' 140 | # interval: 30s 141 | # timeout: 10s 142 | # retries: 5 143 | # start_period: 30s 144 | # environment: 145 | # <<: *airflow-common-env 146 | # # Required to handle warm shutdown of the celery workers properly 147 | # # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation 148 | # DUMB_INIT_SETSID: "0" 149 | # restart: always 150 | # depends_on: 151 | # <<: *airflow-common-depends-on 152 | # airflow-init: 153 | # condition: service_completed_successfully 154 | 155 | # airflow-triggerer: 156 | # container_name: airflow-triggerer 157 | # <<: *airflow-common 158 | # command: triggerer 159 | # healthcheck: 160 | # test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] 161 | # interval: 30s 162 | # timeout: 10s 163 | # retries: 5 164 | # start_period: 30s 165 | # restart: always 166 | # depends_on: 167 | # <<: *airflow-common-depends-on 168 | # airflow-init: 169 | # condition: service_completed_successfully 170 | 171 | # airflow-init: 172 | # <<: *airflow-common 173 | # entrypoint: /bin/bash 174 | # # yamllint disable rule:line-length 175 | # command: 176 | # - -c 177 | # - | 178 | # function ver() { 179 | # printf "%04d%04d%04d%04d" $${1//./ } 180 | # } 181 | # airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version) 182 | # airflow_version_comparable=$$(ver $${airflow_version}) 183 | # min_airflow_version=2.2.0 184 | # min_airflow_version_comparable=$$(ver $${min_airflow_version}) 185 | # if (( airflow_version_comparable < min_airflow_version_comparable )); then 186 | # echo 187 | # echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m" 188 | # echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!" 189 | # echo 190 | # exit 1 191 | # fi 192 | # if [[ -z "${AIRFLOW_UID}" ]]; then 193 | # echo 194 | # echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" 195 | # echo "If you are on Linux, you SHOULD follow the instructions below to set " 196 | # echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." 197 | # echo "For other operating systems you can get rid of the warning with manually created .env file:" 198 | # echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" 199 | # echo 200 | # fi 201 | # one_meg=1048576 202 | # mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg)) 203 | # cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat) 204 | # disk_available=$$(df / | tail -1 | awk '{print $$4}') 205 | # warning_resources="false" 206 | # if (( mem_available < 4000 )) ; then 207 | # echo 208 | # echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m" 209 | # echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))" 210 | # echo 211 | # warning_resources="true" 212 | # fi 213 | # if (( cpus_available < 2 )); then 214 | # echo 215 | # echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m" 216 | # echo "At least 2 CPUs recommended. You have $${cpus_available}" 217 | # echo 218 | # warning_resources="true" 219 | # fi 220 | # if (( disk_available < one_meg * 10 )); then 221 | # echo 222 | # echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m" 223 | # echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))" 224 | # echo 225 | # warning_resources="true" 226 | # fi 227 | # if [[ $${warning_resources} == "true" ]]; then 228 | # echo 229 | # echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" 230 | # echo "Please follow the instructions to increase amount of resources available:" 231 | # echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" 232 | # echo 233 | # fi 234 | # mkdir -p /sources/logs /sources/dags /sources/plugins 235 | # chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} 236 | # exec /entrypoint airflow version 237 | # # yamllint enable rule:line-length 238 | # environment: 239 | # <<: *airflow-common-env 240 | # _AIRFLOW_DB_MIGRATE: 'true' 241 | # _AIRFLOW_WWW_USER_CREATE: 'true' 242 | # _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} 243 | # _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} 244 | # _PIP_ADDITIONAL_REQUIREMENTS: '' 245 | # user: "0:0" 246 | # volumes: 247 | # - /airflow:/sources 248 | 249 | # airflow-cli: 250 | # <<: *airflow-common 251 | # profiles: 252 | # - debug 253 | # environment: 254 | # <<: *airflow-common-env 255 | # CONNECTION_CHECK_MAX_COUNT: "0" 256 | # # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252 257 | # command: 258 | # - bash 259 | # - -c 260 | # - airflow 261 | 262 | # # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up 263 | # # or by explicitly targeted on the command line e.g. docker-compose up flower. 264 | # # See: https://docs.docker.com/compose/profiles/ 265 | # flower: 266 | # <<: *airflow-common 267 | # command: celery flower 268 | # profiles: 269 | # - flower 270 | # ports: 271 | # - "5555:5555" 272 | # healthcheck: 273 | # test: ["CMD", "curl", "--fail", "http://localhost:5555/"] 274 | # interval: 30s 275 | # timeout: 10s 276 | # retries: 5 277 | # start_period: 30s 278 | # restart: always 279 | # depends_on: 280 | # <<: *airflow-common-depends-on 281 | # airflow-init: 282 | # condition: service_completed_successfully 283 | 284 | namenode: 285 | image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 286 | container_name: namenode 287 | restart: always 288 | ports: 289 | - 9870:9870 290 | - 9010:9000 291 | volumes: 292 | - ./namenode/hadoop_namenode:/hadoop/dfs/name 293 | environment: 294 | - CLUSTER_NAME=test 295 | - CORE_CONF_fs_defaultFS=hdfs://namenode:9000 296 | env_file: 297 | - ./hadoop.env 298 | 299 | datanode: 300 | image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 301 | container_name: datanode 302 | restart: always 303 | volumes: 304 | - ./datanode/hadoop_datanode:/hadoop/dfs/data 305 | environment: 306 | SERVICE_PRECONDITION: "namenode:9870" 307 | CORE_CONF_fs_defaultFS: hdfs://namenode:9000 308 | ports: 309 | - "9864:9864" 310 | env_file: 311 | - ./hadoop.env 312 | 313 | resourcemanager: 314 | image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 315 | container_name: resourcemanager 316 | restart: always 317 | environment: 318 | SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864" 319 | env_file: 320 | - ./hadoop.env 321 | 322 | nodemanager1: 323 | image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 324 | container_name: nodemanager 325 | restart: always 326 | environment: 327 | SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088" 328 | env_file: 329 | - ./hadoop.env 330 | 331 | historyserver: 332 | image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 333 | container_name: historyserver 334 | restart: always 335 | environment: 336 | SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088" 337 | volumes: 338 | - /historyserver/hadoop_historyserver:/hadoop/yarn/timeline 339 | env_file: 340 | - ./hadoop.env 341 | 342 | spark-master: 343 | image: bde2020/spark-master:3.0.0-hadoop3.2 344 | container_name: spark-master 345 | depends_on: 346 | - namenode 347 | - datanode 348 | ports: 349 | - "8080:8080" 350 | - "7077:7077" 351 | environment: 352 | - INIT_DAEMON_STEP=setup_spark 353 | - CORE_CONF_fs_defaultFS=hdfs://namenode:9000 354 | 355 | spark-worker-1: 356 | image: bde2020/spark-worker:3.0.0-hadoop3.2 357 | container_name: spark-worker-1 358 | depends_on: 359 | - spark-master 360 | ports: 361 | - "8081:8081" 362 | environment: 363 | - "SPARK_MASTER=spark://spark-master:7077" 364 | - CORE_CONF_fs_defaultFS=hdfs://namenode:9000 365 | 366 | hive-server: 367 | image: bde2020/hive:2.3.2-postgresql-metastore 368 | container_name: hive-server 369 | depends_on: 370 | - namenode 371 | - datanode 372 | env_file: 373 | - ./hadoop-hive.env 374 | environment: 375 | HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" 376 | SERVICE_PRECONDITION: "hive-metastore:9083" 377 | ports: 378 | - "10001:10000" 379 | - "10002:10002" 380 | 381 | hive-metastore: 382 | image: bde2020/hive:2.3.2-postgresql-metastore 383 | container_name: hive-metastore 384 | env_file: 385 | - ./hadoop-hive.env 386 | command: /opt/hive/bin/hive --service metastore 387 | environment: 388 | SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432" 389 | ports: 390 | - "9083:9083" 391 | 392 | 393 | hive-metastore-postgresql: 394 | image: bde2020/hive-metastore-postgresql:2.3.0 395 | container_name: hive-metastore-postgresql 396 | volumes: 397 | - /hive:/var/lib/postgresql/data 398 | 399 | presto-coordinator: 400 | image: shawnzhu/prestodb:0.181 401 | container_name: presto-coordinator 402 | ports: 403 | - "8089:8089" 404 | 405 | superset: 406 | build: 407 | context: ./superset 408 | dockerfile: Dockerfile 409 | container_name: superset 410 | environment: 411 | - ADMIN_USERNAME=admin 412 | - ADMIN_EMAIL=admin@superset.com 413 | - ADMIN_PASSWORD=admin 414 | ports: 415 | - '8088:8088' 416 | 417 | volumes: 418 | hadoop_datanode: 419 | hadoop_namenode: 420 | hadoop_historyserver: 421 | postgres-db-volume: 422 | hive: 423 | 424 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Set some sensible defaults 4 | export CORE_CONF_fs_defaultFS=${CORE_CONF_fs_defaultFS:-hdfs://`hostname -f`:8020} 5 | 6 | function addProperty() { 7 | local path=$1 8 | local name=$2 9 | local value=$3 10 | 11 | local entry="$name${value}" 12 | local escapedEntry=$(echo $entry | sed 's/\//\\\//g') 13 | sed -i "/<\/configuration>/ s/.*/${escapedEntry}\n&/" $path 14 | } 15 | 16 | function configure() { 17 | local path=$1 18 | local module=$2 19 | local envPrefix=$3 20 | 21 | local var 22 | local value 23 | 24 | echo "Configuring $module" 25 | for c in `printenv | perl -sne 'print "$1 " if m/^${envPrefix}_(.+?)=.*/' -- -envPrefix=$envPrefix`; do 26 | name=`echo ${c} | perl -pe 's/___/-/g; s/__/_/g; s/_/./g'` 27 | var="${envPrefix}_${c}" 28 | value=${!var} 29 | echo " - Setting $name=$value" 30 | addProperty $path $name "$value" 31 | done 32 | } 33 | 34 | configure /etc/hadoop/core-site.xml core CORE_CONF 35 | configure /etc/hadoop/hdfs-site.xml hdfs HDFS_CONF 36 | configure /etc/hadoop/yarn-site.xml yarn YARN_CONF 37 | configure /etc/hadoop/httpfs-site.xml httpfs HTTPFS_CONF 38 | configure /etc/hadoop/kms-site.xml kms KMS_CONF 39 | configure /etc/hadoop/mapred-site.xml mapred MAPRED_CONF 40 | configure /opt/hive/conf/hive-site.xml hive HIVE_SITE_CONF 41 | 42 | if [ "$MULTIHOMED_NETWORK" = "1" ]; then 43 | echo "Configuring for multihomed network" 44 | 45 | # HDFS 46 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.rpc-bind-host 0.0.0.0 47 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.servicerpc-bind-host 0.0.0.0 48 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.http-bind-host 0.0.0.0 49 | addProperty /etc/hadoop/hdfs-site.xml dfs.namenode.https-bind-host 0.0.0.0 50 | addProperty /etc/hadoop/hdfs-site.xml dfs.client.use.datanode.hostname true 51 | addProperty /etc/hadoop/hdfs-site.xml dfs.datanode.use.datanode.hostname true 52 | 53 | # YARN 54 | addProperty /etc/hadoop/yarn-site.xml yarn.resourcemanager.bind-host 0.0.0.0 55 | addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 56 | addProperty /etc/hadoop/yarn-site.xml yarn.nodemanager.bind-host 0.0.0.0 57 | addProperty /etc/hadoop/yarn-site.xml yarn.timeline-service.bind-host 0.0.0.0 58 | 59 | # MAPRED 60 | addProperty /etc/hadoop/mapred-site.xml yarn.nodemanager.bind-host 0.0.0.0 61 | fi 62 | 63 | if [ -n "$GANGLIA_HOST" ]; then 64 | mv /etc/hadoop/hadoop-metrics.properties /etc/hadoop/hadoop-metrics.properties.orig 65 | mv /etc/hadoop/hadoop-metrics2.properties /etc/hadoop/hadoop-metrics2.properties.orig 66 | 67 | for module in mapred jvm rpc ugi; do 68 | echo "$module.class=org.apache.hadoop.metrics.ganglia.GangliaContext31" 69 | echo "$module.period=10" 70 | echo "$module.servers=$GANGLIA_HOST:8649" 71 | done > /etc/hadoop/hadoop-metrics.properties 72 | 73 | for module in namenode datanode resourcemanager nodemanager mrappmaster jobhistoryserver; do 74 | echo "$module.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31" 75 | echo "$module.sink.ganglia.period=10" 76 | echo "$module.sink.ganglia.supportsparse=true" 77 | echo "$module.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both" 78 | echo "$module.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40" 79 | echo "$module.sink.ganglia.servers=$GANGLIA_HOST:8649" 80 | done > /etc/hadoop/hadoop-metrics2.properties 81 | fi 82 | 83 | function wait_for_it() 84 | { 85 | local serviceport=$1 86 | local service=${serviceport%%:*} 87 | local port=${serviceport#*:} 88 | local retry_seconds=5 89 | local max_try=100 90 | let i=1 91 | 92 | nc -z $service $port 93 | result=$? 94 | 95 | until [ $result -eq 0 ]; do 96 | echo "[$i/$max_try] check for ${service}:${port}..." 97 | echo "[$i/$max_try] ${service}:${port} is not available yet" 98 | if (( $i == $max_try )); then 99 | echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/" 100 | exit 1 101 | fi 102 | 103 | echo "[$i/$max_try] try in ${retry_seconds}s once again ..." 104 | let "i++" 105 | sleep $retry_seconds 106 | 107 | nc -z $service $port 108 | result=$? 109 | done 110 | echo "[$i/$max_try] $service:${port} is available." 111 | } 112 | 113 | for i in ${SERVICE_PRECONDITION[@]} 114 | do 115 | wait_for_it ${i} 116 | done 117 | 118 | exec $@ 119 | -------------------------------------------------------------------------------- /hadoop-hive.env: -------------------------------------------------------------------------------- 1 | HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore 2 | HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver 3 | HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive 4 | HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive 5 | HIVE_SITE_CONF_datanucleus_autoCreateSchema=false 6 | HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083 7 | HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false 8 | 9 | CORE_CONF_fs_defaultFS=hdfs://namenode:9000 10 | CORE_CONF_hadoop_http_staticuser_user=root 11 | CORE_CONF_hadoop_proxyuser_hue_hosts=* 12 | CORE_CONF_hadoop_proxyuser_hue_groups=* 13 | 14 | HDFS_CONF_dfs_webhdfs_enabled=true 15 | HDFS_CONF_dfs_permissions_enabled=false 16 | 17 | YARN_CONF_yarn_log___aggregation___enable=true 18 | YARN_CONF_yarn_resourcemanager_recovery_enabled=true 19 | YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore 20 | YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate 21 | YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs 22 | YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ 23 | YARN_CONF_yarn_timeline___service_enabled=true 24 | YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true 25 | YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true 26 | YARN_CONF_yarn_resourcemanager_hostname=resourcemanager 27 | YARN_CONF_yarn_timeline___service_hostname=historyserver 28 | YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 29 | YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 30 | YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 31 | -------------------------------------------------------------------------------- /hadoop.env: -------------------------------------------------------------------------------- 1 | CORE_CONF_fs_defaultFS=hdfs://namenode:9000 2 | CORE_CONF_hadoop_http_staticuser_user=root 3 | CORE_CONF_hadoop_proxyuser_hue_hosts=* 4 | CORE_CONF_hadoop_proxyuser_hue_groups=* 5 | CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec 6 | 7 | HDFS_CONF_dfs_webhdfs_enabled=true 8 | HDFS_CONF_dfs_permissions_enabled=false 9 | HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false 10 | 11 | YARN_CONF_yarn_log___aggregation___enable=true 12 | YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ 13 | YARN_CONF_yarn_resourcemanager_recovery_enabled=true 14 | YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore 15 | YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler 16 | YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 17 | YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 18 | YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate 19 | YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true 20 | YARN_CONF_yarn_resourcemanager_hostname=resourcemanager 21 | YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 22 | YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 23 | YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 24 | YARN_CONF_yarn_timeline___service_enabled=true 25 | YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true 26 | YARN_CONF_yarn_timeline___service_hostname=historyserver 27 | YARN_CONF_mapreduce_map_output_compress=true 28 | YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec 29 | YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 30 | YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 31 | YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 32 | YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs 33 | YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle 34 | 35 | MAPRED_CONF_mapreduce_framework_name=yarn 36 | MAPRED_CONF_mapred_child_java_opts=-Xmx4096m 37 | MAPRED_CONF_mapreduce_map_memory_mb=4096 38 | MAPRED_CONF_mapreduce_reduce_memory_mb=8192 39 | MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m 40 | MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m 41 | MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 42 | MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 43 | MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ 44 | -------------------------------------------------------------------------------- /historyserver/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 2 | 3 | MAINTAINER Ivan Ermilov 4 | 5 | HEALTHCHECK CMD curl -f http://localhost:8188/ || exit 1 6 | 7 | ENV YARN_CONF_yarn_timeline___service_leveldb___timeline___store_path=/hadoop/yarn/timeline 8 | RUN mkdir -p /hadoop/yarn/timeline 9 | VOLUME /hadoop/yarn/timeline 10 | 11 | ADD run.sh /run.sh 12 | RUN chmod a+x /run.sh 13 | 14 | EXPOSE 8188 15 | 16 | CMD ["/run.sh"] 17 | -------------------------------------------------------------------------------- /historyserver/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver 4 | -------------------------------------------------------------------------------- /master/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/spark-base:3.0.0-hadoop3.2 2 | 3 | LABEL maintainer="Gezim Sejdiu , Giannis Mouchakis " 4 | 5 | COPY master.sh / 6 | 7 | ENV SPARK_MASTER_PORT 7077 8 | ENV SPARK_MASTER_WEBUI_PORT 8080 9 | ENV SPARK_MASTER_LOG /spark/logs 10 | 11 | EXPOSE 8080 7077 6066 12 | 13 | CMD ["/bin/bash", "/master.sh"] 14 | -------------------------------------------------------------------------------- /master/README.md: -------------------------------------------------------------------------------- 1 | # Spark master 2 | 3 | See [big-data-europe/docker-spark README](https://github.com/big-data-europe/docker-spark). -------------------------------------------------------------------------------- /master/master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export SPARK_MASTER_HOST=`hostname` 4 | 5 | . "/spark/sbin/spark-config.sh" 6 | 7 | . "/spark/bin/load-spark-env.sh" 8 | 9 | mkdir -p $SPARK_MASTER_LOG 10 | 11 | export SPARK_HOME=/spark 12 | 13 | ln -sf /dev/stdout $SPARK_MASTER_LOG/spark-master.out 14 | 15 | cd /spark/bin && /spark/sbin/../bin/spark-class org.apache.spark.deploy.master.Master \ 16 | --ip $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT >> $SPARK_MASTER_LOG/spark-master.out 17 | -------------------------------------------------------------------------------- /namenode/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 2 | 3 | MAINTAINER Ivan Ermilov 4 | 5 | HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1 6 | 7 | ENV HDFS_CONF_dfs_namenode_name_dir=file:///hadoop/dfs/name 8 | RUN mkdir -p /hadoop/dfs/name 9 | VOLUME /hadoop/dfs/name 10 | 11 | ADD run.sh /run.sh 12 | RUN chmod a+x /run.sh 13 | 14 | EXPOSE 9870 15 | 16 | CMD ["/run.sh"] 17 | -------------------------------------------------------------------------------- /namenode/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | namedir=`echo $HDFS_CONF_dfs_namenode_name_dir | perl -pe 's#file://##'` 4 | if [ ! -d $namedir ]; then 5 | echo "Namenode name directory not found: $namedir" 6 | exit 2 7 | fi 8 | 9 | if [ -z "$CLUSTER_NAME" ]; then 10 | echo "Cluster name not specified" 11 | exit 2 12 | fi 13 | 14 | echo "remove lost+found from $namedir" 15 | rm -r $namedir/lost+found 16 | 17 | if [ "`ls -A $namedir`" == "" ]; then 18 | echo "Formatting namenode name directory: $namedir" 19 | $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME 20 | fi 21 | 22 | $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode 23 | -------------------------------------------------------------------------------- /nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx 2 | 3 | MAINTAINER "Ivan Ermilov " 4 | 5 | COPY default.conf /etc/nginx/conf.d/default.conf 6 | COPY materialize.min.css /data/bde-css/materialize.min.css 7 | COPY bde-hadoop.css /data/bde-css/bde-hadoop.css 8 | -------------------------------------------------------------------------------- /nginx/bde-hadoop.css: -------------------------------------------------------------------------------- 1 | body { 2 | background: #F1F1F1; 3 | } 4 | 5 | body > .container { 6 | margin: 5rem auto; 7 | background: white; 8 | box-shadow: 0 2px 5px 0 rgba(0,0,0,0.16), 0 2px 10px 0 rgba(0,0,0,0.12); 9 | } 10 | 11 | header.bs-docs-nav { 12 | position: fixed; 13 | top: 0; 14 | left: 0; 15 | width: 100%; 16 | height: 3rem; 17 | border: none; 18 | background: #A94F74; 19 | box-shadow: 0 2px 5px 0 rgba(0,0,0,0.16), 0 2px 10px 0 rgba(0,0,0,0.12); 20 | } 21 | 22 | header.bs-docs-nav .navbar-brand { 23 | background: inherit; 24 | } 25 | 26 | #ui-tabs .active a { 27 | background: #B96A8B; 28 | } 29 | 30 | #ui-tabs > li > a { 31 | color: white; 32 | } 33 | 34 | .navbar-inverse .navbar-nav > .dropdown > a .caret { 35 | border-top-color: white; 36 | border-bottom-color: white; 37 | } 38 | 39 | .navbar-inverse .navbar-nav > .open > a, 40 | .navbar-inverse .navbar-nav > .open > a:hover, 41 | .navbar-inverse .navbar-nav > .open > a:focus { 42 | background-color: #B96A8B; 43 | } 44 | 45 | .dropdown-menu > li > a { 46 | color: #A94F74; 47 | } 48 | 49 | .modal-dialog .panel-success { 50 | border-color: lightgrey; 51 | } 52 | 53 | .modal-dialog .panel-heading { 54 | background-color: #A94F74 !important; 55 | } 56 | 57 | .modal-dialog .panel-heading select { 58 | margin-top: 1rem; 59 | } -------------------------------------------------------------------------------- /nginx/default.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name localhost; 4 | 5 | root /data; 6 | gzip on; 7 | 8 | location / { 9 | proxy_pass http://127.0.0.1:8000; 10 | proxy_set_header Accept-Encoding ""; 11 | } 12 | 13 | location /bde-css/ { 14 | } 15 | } 16 | 17 | server { 18 | listen 127.0.0.1:8000; 19 | location / { 20 | proxy_pass http://127.0.0.1:8001; 21 | sub_filter '' ' 22 | '; 23 | sub_filter_once on; 24 | proxy_set_header Accept-Encoding ""; 25 | } 26 | } 27 | 28 | server { 29 | listen 127.0.0.1:8001; 30 | gunzip on; 31 | location / { 32 | proxy_pass http://namenode:50070; 33 | proxy_set_header Accept-Encoding gzip; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /nodemanager/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 2 | 3 | MAINTAINER Ivan Ermilov 4 | 5 | HEALTHCHECK CMD curl -f http://localhost:8042/ || exit 1 6 | 7 | ADD run.sh /run.sh 8 | RUN chmod a+x /run.sh 9 | 10 | EXPOSE 8042 11 | 12 | CMD ["/run.sh"] 13 | -------------------------------------------------------------------------------- /nodemanager/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager 4 | -------------------------------------------------------------------------------- /postgresql-42.6.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DucAnhNTT/bigdata-ETL-pipeline/cfe09926b8e7bf0047c6f6f5bfaee6fd52f1b8e2/postgresql-42.6.0.jar -------------------------------------------------------------------------------- /resourcemanager/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 2 | 3 | MAINTAINER Ivan Ermilov 4 | 5 | HEALTHCHECK CMD curl -f http://localhost:8088/ || exit 1 6 | 7 | ADD run.sh /run.sh 8 | RUN chmod a+x /run.sh 9 | 10 | EXPOSE 8088 11 | 12 | CMD ["/run.sh"] 13 | -------------------------------------------------------------------------------- /resourcemanager/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | $HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager 4 | -------------------------------------------------------------------------------- /spark_in_action.MD: -------------------------------------------------------------------------------- 1 | # Leer Spark met opgaven uit Spark in Action, 2nd Edition 2 | 3 | Dit zijn praktijkopdrachten uit [Spark in Action, 2nd Edition van Jean-Georges Perrin](https://www.manning.com/books/spark-in-action-second-edition), die je kunt uitvoeren op de CDEP Hadoop-Spark-Hive omgeving. 4 | De data hoeft niet op HDFS gezet te worden. In deze opdrachten doen we dat wel. 5 | 6 | Als je dat eenmaal gedaan hebt, kun je de Spark commando's zo achter elkaar kopiëren, uitvoeren en kijken hoe het werkt. 7 | 8 | 9 | ## PySpark starten 10 | Start de Docker omgeving volgens [de handleiding](https://github.com/DIKW/CDEP-Docker/blob/master/hadoop/docker-hadoop-spark-hive/README.md). Volg de Quick Start Spark (PySpark) om in te loggen op de Spark master. 11 | 12 | Start PySpark: 13 | ``` 14 | /spark/bin/pyspark --master spark://69280b13519d:7077 15 | ``` 16 | 17 | ## Files klaarzetten 18 | Kopieer de directory met data op de namenode en kopieer ze vandaar naar HDFS 19 | 20 | ``` 21 | docker ps |grep namenode # achterhaal Container ID van namenode 22 | 23 | docker cp Spark_in_Action 2ffec0140800:sparkinaction 24 | 25 | docker exec -it 2ffec0140800 bash 26 | 27 | hdfs dfs -mkdir /data 28 | hdfs dfs -mkdir /data/sparkinaction 29 | 30 | hdfs dfs -put sparkinaction/* /data/sparkinaction 31 | ``` 32 | 33 | 34 | ## Hoofdstuk 1 35 | [Chapter 1 op Github](https://github.com/jgperrin/net.jgp.books.spark.ch01) 36 | 37 | ``` 38 | # Creates a session on a local master 39 | absolute_file_path = "hdfs://namenode:8020/data/sparkinaction/books.csv" 40 | 41 | session = SparkSession.builder.appName("CSV to Dataset").master("spark://69280b13519d:7077").getOrCreate() 42 | 43 | # Reads a CSV file with header, called books.csv, stores it in a dataframe 44 | df = session.read.csv(header=True, inferSchema=True, path=absolute_file_path) 45 | 46 | # Shows at most 5 rows from the dataframe 47 | df.show(5) 48 | 49 | # Good to stop SparkSession at the end of the application 50 | session.stop() 51 | ``` 52 | 53 | 54 | ## Hoofdstuk 2 55 | [Chapter 2 op Github](https://github.com/jgperrin/net.jgp.books.spark.ch02) 56 | In dit hoofdstuk doen we een kleine transformatie: we concatteneren de lastname en firstname. 57 | We slaan hier de database stap over. Feel free om zelf een PostgreSQL database toe te voegen aan de hadoop-spark-hive docker-compose. 58 | 59 | ``` 60 | from pyspark.sql import SparkSession 61 | from pyspark.sql import functions as F 62 | 63 | absolute_file_path = "hdfs://namenode:8020/data/sparkinaction/authors.csv" 64 | 65 | # Step 1: Ingestion 66 | # --------- 67 | # 68 | # Reads a CSV file with header, called authors.csv, stores it in a dataframe 69 | df = spark.read.csv(header=True, inferSchema=True, path=absolute_file_path) 70 | 71 | # Step 2: Transform 72 | # --------- 73 | # Creates a new column called "name" as the concatenation of lname, a 74 | # virtual column containing ", " and the fname column 75 | df = df.withColumn("name", F.concat(F.col("lname"), F.lit(", "), F.col("fname"))) 76 | 77 | df.printSchema() 78 | 79 | # Good to stop SparkSession at the end of the application 80 | spark.stop() 81 | ``` 82 | 83 | 84 | ## Hoofdstuk 3 85 | [Chapter 3 op Github](https://github.com/jgperrin/net.jgp.books.spark.ch03) 86 | In dit hoofdstuk hernoemen we wat kolommen uit een CSV. We halen we een paar kolommen uit het resultaat en we voegen een id kolom toe, gebaseerd op state, county en datasetId. 87 | 88 | ### Kolommen hernoemen, verwijderen en toevoegen 89 | 90 | ``` 91 | from pyspark.sql import SparkSession 92 | from pyspark.sql.functions import lit,col,concat 93 | import json 94 | 95 | absolute_file_path = "hdfs://namenode:8020/data/sparkinaction/Restaurants_in_Wake_County_NC.csv" 96 | 97 | spark = SparkSession.builder.appName("Restaurants in Wake County, NC") \ 98 | .master("spark://69280b13519d:7077").getOrCreate() 99 | 100 | df = spark.read.csv(header=True, inferSchema=True,path=absolute_file_path) 101 | 102 | print("*** Right after ingestion") 103 | df.show(5) 104 | 105 | print("*** Schema as a tree:") 106 | df.printSchema() 107 | ``` 108 | 109 | Hernoemen van kolommen en verwijderen van de OBJECTID, PERMITID en GEOCODESTATUS kolommen uit ons dataframe. 110 | ``` 111 | # Let's transform our dataframe 112 | df = df.withColumn("county", lit("Wake")) \ 113 | .withColumnRenamed("HSISID", "datasetId") \ 114 | .withColumnRenamed("NAME", "name") \ 115 | .withColumnRenamed("ADDRESS1", "address1") \ 116 | .withColumnRenamed("ADDRESS2", "address2") \ 117 | .withColumnRenamed("CITY", "city") \ 118 | .withColumnRenamed("STATE", "state") \ 119 | .withColumnRenamed("POSTALCODE", "zip") \ 120 | .withColumnRenamed("PHONENUMBER", "tel") \ 121 | .withColumnRenamed("RESTAURANTOPENDATE", "dateStart") \ 122 | .withColumnRenamed("FACILITYTYPE", "type") \ 123 | .withColumnRenamed("X", "geoX") \ 124 | .withColumnRenamed("Y", "geoY") \ 125 | .drop("OBJECTID", "PERMITID", "GEOCODESTATUS") 126 | ``` 127 | 128 | Het maken van een nieuwe kolom, geconcatteneerd van state, county en datasetId. 129 | ``` 130 | df = df.withColumn("id", 131 | concat(col("state"), lit("_"), col("county"), lit("_"), col("datasetId"))) 132 | 133 | # Shows at most 5 rows from the dataframe 134 | print("*** Dataframe transformed") 135 | df.show(5) 136 | 137 | print("*** Schema as a tree:") 138 | df.printSchema() 139 | ``` 140 | 141 | We kunnen het dataframe opslaan als json. 142 | ``` 143 | print("*** Schema as string: {}".format(df.schema)) 144 | schemaAsJson = df.schema.json() 145 | parsedSchemaAsJson = json.loads(schemaAsJson) 146 | 147 | print("*** Schema as JSON: {}".format(json.dumps(parsedSchemaAsJson, indent=2))) 148 | 149 | # Good to stop SparkSession at the end of the application 150 | spark.stop() 151 | ``` 152 | 153 | ### Union van twee dataframes. 154 | 155 | ``` 156 | from pyspark.sql.functions import (lit,col,concat,split) 157 | from pyspark.sql import SparkSession 158 | 159 | absolute_file_path1 = "hdfs://namenode:8020/data/sparkinaction/Restaurants_in_Wake_County_NC.csv" 160 | absolute_file_path2 = "hdfs://namenode:8020/data/sparkinaction/Restaurants_in_Durham_County_NC.json" 161 | 162 | spark = SparkSession.builder.appName("Union of two dataframes") \ 163 | .master("local[*]").getOrCreate() 164 | ``` 165 | 166 | Inlezen van data van Wake County (csv formaat). 167 | De kolommen worden aangepast zodat een union mogelijk is. 168 | ``` 169 | df1 = spark.read.csv(path=absolute_file_path1,header=True,inferSchema=True) 170 | 171 | df1 = df1.withColumn("county", lit("Wake")) \ 172 | .withColumnRenamed("HSISID", "datasetId") \ 173 | .withColumnRenamed("NAME", "name") \ 174 | .withColumnRenamed("ADDRESS1", "address1") \ 175 | .withColumnRenamed("ADDRESS2", "address2") \ 176 | .withColumnRenamed("CITY", "city") \ 177 | .withColumnRenamed("STATE", "state") \ 178 | .withColumnRenamed("POSTALCODE", "zip") \ 179 | .withColumnRenamed("PHONENUMBER", "tel") \ 180 | .withColumnRenamed("RESTAURANTOPENDATE", "dateStart") \ 181 | .withColumn("dateEnd", lit(None)) \ 182 | .withColumnRenamed("FACILITYTYPE", "type") \ 183 | .withColumnRenamed("X", "geoX") \ 184 | .withColumnRenamed("Y", "geoY") \ 185 | .drop("OBJECTID", "GEOCODESTATUS", "PERMITID") 186 | 187 | df1 = df1.withColumn("id", concat(col("state"), lit("_"), col("county"), lit("_"), col("datasetId"))) 188 | df1 = df1.repartition(4); 189 | ``` 190 | 191 | 192 | Inlezen data van Durham County (json formaat) 193 | Aanpassen kolomnamen en verwijderen kolommen zodat union mogelijk is. 194 | ``` 195 | df2 = spark.read.json(absolute_file_path2) 196 | 197 | drop_cols = ["fields", "geometry", "record_timestamp", "recordid"] 198 | df2 = df2.withColumn("county", lit("Durham")) \ 199 | .withColumn("datasetId", col("fields.id")) \ 200 | .withColumn("name", col("fields.premise_name")) \ 201 | .withColumn("address1", col("fields.premise_address1")) \ 202 | .withColumn("address2", col("fields.premise_address2")) \ 203 | .withColumn("city", col("fields.premise_city")) \ 204 | .withColumn("state", col("fields.premise_state")) \ 205 | .withColumn("zip", col("fields.premise_zip")) \ 206 | .withColumn("tel", col("fields.premise_phone")) \ 207 | .withColumn("dateStart", col("fields.opening_date")) \ 208 | .withColumn("dateEnd", col("fields.closing_date")) \ 209 | .withColumn("type", split(col("fields.type_description"), " - ").getItem(1)) \ 210 | .withColumn("geoX", col("fields.geolocation").getItem(0)) \ 211 | .withColumn("geoY", col("fields.geolocation").getItem(1)) \ 212 | .drop(*drop_cols) 213 | 214 | df2 = df2.withColumn("id", concat(col("state"), lit("_"), col("county"), lit("_"), col("datasetId"))) 215 | # I left the following line if you want to play with repartitioning 216 | df = df.repartition(4); 217 | ``` 218 | 219 | Union uitvoeren 220 | ``` 221 | df = df1.unionByName(df2) 222 | df.show(5) 223 | df.printSchema() 224 | print("We have {} records.".format(df.count())) 225 | partition_count = df.rdd.getNumPartitions() 226 | print("Partition count: {}".format(partition_count)) 227 | 228 | 229 | spark.stop() 230 | ``` 231 | 232 | 233 | 234 | ### array to dataframe 235 | 236 | ``` 237 | from pyspark.sql import SparkSession 238 | from pyspark.sql.types import StructType,StructField,StringType 239 | 240 | # Creates a session on a local master 241 | spark = SparkSession.builder.appName("Array to Dataframe") \ 242 | .master("spark://69280b13519d:7077").getOrCreate() 243 | 244 | data = [['Jean'], ['Liz'], ['Pierre'], ['Lauric']] 245 | 246 | """ 247 | * data: parameter list1, data to create a dataset 248 | * encoder: parameter list2, implicit encoder 249 | """ 250 | schema = StructType([StructField('name', StringType(), True)]) 251 | 252 | df = spark.createDataFrame(data, schema) 253 | df.show() 254 | df.printSchema() 255 | 256 | spark.stop() 257 | ``` 258 | 259 | 260 | 261 | ## Hoofdstuk 4 262 | [Chapter 4 op Github](https://github.com/jgperrin/net.jgp.books.spark.ch04) 263 | 264 | Deze eerste praktijkoefening gaat erom dat Spark "fundamentally lazy" is. Je kunt transformaties uitvoeren op je data, maar zolang je het resultaat niet opvraagt, gaat Spark nog niet te werk. 265 | 266 | ``` 267 | import time 268 | from pyspark.sql import SparkSession 269 | from pyspark.sql.functions import (lit,col,concat,expr) 270 | 271 | absolute_file_path = "hdfs://namenode:8020/data/sparkinaction/NCHS_-_Teen_Birth_Rates_for_Age_Group_15-19_in_the_United_States_by_County.csv" 272 | 273 | mode="" 274 | t0 = int(round(time.time() * 1000)) 275 | 276 | # Step 1 - Creates a session on a local master 277 | spark = SparkSession.builder.appName("Analysing Catalyst's behavior") \ 278 | .master("spark://69280b13519d:7077").getOrCreate() 279 | 280 | t1 = int(round(time.time() * 1000)) 281 | 282 | print("1. Creating a session ........... {}".format(t1 - t0)) 283 | 284 | # Step 2 - Reads a CSV file with header, stores it in a dataframe 285 | df = spark.read.csv(header=True, inferSchema=True,path=absolute_file_path) 286 | 287 | initalDf = df 288 | t2 = int(round(time.time() * 1000)) 289 | print("2. Loading initial dataset ...... {}".format(t2 - t1)) 290 | 291 | # Step 3 - Build a bigger dataset 292 | for x in range(60): 293 | df = df.union(initalDf) 294 | 295 | t3 = int(round(time.time() * 1000)) 296 | print("3. Building full dataset ........ {}".format(t3 - t2)) 297 | 298 | # Step 4 - Cleanup. preparation 299 | df = df.withColumnRenamed("Lower Confidence Limit", "lcl") \ 300 | .withColumnRenamed("Upper Confidence Limit", "ucl") 301 | 302 | t4 = int(round(time.time() * 1000)) 303 | print("4. Clean-up ..................... {}".format(t4 - t3)) 304 | 305 | # Step 5 - Transformation 306 | if mode.lower != "noop": 307 | df = df.withColumn("avg", expr("(lcl+ucl)/2")) \ 308 | .withColumn("lcl2", col("lcl")) \ 309 | .withColumn("ucl2", col("ucl")) 310 | if mode.lower == "full": 311 | df = df.drop("avg","lcl2","ucl2") 312 | 313 | 314 | t5 = int(round(time.time() * 1000)) 315 | print("5. Transformations ............. {}".format(t5 - t4)) 316 | 317 | # Step 6 - Action 318 | df.collect() 319 | t6 = int(round(time.time() * 1000)) 320 | print("6. Final action ................. {}".format(t6 - t5)) 321 | 322 | print("") 323 | print("# of records .................... {}".format(df.count)) 324 | 325 | spark.stop() 326 | ``` 327 | 328 | 329 | ### Explaining Spark transformations 330 | ``` 331 | from pyspark.sql import SparkSession 332 | from pyspark.sql.functions import (lit,col,concat,expr) 333 | 334 | absolute_file_path = "hdfs://namenode:8020/data/sparkinaction/NCHS_-_Teen_Birth_Rates_for_Age_Group_15-19_in_the_United_States_by_County.csv" 335 | 336 | # Step 1 - Creates a session on a local master 337 | spark = SparkSession.builder.appName("Analysing Catalyst's behavior") \ 338 | .master("local[*]").getOrCreate() 339 | 340 | # Step 2 - Reads a CSV file with header, stores it in a dataframe 341 | df = spark.read.csv(header=True, inferSchema=True,path=absolute_file_path) 342 | 343 | df0 = df 344 | 345 | # Step 3 - Build a bigger dataset 346 | df = df.union(df0) 347 | 348 | # Step 4 - Cleanup. preparation 349 | df = df.withColumnRenamed("Lower Confidence Limit", "lcl") \ 350 | .withColumnRenamed("Upper Confidence Limit", "ucl") 351 | 352 | # Step 5 - Transformation 353 | df = df.withColumn("avg", expr("(lcl+ucl)/2")) \ 354 | .withColumn("lcl2", col("lcl")) \ 355 | .withColumn("ucl2", col("ucl")) 356 | ``` 357 | Met explain kun je zien hoe Spark te werk is gegaan. Dit kan handig zijn om je Spark code te debuggen. 358 | ``` 359 | # Step 6 - explain 360 | df.explain() 361 | 362 | spark.stop() 363 | ``` 364 | -------------------------------------------------------------------------------- /startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hadoop fs -mkdir /tmp 4 | hadoop fs -mkdir -p /user/hive/warehouse 5 | hadoop fs -chmod g+w /tmp 6 | hadoop fs -chmod g+w /user/hive/warehouse 7 | 8 | cd $HIVE_HOME/bin 9 | ./hiveserver2 --hiveconf hive.server2.enable.doAs=false 10 | -------------------------------------------------------------------------------- /superset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM apache/superset:latest 2 | 3 | USER root 4 | 5 | RUN pip install mysqlclient 6 | RUN pip install pyhive 7 | RUN pip install impyla 8 | 9 | ENV ADMIN_USERNAME $ADMIN_USERNAME 10 | ENV ADMIN_EMAIL $ADMIN_EMAIL 11 | ENV ADMIN_PASSWORD $ADMIN_PASSWORD 12 | 13 | COPY superset-init.sh /superset-init.sh 14 | 15 | COPY superset_config.py /app/ 16 | ENV SUPERSET_CONFIG_PATH /app/superset_config.py 17 | 18 | USER superset 19 | ENTRYPOINT [ "/superset-init.sh" ] -------------------------------------------------------------------------------- /superset/superset-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # create Admin user, you can read these values from env or anywhere else possible 4 | superset fab create-admin --username "$ADMIN_USERNAME" --firstname Superset --lastname Admin --email "$ADMIN_EMAIL" --password "$ADMIN_PASSWORD" 5 | 6 | # Upgrading Superset metastore 7 | superset db upgrade 8 | 9 | # setup roles and permissions 10 | superset superset init 11 | 12 | # Starting server 13 | /bin/sh -c /usr/bin/run-server.sh -------------------------------------------------------------------------------- /superset/superset_config.py: -------------------------------------------------------------------------------- 1 | FEATURE_FLAGS = { 2 | "ENABLE_TEMPLATE_PROCESSING": True, 3 | } 4 | 5 | ENABLE_PROXY_FIX = True 6 | SECRET_KEY = '7/rWpXFGCHoqsEhUgxtPCg==' -------------------------------------------------------------------------------- /template/java/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/spark-submit:3.0.0-hadoop3.2 2 | 3 | LABEL maintainer="Gezim Sejdiu , Giannis Mouchakis " 4 | 5 | ENV SPARK_APPLICATION_JAR_NAME application-1.0 6 | 7 | COPY template.sh / 8 | 9 | RUN apk add --no-cache openjdk8 maven\ 10 | && chmod +x /template.sh \ 11 | && mkdir -p /app \ 12 | && mkdir -p /usr/src/app 13 | 14 | # Copy the POM-file first, for separate dependency resolving and downloading 15 | ONBUILD COPY pom.xml /usr/src/app 16 | ONBUILD RUN cd /usr/src/app \ 17 | && mvn dependency:resolve 18 | ONBUILD RUN cd /usr/src/app \ 19 | && mvn verify 20 | 21 | # Copy the source code and build the application 22 | ONBUILD COPY . /usr/src/app 23 | ONBUILD RUN cd /usr/src/app \ 24 | && mvn clean package 25 | 26 | CMD ["/bin/bash", "/template.sh"] 27 | -------------------------------------------------------------------------------- /template/java/README.md: -------------------------------------------------------------------------------- 1 | # Spark Java template 2 | 3 | The Spark Java template image serves as a base image to build your own Java application to run on a Spark cluster. See [big-data-europe/docker-spark README](https://github.com/big-data-europe/docker-spark) for a description how to setup a Spark cluster. 4 | 5 | ### Package your application using Maven 6 | You can build and launch your Java application on a Spark cluster by extending this image with your sources. The template uses [Maven](https://maven.apache.org/) as build tool, so make sure you have a `pom.xml` file for your application specifying all the dependencies. 7 | 8 | The Maven `package` command must create an assembly JAR (or 'uber' JAR) containing your code and its dependencies. Spark and Hadoop dependencies should be listes as `provided`. The [Maven shade plugin](http://maven.apache.org/plugins/maven-shade-plugin/) provides a plugin to build such assembly JARs. 9 | 10 | ### Extending the Spark Java template with your application 11 | 12 | #### Steps to extend the Spark Java template 13 | 1. Create a Dockerfile in the root folder of your project (which also contains a `pom.xml`) 14 | 2. Extend the Spark Java template Docker image 15 | 3. Configure the following environment variables (unless the default value satisfies): 16 | * `SPARK_MASTER_NAME` (default: spark-master) 17 | * `SPARK_MASTER_PORT` (default: 7077) 18 | * `SPARK_APPLICATION_JAR_NAME` (default: application-1.0) 19 | * `SPARK_APPLICATION_MAIN_CLASS` (default: my.main.Application) 20 | * `SPARK_APPLICATION_ARGS` (default: "") 21 | 4. Build and run the image 22 | ``` 23 | docker build --rm=true -t bde/spark-app . 24 | docker run --name my-spark-app -e ENABLE_INIT_DAEMON=false --link spark-master:spark-master -d bde/spark-app 25 | ``` 26 | 27 | The sources in the project folder will be automatically added to `/usr/src/app` if you directly extend the Spark Java template image. Otherwise you will have to add and package the sources by yourself in your Dockerfile with the commands: 28 | 29 | COPY . /usr/src/app 30 | RUN cd /usr/src/app \ 31 | && mvn clean package 32 | 33 | If you overwrite the template's `CMD` in your Dockerfile, make sure to execute the `/template.sh` script at the end. 34 | 35 | #### Example Dockerfile 36 | ``` 37 | FROM bde2020/spark-java-template:2.4.0-hadoop2.7 38 | 39 | MAINTAINER Erika Pauwels 40 | MAINTAINER Gezim Sejdiu 41 | 42 | ENV SPARK_APPLICATION_JAR_NAME my-app-1.0-SNAPSHOT-with-dependencies 43 | ENV SPARK_APPLICATION_MAIN_CLASS eu.bde.my.Application 44 | ENV SPARK_APPLICATION_ARGS "foo bar baz" 45 | ``` 46 | 47 | #### Example application 48 | See [big-data-europe/demo-spark-sensor-data](https://github.com/big-data-europe/demo-spark-sensor-data). 49 | -------------------------------------------------------------------------------- /template/java/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /usr/src/app 4 | cp target/${SPARK_APPLICATION_JAR_NAME}.jar ${SPARK_APPLICATION_JAR_LOCATION} 5 | 6 | sh /submit.sh 7 | -------------------------------------------------------------------------------- /template/python/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/spark-submit:3.0.0-hadoop3.2 2 | 3 | LABEL maintainer="Gezim Sejdiu , Giannis Mouchakis " 4 | 5 | COPY template.sh / 6 | 7 | # Copy the requirements.txt first, for separate dependency resolving and downloading 8 | ONBUILD COPY requirements.txt /app/ 9 | ONBUILD RUN cd /app \ 10 | && pip3 install -r requirements.txt 11 | 12 | # Copy the source code 13 | ONBUILD COPY . /app 14 | 15 | CMD ["/bin/bash", "/template.sh"] 16 | -------------------------------------------------------------------------------- /template/python/README.md: -------------------------------------------------------------------------------- 1 | # Spark Python template 2 | 3 | The Spark Python template image serves as a base image to build your own Python application to run on a Spark cluster. See [big-data-europe/docker-spark README](https://github.com/big-data-europe/docker-spark) for a description how to setup a Spark cluster. 4 | 5 | ### Package your application using pip 6 | You can build and launch your Python application on a Spark cluster by extending this image with your sources. The template uses [pip](https://pip.pypa.io/en/stable/) to manage the dependencies of your 7 | project, so make sure you have a `requirements.txt` file in the root of your application specifying all the dependencies. 8 | 9 | ### Extending the Spark Python template with your application 10 | 11 | #### Steps to extend the Spark Python template 12 | 1. Create a Dockerfile in the root folder of your project (which also contains a `requirements.txt`) 13 | 2. Extend the Spark Python template Docker image 14 | 3. Configure the following environment variables (unless the default value satisfies): 15 | * `SPARK_MASTER_NAME` (default: spark-master) 16 | * `SPARK_MASTER_PORT` (default: 7077) 17 | * `SPARK_APPLICATION_PYTHON_LOCATION` (default: /app/app.py) 18 | * `SPARK_APPLICATION_ARGS` 19 | 4. Build and run the image 20 | ``` 21 | docker build --rm -t bde/spark-app . 22 | docker run --name my-spark-app -e ENABLE_INIT_DAEMON=false --link spark-master:spark-master -d bde/spark-app 23 | ``` 24 | 25 | The sources in the project folder will be automatically added to `/app` if you directly extend the Spark Python template image. Otherwise you will have to add the sources by yourself in your Dockerfile with the command: 26 | 27 | COPY . /app 28 | 29 | If you overwrite the template's `CMD` in your Dockerfile, make sure to execute the `/template.sh` script at the end. 30 | 31 | #### Example Dockerfile 32 | ``` 33 | FROM bde2020/spark-python-template:2.4.0-hadoop2.7 34 | 35 | MAINTAINER You 36 | 37 | ENV SPARK_APPLICATION_PYTHON_LOCATION /app/entrypoint.py 38 | ENV SPARK_APPLICATION_ARGS "foo bar baz" 39 | ``` 40 | 41 | #### Example application 42 | Coming soon 43 | -------------------------------------------------------------------------------- /template/python/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sh /submit.sh 4 | -------------------------------------------------------------------------------- /template/scala/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/spark-submit:3.0.0-hadoop3.2 2 | 3 | LABEL maintainer="Gezim Sejdiu , Giannis Mouchakis " 4 | 5 | ARG SBT_VERSION 6 | ENV SBT_VERSION=${SBT_VERSION:-1.3.12} 7 | 8 | RUN wget -O - https://piccolo.link/sbt-1.3.12.tgz | gunzip | tar -x -C /usr/local 9 | 10 | ENV PATH /usr/local/sbt/bin:${PATH} 11 | 12 | WORKDIR /app 13 | 14 | # Pre-install base libraries 15 | ADD build.sbt /app/ 16 | ADD plugins.sbt /app/project/ 17 | RUN sbt update 18 | 19 | COPY template.sh / 20 | 21 | ENV SPARK_APPLICATION_MAIN_CLASS Application 22 | 23 | # Copy the build.sbt first, for separate dependency resolving and downloading 24 | ONBUILD COPY build.sbt /app/ 25 | ONBUILD COPY project /app/project 26 | ONBUILD RUN sbt update 27 | 28 | # Copy the source code and build the application 29 | ONBUILD COPY . /app 30 | ONBUILD RUN sbt clean assembly 31 | 32 | CMD ["/template.sh"] 33 | -------------------------------------------------------------------------------- /template/scala/README.md: -------------------------------------------------------------------------------- 1 | # Spark Scala template 2 | 3 | The Spark Scala template image serves as a base image to build your own Scala 4 | application to run on a Spark cluster. See 5 | [big-data-europe/docker-spark README](https://github.com/big-data-europe/docker-spark) 6 | for a description how to setup a Spark cluster. 7 | 8 | ## Scala Console 9 | 10 | `sbt console` will create you a Spark Context for testing your code like the 11 | spark-shell: 12 | 13 | ``` 14 | docker run -it --rm bde2020/spark-scala-template sbt console 15 | ``` 16 | 17 | You can also use directly your Docker image and test your own code that way. 18 | 19 | ## Package your application using sbt 20 | 21 | You can build and launch your Scala application on a Spark cluster by extending 22 | this image with your sources. The template uses 23 | [sbt](http://www.scala-sbt.org) as build tool, so you should take the 24 | `build.sbt` file located in this directory and the `project` directory that 25 | includes the 26 | [sbt-assembly](https://github.com/sbt/sbt-assembly). 27 | 28 | When the Docker image is built using this template, you should get a Docker 29 | image that includes a fat JAR containing your application and all its 30 | dependencies. 31 | 32 | ### Extending the Spark Scala template with your application 33 | 34 | #### Steps to extend the Spark Scala template 35 | 36 | 1. Create a Dockerfile in the root folder of your project (which also contains 37 | a `build.sbt`) 38 | 2. Extend the Spark Scala template Docker image 39 | 3. Configure the following environment variables (unless the default value 40 | satisfies): 41 | * `SPARK_MASTER_NAME` (default: spark-master) 42 | * `SPARK_MASTER_PORT` (default: 7077) 43 | * `SPARK_APPLICATION_MAIN_CLASS` (default: Application) 44 | * `SPARK_APPLICATION_ARGS` (default: "") 45 | 4. Build and run the image: 46 | ``` 47 | docker build --rm=true -t bde/spark-app . 48 | docker run --name my-spark-app -e ENABLE_INIT_DAEMON=false --link spark-master:spark-master -d bde/spark-app 49 | ``` 50 | 51 | The sources in the project folder will be automatically added to `/usr/src/app` 52 | if you directly extend the Spark Scala template image. Otherwise you will have 53 | to add and package the sources by yourself in your Dockerfile with the 54 | commands: 55 | 56 | COPY . /usr/src/app 57 | RUN cd /usr/src/app && sbt clean assembly 58 | 59 | If you overwrite the template's `CMD` in your Dockerfile, make sure to execute 60 | the `/template.sh` script at the end. 61 | 62 | #### Example Dockerfile 63 | 64 | ``` 65 | FROM bde2020/spark-scala-template:2.4.0-hadoop2.7 66 | 67 | MAINTAINER Cecile Tonglet 68 | 69 | ENV SPARK_APPLICATION_MAIN_CLASS eu.bde.my.Application 70 | ENV SPARK_APPLICATION_ARGS "foo bar baz" 71 | ``` 72 | 73 | #### Example application 74 | 75 | TODO 76 | -------------------------------------------------------------------------------- /template/scala/build.sbt: -------------------------------------------------------------------------------- 1 | scalaVersion := "2.12.11" 2 | libraryDependencies ++= Seq( 3 | "org.apache.spark" %% "spark-sql" % "3.0.0" % "provided" 4 | ) 5 | -------------------------------------------------------------------------------- /template/scala/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") -------------------------------------------------------------------------------- /template/scala/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SPARK_APPLICATION_JAR_LOCATION=`find /app/target -iname '*-assembly-*.jar' | head -n1` 4 | export SPARK_APPLICATION_JAR_LOCATION 5 | 6 | if [ -z "$SPARK_APPLICATION_JAR_LOCATION" ]; then 7 | echo "Can't find a file *-assembly-*.jar in /app/target" 8 | exit 1 9 | fi 10 | 11 | /submit.sh 12 | -------------------------------------------------------------------------------- /worker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bde2020/spark-base:3.0.0-hadoop3.2 2 | 3 | LABEL maintainer="Gezim Sejdiu , Giannis Mouchakis " 4 | 5 | COPY worker.sh / 6 | 7 | ENV SPARK_WORKER_WEBUI_PORT 8081 8 | ENV SPARK_WORKER_LOG /spark/logs 9 | ENV SPARK_MASTER "spark://spark-master:7077" 10 | 11 | EXPOSE 8081 12 | 13 | CMD ["/bin/bash", "/worker.sh"] 14 | -------------------------------------------------------------------------------- /worker/README.md: -------------------------------------------------------------------------------- 1 | # Spark worker 2 | 3 | See [big-data-europe/docker-spark README](https://github.com/big-data-europe/docker-spark). -------------------------------------------------------------------------------- /worker/worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . "/spark/sbin/spark-config.sh" 4 | 5 | . "/spark/bin/load-spark-env.sh" 6 | 7 | mkdir -p $SPARK_WORKER_LOG 8 | 9 | export SPARK_HOME=/spark 10 | 11 | ln -sf /dev/stdout $SPARK_WORKER_LOG/spark-worker.out 12 | 13 | /spark/sbin/../bin/spark-class org.apache.spark.deploy.worker.Worker \ 14 | --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_WORKER_LOG/spark-worker.out 15 | --------------------------------------------------------------------------------