├── paraflow-commons ├── README.md └── src │ └── main │ ├── java │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── commons │ │ ├── ParaflowFiberPartitioner.java │ │ ├── func │ │ ├── DeserializableFunction.java │ │ └── SerializableFunction.java │ │ ├── utils │ │ ├── FormTopicName.java │ │ ├── BytesUtils.java │ │ └── DateUtil.java │ │ ├── ParaflowThreadExceptionHandler.java │ │ ├── ParaflowThreadFactory.java │ │ ├── ParaflowRecord.java │ │ ├── exceptions │ │ ├── UpdateBlockPathException.java │ │ ├── TblFuncCreationException.java │ │ ├── ColumnsNotExistException.java │ │ ├── ParaFlowExceptionLevel.java │ │ ├── UserCreationException.java │ │ ├── MetaTableCorruptedException.java │ │ ├── DatabaseCreationException.java │ │ ├── MessageDeSerializationException.java │ │ ├── MetaInitException.java │ │ ├── MetaTableCreationException.java │ │ ├── ActionParamNotValidException.java │ │ ├── SQLExecutionException.java │ │ ├── MessageSerializeException.java │ │ ├── MethodNotImplementedException.java │ │ ├── ConfigFileNotFoundException.java │ │ ├── RPCServerIOException.java │ │ ├── TableRenameException.java │ │ └── ColumnRenameException.java │ │ ├── Metric.java │ │ └── TopicFiber.java │ └── resources │ └── paraflow.properties ├── paraflow-http-server └── src │ ├── main │ ├── resources │ │ ├── static │ │ │ ├── assets │ │ │ │ ├── plugins │ │ │ │ │ ├── maps-google │ │ │ │ │ │ ├── plugin.js │ │ │ │ │ │ └── plugin.css │ │ │ │ │ ├── iconfonts │ │ │ │ │ │ └── fonts │ │ │ │ │ │ │ ├── themify │ │ │ │ │ │ │ ├── themify.eot │ │ │ │ │ │ │ ├── themify.ttf │ │ │ │ │ │ │ └── themify.woff │ │ │ │ │ │ │ ├── simple-line-icons │ │ │ │ │ │ │ ├── Simple-Line-Icons.eot │ │ │ │ │ │ │ ├── Simple-Line-Icons.ttf │ │ │ │ │ │ │ ├── Simple-Line-Icons.woff │ │ │ │ │ │ │ └── Simple-Line-Icons.woff2 │ │ │ │ │ │ │ ├── weathericons │ │ │ │ │ │ │ ├── weathericons-regular-webfont.eot │ │ │ │ │ │ │ ├── weathericons-regular-webfont.ttf │ │ │ │ │ │ │ ├── weathericons-regular-webfont.woff │ │ │ │ │ │ │ └── weathericons-regular-webfont.woff2 │ │ │ │ │ │ │ └── materialdesignicons │ │ │ │ │ │ │ ├── materialdesignicons-webfont.eot │ │ │ │ │ │ │ ├── materialdesignicons-webfont.ttf │ │ │ │ │ │ │ ├── materialdesignicons-webfont.woff │ │ │ │ │ │ │ └── materialdesignicons-webfont.woff2 │ │ │ │ │ ├── input-mask │ │ │ │ │ │ └── plugin.js │ │ │ │ │ ├── charts-c3 │ │ │ │ │ │ └── plugin.js │ │ │ │ │ ├── fullcalendar │ │ │ │ │ │ └── plugin.js │ │ │ │ │ └── prismjs │ │ │ │ │ │ └── plugin.js │ │ │ │ ├── sdk │ │ │ │ │ └── icons │ │ │ │ │ │ └── fonts │ │ │ │ │ │ ├── FontAwesome.otf │ │ │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ │ │ ├── fontawesome-webfont.woff │ │ │ │ │ │ ├── fontawesome-webfont.woff2 │ │ │ │ │ │ ├── 6xK3dSBYKcSV-LCoeQqfX1RYOo3qOK7l.woff2 │ │ │ │ │ │ └── 6xKydSBYKcSV-LCoeQqfX1RYOo3i54rwlxdu.woff2 │ │ │ │ └── fonts │ │ │ │ │ └── feather │ │ │ │ │ ├── feather-webfont.eot │ │ │ │ │ ├── feather-webfont.ttf │ │ │ │ │ └── feather-webfont.woff │ │ │ ├── docs.html │ │ │ └── tpl │ │ │ │ ├── tplQuery.htm │ │ │ │ └── tplTable.htm │ │ ├── application.properties │ │ ├── application-prod.properties │ │ └── application-dev.properties │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── http │ │ └── server │ │ ├── model │ │ ├── Metric.java │ │ ├── Pipeline.java │ │ ├── Column.java │ │ ├── ClusterInfo.java │ │ ├── Base.java │ │ ├── Table.java │ │ ├── Json.java │ │ └── DynamicJson.java │ │ ├── ParaflowHttpConfig.java │ │ ├── HttpController.java │ │ ├── ParaflowHttpServer.java │ │ └── configuration │ │ └── CORSConfiguration.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── http │ └── server │ └── TestParaflowHttpServer.java ├── docs ├── images │ └── GitHub-Mark-64px.png ├── _config.yml └── stylesheets │ └── reset.css ├── paraflow-1.0-alpha1 ├── sbin │ ├── paraflow_logclean.sh │ ├── paraflow-collector-init.sh │ ├── paraflow-loader-init.sh │ ├── zookeeper-init.sh │ ├── paraflow-init.sh │ ├── logclean.sh │ ├── clear.sh │ ├── kafka-init.sh │ ├── stop-all.sh │ └── zookeeper-server.sh ├── config │ ├── metaserver.conf │ ├── paraflow.properties │ ├── collector.conf │ └── log4j.properties ├── run │ ├── sh.txt │ ├── exe_script.sh │ ├── exe_mv.sh │ ├── exe_scp.sh │ ├── mv-catalog.sh │ ├── exe_cmd.sh │ ├── paraflow_scp.sh │ └── stop-all.sh └── bin │ ├── paraflow-loader-stop.sh │ ├── paraflow-collector-stop.sh │ ├── paraflow-metaserver-stop.sh │ ├── paraflow-log-clean.sh │ ├── paraflow-metaserver-start.sh │ ├── paraflow-loader-start.sh │ └── paraflow-collector-start.sh ├── paraflow-1.0-config └── paraflow │ ├── sbin │ ├── paraflow_logclean.sh │ ├── paraflow-collector-init.sh │ ├── paraflow-loader-init.sh │ ├── zookeeper-init.sh │ ├── kafka-init.sh │ └── zookeeper-server.sh │ ├── config │ ├── paraflow.properties │ ├── metaserver.conf │ ├── collector.conf │ └── log4j.properties │ ├── paraflow-1.0-alpha1 │ ├── config │ │ ├── paraflow.properties │ │ ├── metaserver.conf │ │ ├── collector.conf │ │ └── log4j.properties │ ├── bin │ │ ├── paraflow-loader-stop.sh │ │ ├── paraflow-collector-stop.sh │ │ ├── paraflow-metaserver-stop.sh │ │ ├── paraflow-metaserver-start.sh │ │ ├── paraflow-loader-start.sh │ │ └── paraflow-collector-start.sh │ ├── configuration │ │ ├── exe.sh │ │ └── mv-catalog.sh │ └── sbin │ │ ├── paraflow-collector-init.sh │ │ ├── paraflow-loader-init.sh │ │ ├── zookeeper-init.sh │ │ ├── kafka-init.sh │ │ └── zookeeper-server.sh │ └── bin │ ├── paraflow-loader-stop.sh │ ├── paraflow-collector-stop.sh │ ├── paraflow-metaserver-stop.sh │ ├── paraflow-metaserver-start.sh │ ├── paraflow-loader-start.sh │ └── paraflow-collector-start.sh ├── paraflow-benchmark └── src │ ├── main │ ├── resources │ │ └── sql │ │ │ ├── region.sql │ │ │ ├── q2.sql │ │ │ ├── nation.sql │ │ │ ├── q1.sql │ │ │ └── customer.sql │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── benchmark │ │ ├── model │ │ ├── Model.java │ │ ├── Column.java │ │ ├── RegionColumn.java │ │ ├── NationColumn.java │ │ ├── Region.java │ │ ├── CustomerColumn.java │ │ └── Nation.java │ │ ├── generator │ │ ├── Generator.java │ │ └── TimeGenerationPool.java │ │ └── query │ │ ├── QueryTemplate.java │ │ ├── DBSelectTemplate.java │ │ ├── QueryGenerator.java │ │ ├── DBDeleteTemplate.java │ │ ├── PrestoSelectTemplate3.java │ │ ├── QueryDistribution.java │ │ └── PrestoSelectTemplate1.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── benchmark │ ├── QuestionerTest.java │ └── TimeGenerationPoolTest.java ├── paraflow-collector ├── README.md └── src │ ├── main │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── collector │ │ ├── MessageSerializationSchema.java │ │ ├── CollectorEnvironment.java │ │ ├── StringMessageSerializationSchema.java │ │ ├── DataSource.java │ │ ├── DataSink.java │ │ ├── utils │ │ ├── DefaultKafkaPartitioner.java │ │ ├── CollectorThroughputMetric.java │ │ └── KafkaPartitioner.java │ │ ├── CollectorRuntime.java │ │ ├── DataFlow.java │ │ └── FiberFlow.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── collector │ ├── TestKafkaBasic.java │ └── TestCollectorThroughputMetric.java ├── paraflow-loader └── src │ ├── main │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── loader │ │ ├── SegmentCallback.java │ │ ├── DataTransformer.java │ │ ├── RunningProcessor.java │ │ ├── ParaflowSortedBuffer.java │ │ ├── Processor.java │ │ └── ParaflowKafkaConsumer.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── loader │ └── utils │ ├── SortTest.java │ └── StringTest.java ├── paraflow-connector └── src │ ├── main │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── connector │ │ ├── exception │ │ ├── ConfigurationException.java │ │ ├── ParaflowCursorException.java │ │ ├── TableNotFoundException.java │ │ ├── UnSupportedFunctionException.java │ │ ├── ParaflowSplitNotOpenException.java │ │ └── ParaflowErrorCode.java │ │ ├── impl │ │ └── DefaultFiberPartitioner.java │ │ ├── StorageFormat.java │ │ ├── ParaflowPartitioningHandle.java │ │ ├── function │ │ └── Function.java │ │ ├── Types.java │ │ ├── ParaflowPlugin.java │ │ └── ParaflowMetadataFactory.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── connector │ └── TestHdfsFile.java ├── paraflow-master └── src │ └── main │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── master │ └── MasterServer.java ├── paraflow-metaserver └── src │ ├── test │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── metaserver │ │ └── server │ │ └── TestMetaServer.java │ └── main │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── metaserver │ ├── server │ ├── StartupHook.java │ └── StartupPipeline.java │ ├── client │ └── MetaCache.java │ ├── action │ ├── Action.java │ ├── GetMetaTablesAction.java │ ├── UpdateBlockPathAction.java │ ├── CreateTblFuncAction.java │ ├── CreateUserAction.java │ ├── ActionResponse.java │ ├── GetUserNameAction.java │ ├── GetDatabaseIdAction.java │ ├── GetUserIdAction.java │ ├── CreateDatabaseAction.java │ └── GetTableIdAction.java │ ├── connection │ ├── Connection.java │ ├── ResultList.java │ ├── JDBCRecord.java │ └── TransactionController.java │ └── utils │ ├── ColType.java │ ├── MetaVersion.java │ ├── Utils.java │ └── MetaConfig.java ├── .gitignore ├── paraflow-assembly └── pom.xml ├── paraflow-examples └── src │ ├── main │ └── java │ │ └── cn │ │ └── edu │ │ └── ruc │ │ └── iir │ │ └── paraflow │ │ └── examples │ │ ├── collector │ │ ├── BasicParaflowFiberPartitioner.java │ │ └── MockDataSink.java │ │ └── loader │ │ └── BasicLoader.java │ └── test │ └── java │ └── cn │ └── edu │ └── ruc │ └── iir │ └── paraflow │ └── examples │ ├── TestTpchDataSource.java │ └── TestSerialization.java └── README.md /paraflow-commons/README.md: -------------------------------------------------------------------------------- 1 | ## Common Exceptions -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/maps-google/plugin.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/maps-google/plugin.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.profiles.active=prod -------------------------------------------------------------------------------- /docs/images/GitHub-Mark-64px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/docs/images/GitHub-Mark-64px.png -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/paraflow_logclean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd /home/iir/opt/paraflow/logs;ls | egrep -v "*.out" | xargs rm -rf 3 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/paraflow_logclean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cd /home/iir/opt/paraflow/logs;ls | egrep -v "*.out" | xargs rm -rf 3 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/resources/sql/region.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE region(r_regionkey int primary key, r_name varchar(25), r_comment varchar(152)); -------------------------------------------------------------------------------- /paraflow-collector/README.md: -------------------------------------------------------------------------------- 1 | # paraflow-loader-producer 2 | 3 | ## APIs 4 | 5 | ## Example 6 | See `cn.edu.iir.paraflow.loader.producer.example.ExampleProducer`. -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/resources/sql/q2.sql: -------------------------------------------------------------------------------- 1 | SELECT SUM(lo_totalprice), AVG(lo_tax), MIN(lo_orderdate), MAX(lo_creation) FROM paraflow.test.tpch WHERE lo_custkey < 10000; -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - 3 | scope: 4 | path: "" # an empty string here means all files in the project 5 | values: 6 | layout: "default" 7 | 8 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/resources/sql/nation.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE nation(n_nationkey int primary key, n_name varchar(25), n_regionkey int references region(r_regionkey), n_comment varchar(152)); -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/SegmentCallback.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | public interface SegmentCallback 4 | { 5 | void callback(); 6 | } 7 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.ttf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/fonts/feather/feather-webfont.woff -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.connector.exception; 2 | 3 | public class ConfigurationException extends Exception 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /paraflow-master/src/main/java/cn/edu/ruc/iir/paraflow/master/MasterServer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.master; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class MasterServer 9 | { 10 | } 11 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.ttf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/themify/themify.woff -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/config/metaserver.conf: -------------------------------------------------------------------------------- 1 | server.port=10012 2 | db.driver=org.postgresql.Driver 3 | db.host=jdbc:postgresql://127.0.0.1:5432/paraflowmeta 4 | db.user=paraflow 5 | db.password=paraflow 6 | hdfs.warehouse=hdfs://dbiir01:9000/paraflow 7 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/config/paraflow.properties: -------------------------------------------------------------------------------- 1 | # paraflow connector configurations 2 | connector.name=paraflow 3 | 4 | metaserver.host=127.0.0.1 5 | metaserver.port=10012 6 | hdfs.warehouse=hdfs://dbiir01:9000/paraflow 7 | 8 | fiber.enabled=true 9 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/resources/sql/q1.sql: -------------------------------------------------------------------------------- 1 | SELECT lo_custkey, lo_lineorderkey, c_name FROM paraflow.test.tpch, postgresql.public.customer WHERE paraflow.test.tpch.lo_custkey=postgresql.public.customer.c_custkey AND paraflow.test.tpch.lo_custkey=1; -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/config/paraflow.properties: -------------------------------------------------------------------------------- 1 | # paraflow connector configurations 2 | connector.name=paraflow 3 | 4 | metaserver.host=10.77.110.11 5 | metaserver.port=10012 6 | hdfs.warehouse=hdfs://dbiir01:9000/paraflow 7 | 8 | fiber.enabled=true -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/config/paraflow.properties: -------------------------------------------------------------------------------- 1 | # paraflow connector configurations 2 | connector.name=paraflow 3 | 4 | metaserver.host=127.0.0.1 5 | metaserver.port=10012 6 | hdfs.warehouse=hdfs://127.0.0.1:9000/paraflow 7 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Metric.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class Metric 9 | { 10 | } 11 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/config/metaserver.conf: -------------------------------------------------------------------------------- 1 | server.port=10012 2 | db.driver=org.postgresql.Driver 3 | db.host=jdbc:postgresql://10.77.110.11:5432/paraflowmeta 4 | db.user=paraflow 5 | db.password=paraflow 6 | hdfs.warehouse=hdfs://dbiir01:9000/paraflow 7 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/6xK3dSBYKcSV-LCoeQqfX1RYOo3qOK7l.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/6xK3dSBYKcSV-LCoeQqfX1RYOo3qOK7l.woff2 -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/ParaflowHttpConfig.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class ParaflowHttpConfig 9 | { 10 | } 11 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/sh.txt: -------------------------------------------------------------------------------- 1 | sed -i 's/loader.parallelism=8/loader.parallelism=16/g' /home/iir/opt/paraflow/config/loader.conf 2 | 3 | ./bin/kafka-topics.sh --delete --zookeeper dbiir02:2180,dbiir03:2180,dbiir04:2180,dbiir05:2180,dbiir06:2180 --topic test-tpch 4 | 5 | 6 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/input-mask/plugin.js: -------------------------------------------------------------------------------- 1 | require.config({ 2 | shim: { 3 | 'input-mask': ['jquery', 'core'] 4 | }, 5 | paths: { 6 | 'input-mask': 'assets/plugins/input-mask/js/jquery.mask.min' 7 | } 8 | }); -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/6xKydSBYKcSV-LCoeQqfX1RYOo3i54rwlxdu.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/sdk/icons/fonts/6xKydSBYKcSV-LCoeQqfX1RYOo3i54rwlxdu.woff2 -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/config/metaserver.conf: -------------------------------------------------------------------------------- 1 | server.port=10012 2 | db.driver=org.postgresql.Driver 3 | db.host=jdbc:postgresql://dbiir00:5432/paraflowmeta 4 | db.user=paraflow 5 | db.password=paraflow 6 | hdfs.warehouse=hdfs://dbiir10:9000/paraflow 7 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.ttf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.woff -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/simple-line-icons/Simple-Line-Icons.woff2 -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.ttf -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.woff -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-loader-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowLoader' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow loader to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/exe_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo $1 4 | 5 | prefix="dbiir" 6 | start=2 7 | end=9 8 | 9 | for ((i=$start; i <=$end; i++)) 10 | do 11 | if [ $i -lt 10 ] 12 | then 13 | ssh $prefix"0"$i $1 14 | else 15 | ssh $prefix$i $1 16 | fi 17 | done 18 | 19 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/weathericons/weathericons-regular-webfont.woff2 -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-collector-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowCollector' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow collector to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-loader-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowLoader' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow loader to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.eot -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.ttf -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/DataTransformer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.ParaflowRecord; 4 | 5 | public interface DataTransformer 6 | { 7 | ParaflowRecord transform(byte[] value, int partition); 8 | } 9 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-metaserver-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowMetaServer' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow metaserver to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/ParaflowFiberPartitioner.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public interface ParaflowFiberPartitioner 9 | { 10 | int getFiberId(byte[] key); 11 | } 12 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.woff -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbiir/paraflow/HEAD/paraflow-http-server/src/main/resources/static/assets/plugins/iconfonts/fonts/materialdesignicons/materialdesignicons-webfont.woff2 -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-collector-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowCollector' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow collector to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/Model.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public interface Model 9 | { 10 | long getRowNumber(); 11 | 12 | String toLine(); 13 | } 14 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-metaserver-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowMetaServer' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow metaserver to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-loader-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowLoader' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow loader to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-collector-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowCollector' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow collector to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-metaserver-stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PIDS=$(ps ax | grep -i 'ParaflowMetaServer' | grep java | grep -v grep | awk '{print $1}') 4 | 5 | if [ -z "$PIDS" ]; then 6 | echo "No paraflow metaserver to stop" 7 | exit 1 8 | else 9 | kill -s TERM ${PIDS} 10 | fi 11 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/charts-c3/plugin.js: -------------------------------------------------------------------------------- 1 | require.config({ 2 | shim: { 3 | 'c3': ['d3', 'core'], 4 | 'd3': ['core'], 5 | }, 6 | paths: { 7 | 'd3': 'assets/plugins/charts-c3/js/d3.v3.min', 8 | 'c3': 'assets/plugins/charts-c3/js/c3.min', 9 | } 10 | }); -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/exe_mv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo $1 4 | echo $2 5 | 6 | for ((i=2; i<=9; i++)) 7 | do 8 | echo "" 9 | echo "*****now is dbiir$i" 10 | #scp /home/iir/opt/pixels/pixels-daemon-0.1.0-SNAPSHOT-full.jar iir@dbiir0$i:/home/iir/opt/pixels/ 11 | scp -r $1 iir@dbiir0$i:$2/ 12 | 13 | done 14 | echo "task is done." 15 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/resources/sql/customer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE customer(c_custkey int primary key, c_name varchar(25), c_address varchar(40), c_nationkey int references nation(n_nationkey), c_phone varchar(15), c_acctbal double precision, c_mktsegment varchar(10), c_comment varchar(117)); 2 | CREATE INDEX customer_name_index ON customer(c_name); -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/exe_scp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo $1 4 | echo $2 5 | 6 | for ((i=1; i<=9; i++)) 7 | do 8 | echo "" 9 | echo "*****now is dbiir$i" 10 | #scp /home/iir/opt/pixels/pixels-daemon-0.1.0-SNAPSHOT-full.jar iir@dbiir0$i:/home/iir/opt/pixels/ 11 | scp -r $1 iir@dbiir0$i:$2/ 12 | 13 | done 14 | echo "task is done." 15 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/fullcalendar/plugin.js: -------------------------------------------------------------------------------- 1 | require.config({ 2 | shim: { 3 | 'fullcalendar': ['moment', 'jquery'], 4 | }, 5 | paths: { 6 | 'fullcalendar': 'assets/plugins/fullcalendar/js/fullcalendar.min', 7 | 'moment': 'assets/plugins/fullcalendar/js/moment.min', 8 | } 9 | }); -------------------------------------------------------------------------------- /paraflow-metaserver/src/test/java/cn/edu/ruc/iir/paraflow/metaserver/server/TestMetaServer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.server; 2 | 3 | import org.junit.Test; 4 | 5 | public class TestMetaServer 6 | { 7 | @Test 8 | public void testServer() 9 | { 10 | System.out.println("Hello World!"); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/configuration/exe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo $1 4 | echo $2 5 | 6 | for ((i=2; i<=9; i++)) 7 | do 8 | echo "" 9 | echo "*****now is dbiir$i" 10 | #scp /home/iir/opt/pixels/pixels-daemon-0.1.0-SNAPSHOT-full.jar iir@dbiir0$i:/home/iir/opt/pixels/ 11 | scp -r $1 iir@dbiir0$i:$2/ 12 | 13 | done 14 | echo "task is done." 15 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/generator/Generator.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.generator; 2 | 3 | import java.util.Iterator; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public interface Generator 11 | extends Iterable 12 | { 13 | @Override 14 | Iterator iterator(); 15 | } 16 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/server/StartupHook.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.server; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public interface StartupHook 11 | { 12 | void run() throws ParaFlowException; 13 | } 14 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/func/DeserializableFunction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.func; 2 | 3 | import java.io.Serializable; 4 | import java.util.function.Function; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public interface DeserializableFunction extends Function, Serializable 12 | { 13 | } 14 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/func/SerializableFunction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.func; 2 | 3 | import java.io.Serializable; 4 | import java.util.function.Function; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public interface SerializableFunction extends Function, Serializable 12 | { 13 | } 14 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/client/MetaCache.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.client; 2 | 3 | /** 4 | * MetaCache is a client-cache for metadata. It can be used in application to 5 | * reduce meta server rpc calls 6 | * 7 | * @author guodong 8 | */ 9 | public class MetaCache 10 | { 11 | // todo meta cache 12 | } 13 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/utils/FormTopicName.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.utils; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class FormTopicName 9 | { 10 | private FormTopicName() 11 | { 12 | } 13 | 14 | public static String formTopicName(String database, String table) 15 | { 16 | return database + "." + table; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/Action.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 5 | 6 | public abstract class Action 7 | { 8 | public abstract ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException; 9 | } 10 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/MessageSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public interface MessageSerializationSchema 13 | extends Serializable 14 | { 15 | Message serialize(int keyIdx, int timeIdx, T value); 16 | } 17 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/config/collector.conf: -------------------------------------------------------------------------------- 1 | # Collector 2 | collector.id=collector0 3 | 4 | # Kafka producer configuration 5 | botstrap.servers=dbiir02:9192,dbiir03:9193,dbiir04:9194,dbiir05:9195 6 | acks=1 7 | retries=1 8 | batch.size=1048576 9 | linger.ms=50 10 | buffer.memory=4194304000 11 | compression.type=lz4 12 | 13 | # Meta server 14 | meta.server.host=127.0.0.1 15 | meta.server.port=10012 16 | 17 | # PushGateWay 18 | metric.enabled=false 19 | gateway.url=dbiir00:9101 20 | 21 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/mv-catalog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for ((i=2; i<=9; i++)) 4 | do 5 | echo "*****now is dbiir$i" 6 | #scp /home/iir/opt/pixels/pixels-daemon-0.1.0-SNAPSHOT-full.jar iir@dbiir0$i:/home/iir/opt/pixels/ 7 | scp /home/iir/opt/paraflow/config/paraflow.properties iir@dbiir0$i:/home/iir/opt/presto-server-0.192/etc/catalog/ 8 | #scp /home/iir/opt/presto-server-0.192/etc/event-listener.properties iir@dbiir0$i:/home/iir/opt/presto-server-0.192/etc/ 9 | 10 | done 11 | 12 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/QueryTemplate.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public abstract class QueryTemplate 9 | { 10 | protected final String table; 11 | 12 | QueryTemplate(String table) 13 | { 14 | this.table = table; 15 | } 16 | 17 | abstract String makeQuery(); 18 | 19 | abstract QueryGenerator.QueryType getType(); 20 | } 21 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/impl/DefaultFiberPartitioner.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.connector.impl; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.ParaflowFiberPartitioner; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class DefaultFiberPartitioner 11 | implements ParaflowFiberPartitioner 12 | { 13 | @Override 14 | public int getFiberId(byte[] key) 15 | { 16 | return 0; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/docs.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Redirecting… 5 | 6 | 7 | 8 | 9 | 13 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/exe_cmd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ $# != 1 ]; then 4 | echo "1> " 5 | echo "2> " 6 | echo "3> " 7 | echo "4> " 8 | exit 1; 9 | fi 10 | 11 | if [ $1 -eq 1 ]; then 12 | /home/iir/opt/paraflow/run/exe_script.sh "tail -10 ~/opt/presto-server-0.192/data/var/log/server.log" 13 | elif [ $1 -eq 2 ]; then 14 | /home/iir/opt/paraflow/run/exe_script.sh "tail -f ~/opt/presto-server-0.192/data/var/log/server.log" 15 | else 16 | echo "Param is invalid." 17 | fi 18 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/HttpController.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server; 2 | 3 | import org.springframework.stereotype.Controller; 4 | import org.springframework.web.bind.annotation.RequestMapping; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | @Controller 12 | public class HttpController 13 | { 14 | @RequestMapping("/") 15 | public String index() 16 | { 17 | return "index.html"; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-log-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | base_dir=$(dirname $0) 4 | 5 | if [ "x$PARAFLOW_HOME" = "x" ]; then 6 | export PARAFLOW_HOME="$base_dir/../" 7 | fi 8 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 9 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 10 | fi 11 | 12 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 13 | export PARAFLOW_HEAP_OPTS="-Xmx1G -Xms512M" 14 | fi 15 | 16 | java -jar $base_dir/paraflow-client-1.0-alpha1-allinone.jar clear 5 17 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/configuration/mv-catalog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for ((i=2; i<=9; i++)) 4 | do 5 | echo "*****now is dbiir$i" 6 | #scp /home/iir/opt/pixels/pixels-daemon-0.1.0-SNAPSHOT-full.jar iir@dbiir0$i:/home/iir/opt/pixels/ 7 | scp /home/iir/opt/paraflow/config/paraflow.properties iir@dbiir0$i:/home/iir/opt/presto-server-0.192/etc/catalog/ 8 | #scp /home/iir/opt/presto-server-0.192/etc/event-listener.properties iir@dbiir0$i:/home/iir/opt/presto-server-0.192/etc/ 9 | 10 | done 11 | 12 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/config/collector.conf: -------------------------------------------------------------------------------- 1 | # Collector 2 | collector.id=collector-dbiir02 3 | 4 | # Kafka producer configuration 5 | bootstrap.servers=dbiir02:9192,dbiir03:9193,dbiir04:9194,dbiir05:9195,dbiir06:9196,dbiir07:9197,dbiir08:9198 6 | acks=1 7 | retries=1 8 | batch.size=1048576 9 | linger.ms=50 10 | buffer.memory=4194304000 11 | compression.type=lz4 12 | 13 | # Meta server 14 | meta.server.host=dbiir01 15 | meta.server.port=10012 16 | 17 | # PushGateWay 18 | metric.enabled=false 19 | gateway.url=dbiir00:9101 20 | 21 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/config/collector.conf: -------------------------------------------------------------------------------- 1 | # Collector 2 | collector.id=collector0 3 | 4 | # Kafka producer configuration 5 | bootstrap.servers=dbiir02:9192,dbiir03:9193,dbiir04:9194,dbiir05:9195,dbiir06:9196,dbiir07:9197,dbiir08:9198 6 | acks=1 7 | retries=1 8 | batch.size=1048576 9 | linger.ms=50 10 | buffer.memory=4194304000 11 | compression.type=lz4 12 | 13 | # Meta server 14 | meta.server.host=dbiir00 15 | meta.server.port=10012 16 | 17 | # PushGateWay 18 | metric.enabled=false 19 | gateway.url=dbiir00:9101 20 | 21 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/assets/plugins/prismjs/plugin.js: -------------------------------------------------------------------------------- 1 | require.config({ 2 | paths: { 3 | 'prismjs': 'assets/plugins/prismjs/js/prism.pack', 4 | }, 5 | shim: { 6 | prism: { 7 | exports: "Prism" 8 | } 9 | } 10 | }); 11 | 12 | require(['prismjs', 'jquery'], function (prismjs, $) { 13 | $(document).ready(function () { 14 | // $('[class^="language-"]').each(function(i, block) { 15 | // Prism.highlightElement(block); 16 | // }); 17 | }); 18 | }); -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/RunningProcessor.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | public class RunningProcessor 4 | implements Runnable 5 | { 6 | private final Processor processor; 7 | 8 | public RunningProcessor(Processor processor) 9 | { 10 | this.processor = processor; 11 | } 12 | 13 | @Override 14 | public void run() 15 | { 16 | processor.run(); 17 | } 18 | 19 | public void stop() 20 | { 21 | processor.stop(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/paraflow-collector-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | PARAFLOW_HOME="/home/iir/opt/paraflow" 5 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 6 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/collector.conf" 7 | 8 | # modify the configuration file 9 | modify_config() 10 | { 11 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 12 | sed -i '2s/.*/collector.id=collector-'$NODE_NAME'/' $PARAFLOW_CONFIG_FILE 13 | else 14 | echo "no write permission on collector.conf file" 15 | exit 0 16 | fi 17 | } 18 | 19 | modify_config 20 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/paraflow-collector-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | PARAFLOW_HOME="/home/iir/opt/paraflow" 5 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 6 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/collector.conf" 7 | 8 | # modify the configuration file 9 | modify_config() 10 | { 11 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 12 | sed -i '2s/.*/collector.id=collector-'$NODE_NAME'/' $PARAFLOW_CONFIG_FILE 13 | else 14 | echo "no write permission on collector.conf file" 15 | exit 0 16 | fi 17 | } 18 | 19 | modify_config 20 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/ParaflowThreadExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | import java.util.Arrays; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class ParaflowThreadExceptionHandler 11 | implements Thread.UncaughtExceptionHandler 12 | { 13 | @Override 14 | public void uncaughtException(Thread thread, Throwable t) 15 | { 16 | System.err.println("Uncaught exception detected " + t + " st: " + Arrays.toString(t.getStackTrace())); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /paraflow-http-server/src/test/java/cn/edu/ruc/iir/paraflow/http/server/TestParaflowHttpServer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | /** 9 | * paraflow 10 | * 11 | * @author guodong 12 | */ 13 | @RunWith(SpringRunner.class) 14 | @SpringBootTest 15 | public class TestParaflowHttpServer 16 | { 17 | @Test 18 | public void contextLoads() 19 | { 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/sbin/paraflow-collector-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | PARAFLOW_HOME="/home/iir/opt/paraflow" 5 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 6 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/collector.conf" 7 | 8 | # modify the configuration file 9 | modify_config() 10 | { 11 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 12 | sed -i '2s/.*/collector.id=collector-'$NODE_NAME'/' $PARAFLOW_CONFIG_FILE 13 | else 14 | echo "no write permission on collector.conf file" 15 | exit 0 16 | fi 17 | } 18 | 19 | modify_config 20 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/utils/BytesUtils.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.utils; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class BytesUtils 9 | { 10 | private BytesUtils() 11 | { 12 | } 13 | 14 | public static byte[] toBytes(int v) 15 | { 16 | byte[] result = new byte[4]; 17 | result[0] = (byte) (v >> 24); 18 | result[1] = (byte) (v >> 16); 19 | result[2] = (byte) (v >> 8); 20 | result[3] = (byte) (v >> 0); 21 | return result; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Pipeline.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class Pipeline 9 | { 10 | private final String name; 11 | private float speed = 100.1f; 12 | 13 | public Pipeline( 14 | String name) 15 | { 16 | this.name = name; 17 | } 18 | 19 | public String getName() 20 | { 21 | return name; 22 | } 23 | 24 | public float getSpeed() 25 | { 26 | return speed; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/CollectorEnvironment.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | /** 4 | * collector environment 5 | * 6 | * @author guodong 7 | */ 8 | public class CollectorEnvironment 9 | { 10 | private CollectorEnvironment() 11 | { 12 | } 13 | 14 | public static CollectorEnvironment getEnvironment() 15 | { 16 | return new CollectorEnvironment(); 17 | } 18 | 19 | public DataFlow addSource(DataSource source) 20 | { 21 | return new FiberFlow<>(source.getName(), source); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/Column.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import io.airlift.tpch.TpchColumnType; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public interface Column 11 | { 12 | String TPCH_COLUMN_VALID_PREFIX_REGEX = "(?i)^(lo|c|n|r)_"; 13 | 14 | String getColumnName(); 15 | 16 | TpchColumnType getType(); 17 | 18 | default String getSimplifiedColumnName() 19 | { 20 | return getColumnName().replaceFirst(TPCH_COLUMN_VALID_PREFIX_REGEX, ""); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/DBSelectTemplate.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class DBSelectTemplate 9 | extends QueryTemplate 10 | { 11 | DBSelectTemplate(String table) 12 | { 13 | super(table); 14 | } 15 | 16 | @Override 17 | String makeQuery() 18 | { 19 | return null; 20 | } 21 | 22 | @Override 23 | QueryGenerator.QueryType getType() 24 | { 25 | return QueryGenerator.QueryType.SELECT; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/StringMessageSerializationSchema.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | 5 | /** 6 | * default message serialization schema 7 | * 8 | * @author guodong 9 | */ 10 | public class StringMessageSerializationSchema 11 | implements MessageSerializationSchema 12 | { 13 | private static final long serialVersionUID = 7172445062380483822L; 14 | 15 | @Override 16 | public Message serialize(int keyIdx, int timeIdx, T value) 17 | { 18 | return null; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/QueryGenerator.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public abstract class QueryGenerator 9 | { 10 | protected QueryDistribution distribution; 11 | 12 | public QueryGenerator(QueryDistribution distribution) 13 | { 14 | this.distribution = distribution; 15 | } 16 | 17 | public abstract boolean hasNext(); 18 | 19 | public abstract String next(); 20 | 21 | public enum QueryType 22 | { 23 | UPDATE, INSERT, DELETE, SELECT 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /paraflow-loader/src/test/java/cn/edu/ruc/iir/paraflow/loader/utils/SortTest.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader.utils; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.Arrays; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class SortTest 13 | { 14 | @Test 15 | public void testSort() 16 | { 17 | int[] content = new int[10]; 18 | for (int i = 0; i < 10; i++) { 19 | content[i] = 100 - i; 20 | } 21 | Arrays.sort(content); 22 | for (int i = 0; i < 10; i++) { 23 | System.out.println(content[i]); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | paraflow-1.0-alpha1/libs 4 | paraflow-1.0-alpha1/plugin 5 | paraflow-1.0-alpha1/bin/launcher* 6 | paraflow-1.0-alpha1/bin/procname 7 | paraflow-1.0-alpha1/logs 8 | _site 9 | paraflow-benchmark/data 10 | 11 | *.exp 12 | *~ 13 | 14 | # Compiled class file 15 | *.class 16 | 17 | # Log file 18 | *.log 19 | 20 | # BlueJ files 21 | *.ctxt 22 | 23 | # Mobile Tools for Java (J2ME) 24 | .mtj.tmp/ 25 | 26 | # Package Files # 27 | *.jar 28 | *.war 29 | *.ear 30 | *.zip 31 | *.tar.gz 32 | *.rar 33 | 34 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 35 | hs_err_pid* 36 | 37 | target 38 | 39 | .DS_Store -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Column.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | public class Column extends Base 4 | { 5 | private static final long serialVersionUID = 7838057854561622243L; 6 | private String name; 7 | private String type; 8 | 9 | public String getName() 10 | { 11 | return name; 12 | } 13 | 14 | public void setName(String name) 15 | { 16 | this.name = name; 17 | } 18 | 19 | public String getType() 20 | { 21 | return type; 22 | } 23 | 24 | public void setType(String type) 25 | { 26 | this.type = type; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /docs/stylesheets/reset.css: -------------------------------------------------------------------------------- 1 | /* MeyerWeb Reset */ 2 | 3 | html, body, div, span, applet, object, iframe, 4 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 5 | a, abbr, acronym, address, big, cite, code, 6 | del, dfn, em, img, ins, kbd, q, s, samp, 7 | small, strike, strong, sub, sup, tt, var, 8 | b, u, i, center, 9 | dl, dt, dd, ol, ul, li, 10 | fieldset, form, label, legend, 11 | table, caption, tbody, tfoot, thead, tr, th, td, 12 | article, aside, canvas, details, embed, 13 | figure, figcaption, footer, header, hgroup, 14 | menu, nav, output, ruby, section, summary, 15 | time, mark, audio, video { 16 | margin: 0; 17 | padding: 0; 18 | border: 0; 19 | font: inherit; 20 | vertical-align: baseline; 21 | } 22 | -------------------------------------------------------------------------------- /paraflow-collector/src/test/java/cn/edu/ruc/iir/paraflow/collector/TestKafkaBasic.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ConfigFileNotFoundException; 4 | import org.junit.Test; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class TestKafkaBasic 12 | { 13 | @Test 14 | public void createTopic() 15 | { 16 | try { 17 | DefaultCollector collector = new DefaultCollector<>(); 18 | collector.createTopic("exampledb-exampletbl", 10, (short) 1); 19 | } 20 | catch (ConfigFileNotFoundException e) { 21 | e.printStackTrace(); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/application-prod.properties: -------------------------------------------------------------------------------- 1 | server.port=2018 2 | # 设置IO线程数, 它主要执行非阻塞的任务,它们会负责多个连接, 默认设置每个CPU核心一个线程 3 | server.undertow.io-threads=4 4 | # 阻塞任务线程池, 当执行类似servlet请求阻塞操作, undertow会从这个线程池中取得线程,它的值设置取决于系统的负载 5 | server.undertow.worker-threads=60 6 | # 以下的配置会影响buffer,这些buffer会用于服务器连接的IO操作,有点类似netty的池化内存管理 7 | # 每块buffer的空间大小,越小的空间被利用越充分 8 | server.undertow.buffer-size=1024 9 | # 每个区分配的buffer数量 , 所以pool的大小是buffer-size * buffers-per-region 10 | server.undertow.buffers-per-region=1024 11 | # 是否分配的直接内存 12 | server.undertow.direct-buffers=true 13 | # 中文乱码 14 | spring.http.encoding.force=true 15 | spring.http.encoding.charset=UTF-8 16 | spring.http.encoding.enabled=true 17 | server.tomcat.uri-encoding=UTF-8 -------------------------------------------------------------------------------- /paraflow-assembly/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | paraflow 7 | cn.edu.ruc.iir 8 | 1.0-alpha1 9 | 10 | 4.0.0 11 | pom 12 | 13 | paraflow-assembly 14 | 15 | 16 | ${project.parent.basedir} 17 | 18 | 19 | -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/ParaflowSortedBuffer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.ParaflowRecord; 4 | 5 | public class ParaflowSortedBuffer 6 | { 7 | private final ParaflowRecord[] sortedRecords; 8 | private final int partition; 9 | 10 | public ParaflowSortedBuffer(ParaflowRecord[] sortedRecords, int partition) 11 | { 12 | this.sortedRecords = sortedRecords; 13 | this.partition = partition; 14 | } 15 | 16 | public ParaflowRecord[] getSortedRecords() 17 | { 18 | return sortedRecords; 19 | } 20 | 21 | public int getPartition() 22 | { 23 | return partition; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /paraflow-collector/src/test/java/cn/edu/ruc/iir/paraflow/collector/TestCollectorThroughputMetric.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.collector.utils.CollectorThroughputMetric; 4 | import org.junit.Test; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class TestCollectorThroughputMetric 12 | { 13 | @Test 14 | public void testMetric() 15 | throws InterruptedException 16 | { 17 | CollectorThroughputMetric metric = new CollectorThroughputMetric("collector0", "dbiir00:9101"); 18 | for (int i = 0; i < 10; i++) { 19 | metric.addValue(1000.0 * i); 20 | Thread.sleep(2000); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/server/StartupPipeline.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.server; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | public class StartupPipeline 9 | { 10 | private final List hooks; 11 | 12 | public StartupPipeline() 13 | { 14 | hooks = new ArrayList<>(); 15 | } 16 | 17 | public void addStartupHook(StartupHook hook) 18 | { 19 | hooks.add(hook); 20 | } 21 | 22 | public void startUp() throws ParaFlowException 23 | { 24 | for (StartupHook hook : hooks) { 25 | hook.run(); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/DataSource.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public abstract class DataSource 11 | { 12 | private final String name; 13 | 14 | public DataSource(String name) 15 | { 16 | this.name = name; 17 | } 18 | 19 | public abstract Message read(); 20 | 21 | public String getName() 22 | { 23 | return name; 24 | } 25 | 26 | @Override 27 | public abstract int hashCode(); 28 | 29 | @Override 30 | public abstract boolean equals(Object other); 31 | 32 | @Override 33 | public abstract String toString(); 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-metaserver-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | base_dir=$(dirname $0) 4 | 5 | if [ "x$PARAFLOW_HOME" = "x" ]; then 6 | export PARAFLOW_HOME="$base_dir/../" 7 | fi 8 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 9 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 10 | fi 11 | 12 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 13 | export PARAFLOW_HEAP_OPTS="-Xmx1G -Xms512M" 14 | fi 15 | 16 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowMetaServer'} 17 | 18 | COMMAND=$1 19 | case $COMMAND in 20 | -daemon) 21 | EXTRA_ARGS="-daemon "$EXTRA_ARGS 22 | shift 23 | ;; 24 | *) 25 | ;; 26 | esac 27 | 28 | exec $base_dir/paraflow-run-class.sh $EXTRA_ARGS cn.edu.ruc.iir.paraflow.metaserver.server.MetaServer 29 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-metaserver-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | base_dir=$(dirname $0) 4 | 5 | if [ "x$PARAFLOW_HOME" = "x" ]; then 6 | export PARAFLOW_HOME="$base_dir/../" 7 | fi 8 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 9 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 10 | fi 11 | 12 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 13 | export PARAFLOW_HEAP_OPTS="-Xmx1G -Xms512M" 14 | fi 15 | 16 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowMetaServer'} 17 | 18 | COMMAND=$1 19 | case $COMMAND in 20 | -daemon) 21 | EXTRA_ARGS="-daemon "$EXTRA_ARGS 22 | shift 23 | ;; 24 | *) 25 | ;; 26 | esac 27 | 28 | exec $base_dir/paraflow-run-class.sh $EXTRA_ARGS cn.edu.ruc.iir.paraflow.metaserver.server.MetaServer 29 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/DBDeleteTemplate.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class DBDeleteTemplate 9 | extends QueryTemplate 10 | { 11 | private static final int BASE = 10_000_000; 12 | private int counter = 0; 13 | 14 | DBDeleteTemplate(String table) 15 | { 16 | super(table); 17 | } 18 | 19 | @Override 20 | String makeQuery() 21 | { 22 | counter++; 23 | return "DELETE FROM " + table + " WHERE c_custkey=" + (BASE + counter) + ";"; 24 | } 25 | 26 | @Override 27 | QueryGenerator.QueryType getType() 28 | { 29 | return QueryGenerator.QueryType.DELETE; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /paraflow-loader/src/test/java/cn/edu/ruc/iir/paraflow/loader/utils/StringTest.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader.utils; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class StringTest 13 | { 14 | @Test 15 | public void testSubStr() 16 | { 17 | String segmentPath = "hdfs://127.0.0.1:9000/paraflow/db/table/file-0"; 18 | int fileNamePoint = segmentPath.lastIndexOf("/"); 19 | int tblPoint = segmentPath.lastIndexOf("/", fileNamePoint - 1); 20 | int dbPoint = segmentPath.lastIndexOf("/", tblPoint - 1); 21 | String suffix = segmentPath.substring(dbPoint + 1); 22 | assertEquals("db/table/file-0", suffix); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-metaserver-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | base_dir=$(dirname $0) 4 | 5 | if [ "x$PARAFLOW_HOME" = "x" ]; then 6 | export PARAFLOW_HOME="$base_dir/../" 7 | fi 8 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 9 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 10 | fi 11 | 12 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 13 | export PARAFLOW_HEAP_OPTS="-Xmx1G -Xms512M" 14 | fi 15 | 16 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowMetaServer'} 17 | 18 | COMMAND=$1 19 | case $COMMAND in 20 | -daemon) 21 | EXTRA_ARGS="-daemon "$EXTRA_ARGS 22 | shift 23 | ;; 24 | *) 25 | ;; 26 | esac 27 | 28 | exec $base_dir/paraflow-run-class.sh $EXTRA_ARGS cn.edu.ruc.iir.paraflow.metaserver.server.MetaServer 29 | -------------------------------------------------------------------------------- /paraflow-examples/src/main/java/cn/edu/ruc/iir/paraflow/examples/collector/BasicParaflowFiberPartitioner.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.examples.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.ParaflowFiberPartitioner; 4 | 5 | import java.nio.ByteBuffer; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class BasicParaflowFiberPartitioner 13 | implements ParaflowFiberPartitioner 14 | { 15 | private final int partitionNum; 16 | 17 | public BasicParaflowFiberPartitioner(int partitionNum) 18 | { 19 | this.partitionNum = partitionNum; 20 | } 21 | 22 | @Override 23 | public int getFiberId(byte[] key) 24 | { 25 | ByteBuffer buffer = ByteBuffer.wrap(key); 26 | return buffer.getInt() % partitionNum; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/ClusterInfo.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class ClusterInfo 9 | { 10 | private String version; 11 | private String uptime; 12 | 13 | public ClusterInfo() 14 | { 15 | this.version = "2.0-alpha"; 16 | this.uptime = "100h"; 17 | } 18 | 19 | public String getVersion() 20 | { 21 | return version; 22 | } 23 | 24 | public void setVersion(String version) 25 | { 26 | this.version = version; 27 | } 28 | 29 | public String getUptime() 30 | { 31 | return uptime; 32 | } 33 | 34 | public void setUptime(String uptime) 35 | { 36 | this.uptime = uptime; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/paraflow_scp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | 5 | if [ $1 = 'y' ]; then 6 | cd /home/tao/software/station/DBIIR/paraflow 7 | mvn package 8 | fi 9 | 10 | host="dbiir01" 11 | PRESTO_DIR=/home/iir/opt/presto-server-0.192/ 12 | 13 | ssh iir@$host "$PRESTO_DIR/sbin/stop-all.sh" 14 | 15 | scp -r /home/tao/software/station/DBIIR/paraflow/paraflow-connector/target/paraflow-connector-1.0-alpha1 iir@$host:/home/iir/opt/presto-server-0.192/plugin/ 16 | 17 | #ssh iir@$host "$PRESTO_DIR/sbin/mv-plugins.sh" 18 | 19 | echo "task is done." 20 | 21 | #/home/iir/opt/paraflow/run/exe_script.sh "tail -f /home/iir/opt/presto-server-0.192/data/var/log/server.log" 22 | 23 | #/home/iir/opt/paraflow/run/exe_script.sh "tail -10 ~/opt/presto-server-0.192/data/var/log/server.log" 24 | 25 | ssh iir@$host "$PRESTO_DIR/sbin/cmd.sh true" 26 | 27 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/GetMetaTablesAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 5 | import cn.edu.ruc.iir.paraflow.metaserver.utils.MetaConstants; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class GetMetaTablesAction extends Action 13 | { 14 | @Override 15 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 16 | { 17 | ActionResponse response = new ActionResponse(); 18 | response.setResponseResultList( 19 | connection.executeQuery(MetaConstants.getMetaTablesSql)); 20 | return response; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /paraflow-examples/src/test/java/cn/edu/ruc/iir/paraflow/examples/TestTpchDataSource.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.examples; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | import cn.edu.ruc.iir.paraflow.examples.collector.TpchDataSource; 5 | import org.testng.annotations.Test; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class TestTpchDataSource 13 | { 14 | @Test 15 | public void testTpchDataGeneration() 16 | { 17 | TpchDataSource dataSource = new TpchDataSource(1, 1, 1, 0, 1_000_000); 18 | int counter = 0; 19 | while (true) { 20 | Message message = dataSource.read(); 21 | if (message == null) { 22 | break; 23 | } 24 | counter++; 25 | } 26 | System.out.println(counter); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/paraflow-loader-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 5 | PARAFLOW_NODE_ID=$(($NODE_ID)) 6 | PARAFLOW_HOME="/home/iir/opt/paraflow" 7 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 8 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/loader.conf" 9 | 10 | # create a symbolic file and data directories 11 | create_dirs() 12 | { 13 | mkdir /dev/shm/paraflow 14 | cd $PARAFLOW_DIR/.. && ln -s $PARAFLOW_DIR paraflow && chown -h iir:iir paraflow 15 | } 16 | 17 | # modify the configuration file 18 | modify_config() 19 | { 20 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 21 | sed -i '2s/.*/loader.id='$PARAFLOW_NODE_ID'/' $PARAFLOW_CONFIG_FILE 22 | else 23 | echo "no write permission on loader.conf file" 24 | exit 0 25 | fi 26 | } 27 | 28 | create_dirs 29 | modify_config 30 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/ParaflowHttpServer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.ComponentScan; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | @SpringBootApplication 13 | @ComponentScan 14 | public class ParaflowHttpServer 15 | { 16 | public static void main(String[] args) 17 | { 18 | ParaflowHttpServer httpServer = new ParaflowHttpServer(); 19 | httpServer.init(); 20 | SpringApplication.run(ParaflowHttpServer.class, args); 21 | } 22 | 23 | private void init() 24 | { 25 | System.out.println("Paraflow http server is starting..."); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/zookeeper-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path="/data1/zookeeper" 4 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 5 | ZK_HOME="/home/iir/opt/zookeeper" 6 | ZK_CONFIG_FILE="$ZK_HOME/conf/zoo.cfg" 7 | NODE_NAME=$HOSTNAME 8 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 9 | ZK_NODE_ID=$(($NODE_ID - 2)) 10 | 11 | create_dirs() 12 | { 13 | if [ $(id -u) != "0" ]; then 14 | echo "please run $0 in root." 15 | exit 0 16 | fi 17 | cd $ZK_DIR/.. && ln -s $ZK_DIR zookeeper && chown -h iir:iir zookeeper 18 | mkdir $data_path && chown iir:iir $data_path 19 | } 20 | 21 | modify_config() 22 | { 23 | if [ -w $ZK_CONFIG_FILE ]; then 24 | sed -i '14s/.*/clientPort='$((2181 + $ZK_NODE_ID))'/' $ZK_CONFIG_FILE 25 | else 26 | echo "no write permission on zoo.cfg file" 27 | exit 0 28 | fi 29 | } 30 | 31 | create_dirs 32 | modify_config 33 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/paraflow-loader-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 5 | PARAFLOW_NODE_ID=$(($NODE_ID)) 6 | PARAFLOW_HOME="/home/iir/opt/paraflow" 7 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 8 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/loader.conf" 9 | 10 | # create a symbolic file and data directories 11 | create_dirs() 12 | { 13 | mkdir /dev/shm/paraflow 14 | cd $PARAFLOW_DIR/.. && ln -s $PARAFLOW_DIR paraflow && chown -h iir:iir paraflow 15 | } 16 | 17 | # modify the configuration file 18 | modify_config() 19 | { 20 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 21 | sed -i '2s/.*/loader.id='$PARAFLOW_NODE_ID'/' $PARAFLOW_CONFIG_FILE 22 | else 23 | echo "no write permission on loader.conf file" 24 | exit 0 25 | fi 26 | } 27 | 28 | create_dirs 29 | modify_config 30 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/connection/Connection.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.connection; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | 5 | public abstract class Connection 6 | { 7 | public abstract int executeUpdate(String statement) throws ParaFlowException; 8 | 9 | public abstract int[] executeUpdateInBatch(String[] statements) throws ParaFlowException; 10 | 11 | public abstract ResultList executeQuery(String statement) throws ParaFlowException; 12 | 13 | public abstract void setAutoCommit(boolean autoCommit) throws ParaFlowException; 14 | 15 | public abstract void commit() throws ParaFlowException; 16 | 17 | public abstract void rollback() throws ParaFlowException; 18 | 19 | public abstract void close() throws ParaFlowException; 20 | } 21 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/zookeeper-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path="/data1/zookeeper" 4 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 5 | ZK_HOME="/home/iir/opt/zookeeper" 6 | ZK_CONFIG_FILE="$ZK_HOME/conf/zoo.cfg" 7 | NODE_NAME=$HOSTNAME 8 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 9 | ZK_NODE_ID=$(($NODE_ID - 2)) 10 | 11 | create_dirs() 12 | { 13 | if [ $(id -u) != "0" ]; then 14 | echo "please run $0 in root." 15 | exit 0 16 | fi 17 | cd $ZK_DIR/.. && ln -s $ZK_DIR zookeeper && chown -h iir:iir zookeeper 18 | mkdir $data_path && chown iir:iir $data_path 19 | } 20 | 21 | modify_config() 22 | { 23 | if [ -w $ZK_CONFIG_FILE ]; then 24 | sed -i '14s/.*/clientPort='$((2181 + $ZK_NODE_ID))'/' $ZK_CONFIG_FILE 25 | else 26 | echo "no write permission on zoo.cfg file" 27 | exit 0 28 | fi 29 | } 30 | 31 | create_dirs 32 | modify_config 33 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/StorageFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector; 15 | 16 | /** 17 | * @author jelly.guodong.jin@gmail.com 18 | */ 19 | public enum StorageFormat 20 | { 21 | PARQUET, ORC, RCFILE 22 | } 23 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/paraflow-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 5 | PARAFLOW_NODE_ID=$(($NODE_ID)) 6 | PARAFLOW_HOME="/home/iir/opt/paraflow" 7 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 8 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/loader.conf" 9 | 10 | # create a symbolic file and data directories 11 | create_dirs() 12 | { 13 | mkdir /dev/shm/paraflow && chown -R iir:iir /dev/shm/paraflow 14 | cd $PARAFLOW_DIR/.. && ln -s $PARAFLOW_DIR paraflow && chown -h iir:iir paraflow 15 | } 16 | 17 | # modify the configuration file 18 | modify_config() 19 | { 20 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 21 | sed -i '2s/.*/loader.id='$PARAFLOW_NODE_ID'/' $PARAFLOW_CONFIG_FILE 22 | else 23 | echo "no write permission on loader.conf file" 24 | exit 0 25 | fi 26 | } 27 | 28 | create_dirs 29 | modify_config 30 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/sbin/paraflow-loader-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | NODE_NAME=$HOSTNAME 4 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 5 | PARAFLOW_NODE_ID=$(($NODE_ID)) 6 | PARAFLOW_HOME="/home/iir/opt/paraflow" 7 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 8 | PARAFLOW_CONFIG_FILE="$PARAFLOW_HOME/config/loader.conf" 9 | 10 | # create a symbolic file and data directories 11 | create_dirs() 12 | { 13 | mkdir /dev/shm/paraflow 14 | cd $PARAFLOW_DIR/.. && ln -s $PARAFLOW_DIR paraflow && chown -h iir:iir paraflow 15 | } 16 | 17 | # modify the configuration file 18 | modify_config() 19 | { 20 | if [ -w $PARAFLOW_CONFIG_FILE ]; then 21 | sed -i '2s/.*/loader.id='$PARAFLOW_NODE_ID'/' $PARAFLOW_CONFIG_FILE 22 | else 23 | echo "no write permission on loader.conf file" 24 | exit 0 25 | fi 26 | } 27 | 28 | create_dirs 29 | modify_config 30 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/DataSink.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public abstract class DataSink 9 | { 10 | private final String name; 11 | 12 | public DataSink(String name) 13 | { 14 | this.name = name; 15 | } 16 | 17 | public String getName() 18 | { 19 | return name; 20 | } 21 | 22 | public abstract String getDb(); 23 | 24 | public abstract String getTbl(); 25 | 26 | public abstract DataType getType(); 27 | 28 | @Override 29 | public abstract int hashCode(); 30 | 31 | @Override 32 | public abstract boolean equals(Object other); 33 | 34 | @Override 35 | public abstract String toString(); 36 | 37 | public enum DataType 38 | { 39 | Parquet, ORC; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/test/java/cn/edu/ruc/iir/paraflow/benchmark/QuestionerTest.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark; 2 | 3 | import cn.edu.ruc.iir.paraflow.benchmark.query.DBQuestioner; 4 | import cn.edu.ruc.iir.paraflow.benchmark.query.PrestoQuestioner; 5 | import org.junit.Test; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class QuestionerTest 13 | { 14 | @Test 15 | public void testDBQuestioner() 16 | { 17 | DBQuestioner questioner = new DBQuestioner("jdbc:postgresql://dbiir00:5432/tpch", "customer", "u"); 18 | questioner.question(); 19 | } 20 | 21 | @Test 22 | public void testPrestoQuestioner() 23 | { 24 | PrestoQuestioner questioner = new PrestoQuestioner("jdbc:presto://dbiir10:8080", "paraflow.test.tpch", "postgresql.public.customer"); 25 | questioner.question(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/sbin/zookeeper-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path="/data1/zookeeper" 4 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 5 | ZK_HOME="/home/iir/opt/zookeeper" 6 | ZK_CONFIG_FILE="$ZK_HOME/conf/zoo.cfg" 7 | NODE_NAME=$HOSTNAME 8 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 9 | ZK_NODE_ID=$(($NODE_ID - 2)) 10 | 11 | create_dirs() 12 | { 13 | if [ $(id -u) != "0" ]; then 14 | echo "please run $0 in root." 15 | exit 0 16 | fi 17 | cd $ZK_DIR/.. && ln -s $ZK_DIR zookeeper && chown -h iir:iir zookeeper 18 | mkdir $data_path && chown iir:iir $data_path 19 | } 20 | 21 | modify_config() 22 | { 23 | if [ -w $ZK_CONFIG_FILE ]; then 24 | sed -i '14s/.*/clientPort='$((2181 + $ZK_NODE_ID))'/' $ZK_CONFIG_FILE 25 | else 26 | echo "no write permission on zoo.cfg file" 27 | exit 0 28 | fi 29 | } 30 | 31 | create_dirs 32 | modify_config 33 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/test/java/cn/edu/ruc/iir/paraflow/benchmark/TimeGenerationPoolTest.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark; 2 | 3 | import cn.edu.ruc.iir.paraflow.benchmark.generator.TimeGenerationPool; 4 | import org.junit.Test; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class TimeGenerationPoolTest 12 | { 13 | @Test 14 | public void testTimeGeneration() 15 | { 16 | TimeGenerationPool timeGenerationPool = TimeGenerationPool.INSTANCE(); 17 | timeGenerationPool.init(System.currentTimeMillis() - (7 * 24 * 60 * 60 * 1000), 1, 80000); 18 | long initTime = 0; 19 | for (int i = 0; i < 800000; i++) { 20 | long time = timeGenerationPool.nextTime(); 21 | if (time != initTime) { 22 | System.out.println(time); 23 | initTime = time; 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-loader-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 4 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db table partitionFrom partitionTo" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.loader.BasicLoader" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx16G -Xms8G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowLoader'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-loader-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 4 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db table partitionFrom partitionTo" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.loader.BasicLoader" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx16G -Xms8G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowLoader'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/tpl/tplQuery.htm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | <% 5 | for( var j=0; j< tplData.column.length; j++){ 6 | var aColumn = tplData.column[j]; 7 | %> 8 | 9 | <%} %> 10 | 11 | 12 | 13 | <% 14 | for( var i=0;i< tplData.data.length;i++){ 15 | var aItem = tplData.data[i]; 16 | %> 17 | 18 | <% 19 | for (x in aItem) { 20 | %> 21 | 24 | <%} %> 25 | 26 | <%} %> 27 | <% if ( tplData.length<=0){ %> 28 | 29 | 32 | 33 | <%} %> 34 |
<%=aColumn %>
22 | <%=aItem[x] %> 23 |
30 | no result. 31 |
-------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-loader-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 4 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db table partitionFrom partitionTo" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.loader.BasicLoader" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx16G -Xms8G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowLoader'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-examples/src/main/java/cn/edu/ruc/iir/paraflow/examples/loader/BasicLoader.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.examples.loader; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ConfigFileNotFoundException; 4 | import cn.edu.ruc.iir.paraflow.loader.DefaultLoader; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class BasicLoader 12 | { 13 | private BasicLoader() 14 | { 15 | } 16 | 17 | public static void main(String[] args) 18 | { 19 | String db = args[0]; 20 | String table = args[1]; 21 | int partitionFrom = Integer.parseInt(args[2]); 22 | int partitionTo = Integer.parseInt(args[3]); 23 | 24 | try { 25 | DefaultLoader loader = new DefaultLoader(db, table, partitionFrom, partitionTo); 26 | loader.run(); 27 | } 28 | catch (ConfigFileNotFoundException e) { 29 | e.printStackTrace(); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/connection/ResultList.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.connection; 2 | 3 | import java.util.LinkedList; 4 | 5 | /** 6 | * paraflow 7 | */ 8 | public class ResultList 9 | { 10 | private LinkedList jdbcRecords; 11 | 12 | public ResultList() 13 | { 14 | jdbcRecords = new LinkedList<>(); 15 | } 16 | 17 | public JDBCRecord get(int index) 18 | { 19 | if (index < jdbcRecords.size()) { 20 | return jdbcRecords.get(index); 21 | } 22 | else { 23 | return null; 24 | } 25 | } 26 | 27 | public void add(JDBCRecord jdbcRecord) 28 | { 29 | jdbcRecords.add(jdbcRecord); 30 | } 31 | 32 | public int size() 33 | { 34 | return jdbcRecords.size(); 35 | } 36 | 37 | public boolean isEmpty() 38 | { 39 | return jdbcRecords.isEmpty(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/ParaflowThreadFactory.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | import java.util.concurrent.Executors; 4 | import java.util.concurrent.ThreadFactory; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class ParaflowThreadFactory 12 | implements ThreadFactory 13 | { 14 | private static final ThreadFactory defaultTF = Executors.defaultThreadFactory(); 15 | private final Thread.UncaughtExceptionHandler uncaughtExceptionHandler; 16 | 17 | public ParaflowThreadFactory(Thread.UncaughtExceptionHandler uncaughtExceptionHandler) 18 | { 19 | this.uncaughtExceptionHandler = uncaughtExceptionHandler; 20 | } 21 | 22 | @Override 23 | public Thread newThread(Runnable runnable) 24 | { 25 | Thread thread = defaultTF.newThread(runnable); 26 | thread.setUncaughtExceptionHandler(uncaughtExceptionHandler); 27 | return thread; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/bin/paraflow-collector-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 9 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db_name table_name parallelism partition_num sf part part_count min_custkey max_custkey" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.collector.BasicCollector" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx8G -Xms6G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowCollector'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/ParaflowPartitioningHandle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector; 15 | 16 | import com.facebook.presto.spi.connector.ConnectorPartitioningHandle; 17 | 18 | /** 19 | * @author jelly.guodong.jin@gmail.com 20 | */ 21 | public class ParaflowPartitioningHandle 22 | implements ConnectorPartitioningHandle 23 | { 24 | } 25 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/bin/paraflow-collector-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 9 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db_name table_name parallelism partition_num sf part part_count min_custkey max_custkey" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.collector.BasicCollector" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx8G -Xms6G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowCollector'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/ParaflowRecord.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public abstract class ParaflowRecord 9 | { 10 | private long key; 11 | private long timestamp; 12 | private int fiberId; 13 | 14 | public ParaflowRecord() 15 | { 16 | } 17 | 18 | public long getKey() 19 | { 20 | return key; 21 | } 22 | 23 | public void setKey(long key) 24 | { 25 | this.key = key; 26 | } 27 | 28 | public long getTimestamp() 29 | { 30 | return timestamp; 31 | } 32 | 33 | public void setTimestamp(long timestamp) 34 | { 35 | this.timestamp = timestamp; 36 | } 37 | 38 | public int getFiberId() 39 | { 40 | return fiberId; 41 | } 42 | 43 | public void setFiberId(int fiberId) 44 | { 45 | this.fiberId = fiberId; 46 | } 47 | 48 | public abstract Object getValue(int idx); 49 | } 50 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Base.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @version V1.0 7 | * @Package: cn.edu.ruc.iir.pixels.common.metadata.domain 8 | * @ClassName: Base 9 | * @Description: 10 | * @author: tao 11 | * @date: Create in 2018-09-18 15:07 12 | **/ 13 | public class Base implements Serializable 14 | { 15 | private static final long serialVersionUID = -7598432013090890964L; 16 | private int id; 17 | 18 | public int getId() 19 | { 20 | return id; 21 | } 22 | 23 | public void setId(int id) 24 | { 25 | this.id = id; 26 | } 27 | 28 | @Override 29 | public int hashCode() 30 | { 31 | return this.id; 32 | } 33 | 34 | @Override 35 | public boolean equals(Object o) 36 | { 37 | if (o instanceof Base) { 38 | return this.id == ((Base) o).id; 39 | } 40 | return false; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/bin/paraflow-collector-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 9 ]; 4 | then 5 | echo "USAGE: $0 [-daemon] db_name table_name parallelism partition_num sf part part_count min_custkey max_custkey" 6 | exit 1 7 | fi 8 | base_dir=$(dirname $0) 9 | 10 | MAIN_CLASS="cn.edu.ruc.iir.paraflow.examples.collector.BasicCollector" 11 | 12 | if [ "xPARAFLOW_HOME" = "x" ]; then 13 | export PARAFLOW_HOME="$base_dir/../" 14 | fi 15 | if [ "x$PARAFLOW_LOG4J_OPTS" = "x" ]; then 16 | export PARAFLOW_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" 17 | fi 18 | 19 | if [ "x$PARAFLOW_HEAP_OPTS" = "x" ]; then 20 | export PARAFLOW_HEAP_OPTS="-Xmx8G -Xms6G" 21 | fi 22 | 23 | EXTRA_ARGS=${EXTRA_ARGS-'-name ParaflowCollector'} 24 | 25 | COMMAND=$1 26 | case ${COMMAND} in 27 | -daemon) 28 | EXTRA_ARGS="-daemon "${EXTRA_ARGS} 29 | shift 30 | ;; 31 | *) 32 | ;; 33 | esac 34 | 35 | exec ${base_dir}/paraflow-run-class.sh ${EXTRA_ARGS} ${MAIN_CLASS} "$@" 36 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/UpdateBlockPathException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class UpdateBlockPathException extends ParaFlowException 11 | { 12 | @Override 13 | public String getMessage() 14 | { 15 | return "update block path exception"; 16 | } 17 | 18 | @Override 19 | public StatusProto.ResponseStatus getResponseStatus() 20 | { 21 | return StatusProto.ResponseStatus.newBuilder() 22 | .setStatus(StatusProto.ResponseStatus.State.BLOCK_PATH_UPDATE_ERROR) 23 | .build(); 24 | } 25 | 26 | @Override 27 | public String getHint() 28 | { 29 | return "Check and try again"; 30 | } 31 | 32 | @Override 33 | public ParaFlowExceptionLevel getLevel() 34 | { 35 | return ParaFlowExceptionLevel.ERROR; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/TblFuncCreationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class TblFuncCreationException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = 2586643482592134064L; 8 | 9 | @Override 10 | public String getMessage() 11 | { 12 | return "Table function creation exception"; 13 | } 14 | 15 | @Override 16 | public StatusProto.ResponseStatus getResponseStatus() 17 | { 18 | return StatusProto.ResponseStatus.newBuilder() 19 | .setStatus(StatusProto.ResponseStatus.State.TBL_FUNC_CREATION_ERROR) 20 | .build(); 21 | } 22 | 23 | @Override 24 | public String getHint() 25 | { 26 | return "Check and try again"; 27 | } 28 | 29 | @Override 30 | public ParaFlowExceptionLevel getLevel() 31 | { 32 | return ParaFlowExceptionLevel.ERROR; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-examples/src/test/java/cn/edu/ruc/iir/paraflow/examples/TestSerialization.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.examples; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | import cn.edu.ruc.iir.paraflow.commons.ParaflowRecord; 5 | import cn.edu.ruc.iir.paraflow.examples.collector.TpchDataSource; 6 | import cn.edu.ruc.iir.paraflow.examples.loader.TpchDataTransformer; 7 | import org.testng.annotations.Test; 8 | 9 | /** 10 | * paraflow 11 | * 12 | * @author guodong 13 | */ 14 | public class TestSerialization 15 | { 16 | @Test 17 | public void testSerDe() 18 | { 19 | TpchDataSource dataSource = new TpchDataSource(1, 1, 1, 0, 1_000_000); 20 | TpchDataTransformer transformer = new TpchDataTransformer(); 21 | 22 | Message message = dataSource.read(); 23 | if (message != null) { 24 | byte[] serialized = message.getValue(); 25 | ParaflowRecord record = transformer.transform(serialized, 0); 26 | assert message.getTimestamp() == record.getTimestamp(); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/logclean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PREFIX="dbiir" 3 | START=2 4 | END=9 5 | PARAFLOW_HOME="/home/iir/opt/paraflow" 6 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 7 | 8 | # deployment 9 | deploy() 10 | { 11 | if [ $(id -u) != "0" ]; then 12 | echo "please run $0 $1 in root." 13 | exit 0 14 | fi 15 | 16 | for ((i=$START; i<=$END; i++)) 17 | do 18 | if [ $i -lt 10 ]; then 19 | echo "init the logclc.sh on dbiir0$i" 20 | scp $PARAFLOW_DIR"/sbin/paraflow_logclean.sh" $PREFIX"0"$i:$PARAFLOW_DIR"/sbin/" 21 | ssh $PREFIX"0"$i "chown -R iir:iir $PARAFLOW_DIR" 22 | fi 23 | done 24 | } 25 | 26 | # start 27 | startup() 28 | { 29 | for ((i=$START; i<=$END; i++)) 30 | do 31 | ssh $PREFIX"0"$i "$PARAFLOW_DIR/sbin/paraflow_logclean.sh" 32 | echo "0$i clean have finished,content: the log produced by paraflow" 33 | done 34 | } 35 | 36 | 37 | 38 | if [ "$1" == "deploy" ]; then 39 | deploy 40 | elif [ "$1" == "start" ]; then 41 | startup 42 | 43 | # nothing 44 | else 45 | echo "Usage: $0 deploy|start" 46 | fi 47 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/utils/ColType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.metaserver.utils; 15 | 16 | public enum ColType 17 | { 18 | REGULAR(0), 19 | FIBER(1), 20 | TIMESTAMP(2); 21 | 22 | private int colTypeId; 23 | 24 | ColType(int colTypeId) 25 | { 26 | this.colTypeId = colTypeId; 27 | } 28 | 29 | public int getColTypeId() 30 | { 31 | return colTypeId; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/configuration/CORSConfiguration.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.configuration; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.servlet.config.annotation.CorsRegistry; 6 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 7 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; 8 | 9 | @Configuration 10 | public class CORSConfiguration 11 | { 12 | @Bean 13 | public WebMvcConfigurer corsConfigurer() 14 | { 15 | return new WebMvcConfigurerAdapter() 16 | { 17 | @Override 18 | public void addCorsMappings(CorsRegistry registry) 19 | { 20 | registry.addMapping("/**") 21 | .allowedHeaders("*") 22 | .allowedMethods("*") 23 | .allowedOrigins("*"); 24 | } 25 | }; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/ColumnsNotExistException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class ColumnsNotExistException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = 5600165987123351243L; 8 | 9 | private final String dbName; 10 | private final String tblName; 11 | 12 | public ColumnsNotExistException(String dbName, String tblName) 13 | { 14 | this.dbName = dbName; 15 | this.tblName = tblName; 16 | } 17 | 18 | @Override 19 | public String getMessage() 20 | { 21 | return null; 22 | } 23 | 24 | @Override 25 | public StatusProto.ResponseStatus getResponseStatus() 26 | { 27 | return null; 28 | } 29 | 30 | @Override 31 | public String getHint() 32 | { 33 | return null; 34 | } 35 | 36 | @Override 37 | public ParaFlowExceptionLevel getLevel() 38 | { 39 | return null; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/utils/DefaultKafkaPartitioner.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector.utils; 2 | 3 | import org.apache.kafka.common.Cluster; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class DefaultKafkaPartitioner extends KafkaPartitioner 11 | { 12 | /** 13 | * Compute the partition for the given record. 14 | * 15 | * @param topic The topic name 16 | * @param key The key to partition on (or null if no key) 17 | * @param keyBytes The serialized key to partition on( or null if no key) 18 | * @param value The value to partition on or null 19 | * @param valueBytes The serialized value to partition on or null 20 | * @param cluster The current cluster metadata 21 | */ 22 | @Override 23 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) 24 | { 25 | int partitionNum = cluster.partitionCountForTopic(topic); 26 | return (int) key % partitionNum; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/RegionColumn.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import io.airlift.tpch.TpchColumnType; 4 | 5 | import static io.airlift.tpch.TpchColumnTypes.IDENTIFIER; 6 | import static io.airlift.tpch.TpchColumnTypes.varchar; 7 | 8 | /** 9 | * paraflow 10 | * 11 | * @author guodong 12 | */ 13 | public enum RegionColumn 14 | implements Column 15 | { 16 | @SuppressWarnings("SpellCheckingInspection") 17 | REGION_KEY("r_regionkey", IDENTIFIER), 18 | NAME("r_name", varchar(25)), 19 | COMMENT("r_comment", varchar(152)); 20 | 21 | private final String columnName; 22 | private final TpchColumnType type; 23 | 24 | RegionColumn(String columnName, TpchColumnType type) 25 | { 26 | this.columnName = columnName; 27 | this.type = type; 28 | } 29 | 30 | @Override 31 | public String getColumnName() 32 | { 33 | return columnName; 34 | } 35 | 36 | @Override 37 | public TpchColumnType getType() 38 | { 39 | return type; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/ParaflowCursorException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.exception; 15 | 16 | import com.facebook.presto.spi.PrestoException; 17 | 18 | /** 19 | * @author jelly.guodong.jin@gmail.com 20 | */ 21 | public class ParaflowCursorException 22 | extends PrestoException 23 | { 24 | public ParaflowCursorException() 25 | { 26 | super(ParaflowErrorCode.HDFS_CURSOR_ERROR, "HDFS cursor error"); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/Processor.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | import java.util.concurrent.atomic.AtomicBoolean; 4 | 5 | public abstract class Processor 6 | { 7 | final String name; 8 | final String db; 9 | final String table; 10 | final AtomicBoolean isReadyToStop = new AtomicBoolean(false); 11 | private final int parallelism; 12 | 13 | public Processor(String name, String db, String table, int parallelism) 14 | { 15 | this.name = name; 16 | this.db = db; 17 | this.table = table; 18 | this.parallelism = parallelism; 19 | } 20 | 21 | public abstract void run(); 22 | 23 | public String getName() 24 | { 25 | return name; 26 | } 27 | 28 | public String getDb() 29 | { 30 | return db; 31 | } 32 | 33 | public String getTable() 34 | { 35 | return table; 36 | } 37 | 38 | public int getParallelism() 39 | { 40 | return parallelism; 41 | } 42 | 43 | public void stop() 44 | { 45 | isReadyToStop.set(true); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/utils/MetaVersion.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.metaserver.utils; 15 | 16 | /** 17 | * @author jelly.guodong.jin@gmail.com 18 | */ 19 | public enum MetaVersion 20 | { 21 | ONE_ALPHA_ONE("1.0-alpha1"); 22 | 23 | private String versionId; 24 | 25 | MetaVersion(String versionId) 26 | { 27 | this.versionId = versionId; 28 | } 29 | 30 | public String getVersionId() 31 | { 32 | return versionId; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/TableNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.exception; 15 | 16 | import com.facebook.presto.spi.PrestoException; 17 | 18 | /** 19 | * presto-root 20 | * 21 | * @author Jelly 22 | */ 23 | public class TableNotFoundException 24 | extends PrestoException 25 | { 26 | public TableNotFoundException(String tableName) 27 | { 28 | super(ParaflowErrorCode.TABLE_NOT_FOUND, "Table " + tableName + " not found."); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/clear.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | PREFIX="dbiir" 3 | START=2 4 | END=9 5 | PARAFLOW_HOME="/home/iir/opt/paraflow" 6 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 7 | 8 | # deployment 9 | deploy() 10 | { 11 | if [ $(id -u) != "0" ]; then 12 | echo "please run $0 $1 in root." 13 | exit 0 14 | fi 15 | 16 | for ((i=$START; i<=$END; i++)) 17 | do 18 | if [ $i -lt 10 ]; then 19 | echo "init the logclc.sh on dbiir0$i" 20 | scp $PARAFLOW_DIR"/sbin/paraflow_logclean.sh" $PREFIX"0"$i:$PARAFLOW_DIR"/sbin/" 21 | ssh $PREFIX"0"$i "chown -R iir:iir $PARAFLOW_DIR" 22 | fi 23 | done 24 | } 25 | 26 | # start 27 | startup() 28 | { 29 | for ((i=$START; i<=$END; i++)) 30 | do 31 | ssh $PREFIX"0"$i "$PARAFLOW_DIR/sbin/paraflow_logclean.sh" 32 | echo "0$i clean have finished,content: the log produced by paraflow" 33 | done 34 | } 35 | 36 | 37 | 38 | if [ "$1" == "deploy" ]; then 39 | deploy 40 | elif [ "$1" == "start" ]; then 41 | startup 42 | 43 | # nothing 44 | else 45 | echo "Usage: $0 deploy|start" 46 | echo "run function: deploy & start" 47 | deploy 48 | startup 49 | echo "clear log ok" 50 | fi -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/UnSupportedFunctionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.exception; 15 | 16 | import com.facebook.presto.spi.PrestoException; 17 | 18 | /** 19 | * presto-root 20 | * 21 | * @author Jelly 22 | */ 23 | public class UnSupportedFunctionException extends PrestoException 24 | { 25 | public UnSupportedFunctionException(String function) 26 | { 27 | super(ParaflowErrorCode.FUNCTION_UNSUPPORTED, "Function " + function + " not exists."); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Table.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | import java.util.List; 4 | 5 | public class Table extends Base 6 | { 7 | private static final long serialVersionUID = 8083493591644033068L; 8 | private String name; 9 | private List columns; 10 | 11 | public Table() 12 | { 13 | } 14 | 15 | public Table(String name, List columns) 16 | { 17 | this.name = name; 18 | this.columns = columns; 19 | } 20 | 21 | public String getName() 22 | { 23 | return name; 24 | } 25 | 26 | public void setName(String name) 27 | { 28 | this.name = name; 29 | } 30 | 31 | public List getColumns() 32 | { 33 | return columns; 34 | } 35 | 36 | public void setColumns(List columns) 37 | { 38 | this.columns = columns; 39 | } 40 | 41 | @Override 42 | public String toString() 43 | { 44 | return "Table{" + 45 | "name='" + name + '\'' + 46 | ", columns=" + columns + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/ParaflowSplitNotOpenException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.exception; 15 | 16 | import com.facebook.presto.spi.PrestoException; 17 | import org.apache.hadoop.fs.Path; 18 | 19 | /** 20 | * @author jelly.guodong.jin@gmail.com 21 | */ 22 | public class ParaflowSplitNotOpenException 23 | extends PrestoException 24 | { 25 | public ParaflowSplitNotOpenException(Path path) 26 | { 27 | super(ParaflowErrorCode.HDFS_SPLIT_NOT_OPEN, "HDFS Split " + path + " cannot open"); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/NationColumn.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import io.airlift.tpch.TpchColumnType; 4 | 5 | import static io.airlift.tpch.TpchColumnTypes.IDENTIFIER; 6 | import static io.airlift.tpch.TpchColumnTypes.varchar; 7 | 8 | /** 9 | * paraflow 10 | * 11 | * @author guodong 12 | */ 13 | public enum NationColumn 14 | implements Column 15 | { 16 | @SuppressWarnings("SpellCheckingInspection") 17 | NATION_KEY("n_nationkey", IDENTIFIER), 18 | NAME("n_name", varchar(25)), 19 | @SuppressWarnings("SpellCheckingInspection") 20 | REGION_KEY("n_regionkey", IDENTIFIER), 21 | COMMENT("n_comment", varchar(152)); 22 | 23 | private final String columnName; 24 | private final TpchColumnType type; 25 | 26 | NationColumn(String columnName, TpchColumnType type) 27 | { 28 | this.columnName = columnName; 29 | this.type = type; 30 | } 31 | 32 | @Override 33 | public String getColumnName() 34 | { 35 | return columnName; 36 | } 37 | 38 | @Override 39 | public TpchColumnType getType() 40 | { 41 | return type; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/kafka-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path=( "/data1/kafka" "/data1/kafka/logs" "/data2/kafka" "/data2/kafka/logs" ) 4 | NODE_NAME=$HOSTNAME 5 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 6 | KAFKA_NODE_ID=$(($NODE_ID - 2)) 7 | KAFKA_HOME="/home/iir/opt/kafka" 8 | KAFKA_DIR="/home/iir/opt/kafka_2.11-1.1.1" 9 | KAFKA_CONFIG_FILE="$KAFKA_HOME/config/server.properties" 10 | 11 | # create a symbolic file and data directories 12 | create_dirs() 13 | { 14 | if [ $(id -u) != "0" ]; then 15 | echo "please run $0 in root." 16 | exit 0 17 | fi 18 | cd $KAFKA_DIR/.. && ln -s $KAFKA_DIR kafka && chown -h iir:iir kafka 19 | for path in "${data_path[@]}"; do 20 | if [ ! -e $path ]; then 21 | mkdir $path 22 | chown iir:iir $path 23 | fi 24 | done 25 | } 26 | 27 | # modify the configuration file 28 | modify_config() 29 | { 30 | if [ -w $KAFKA_CONFIG_FILE ]; then 31 | sed -i '21s/.*/broker.id='$KAFKA_NODE_ID'/' $KAFKA_CONFIG_FILE 32 | sed -i '31s/.*/listeners=PLAINTEXT:\/\/:'$((9192 + $KAFKA_NODE_ID))'/' $KAFKA_CONFIG_FILE 33 | else 34 | echo "no write permission on server.properties file" 35 | exit 0 36 | fi 37 | } 38 | 39 | create_dirs 40 | modify_config 41 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/utils/Utils.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.utils; 2 | 3 | /** 4 | * paraflow 5 | * 6 | * @author guodong 7 | */ 8 | public class Utils 9 | { 10 | private Utils() 11 | { 12 | } 13 | 14 | public static String formatDbUrl(String dbName) 15 | { 16 | String base = MetaConfig.INSTANCE().getHDFSWarehouse(); 17 | if (!base.endsWith("/")) { 18 | base = base + "/"; 19 | } 20 | if (dbName.startsWith("/")) { 21 | dbName = dbName.substring(1, dbName.length()); 22 | } 23 | return base + dbName; 24 | } 25 | 26 | public static String formatTblUrl(String dbName, String tblName) 27 | { 28 | String base = MetaConfig.INSTANCE().getHDFSWarehouse(); 29 | if (!base.endsWith("/")) { 30 | base = base + "/"; 31 | } 32 | if (dbName.startsWith("/")) { 33 | dbName = dbName.substring(1, dbName.length()); 34 | } 35 | if (tblName.startsWith("/")) { 36 | tblName = tblName.substring(1, tblName.length()); 37 | } 38 | return base + dbName + "/" + tblName; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/kafka-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path=( "/data1/kafka" "/data1/kafka/logs" "/data2/kafka" "/data2/kafka/logs" ) 4 | NODE_NAME=$HOSTNAME 5 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 6 | KAFKA_NODE_ID=$(($NODE_ID - 2)) 7 | KAFKA_HOME="/home/iir/opt/kafka" 8 | KAFKA_DIR="/home/iir/opt/kafka_2.11-1.1.1" 9 | KAFKA_CONFIG_FILE="$KAFKA_HOME/config/server.properties" 10 | 11 | # create a symbolic file and data directories 12 | create_dirs() 13 | { 14 | if [ $(id -u) != "0" ]; then 15 | echo "please run $0 in root." 16 | exit 0 17 | fi 18 | cd $KAFKA_DIR/.. && ln -s $KAFKA_DIR kafka && chown -h iir:iir kafka 19 | for path in "${data_path[@]}"; do 20 | if [ ! -e $path ]; then 21 | mkdir $path 22 | chown iir:iir $path 23 | fi 24 | done 25 | } 26 | 27 | # modify the configuration file 28 | modify_config() 29 | { 30 | if [ -w $KAFKA_CONFIG_FILE ]; then 31 | sed -i '21s/.*/broker.id='$KAFKA_NODE_ID'/' $KAFKA_CONFIG_FILE 32 | sed -i '31s/.*/listeners=PLAINTEXT:\/\/:'$((9192 + $KAFKA_NODE_ID))'/' $KAFKA_CONFIG_FILE 33 | else 34 | echo "no write permission on server.properties file" 35 | exit 0 36 | fi 37 | } 38 | 39 | create_dirs 40 | modify_config 41 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/Metric.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | import io.prometheus.client.CollectorRegistry; 4 | import io.prometheus.client.Gauge; 5 | import io.prometheus.client.exporter.PushGateway; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * paraflow 11 | * 12 | * @author guodong 13 | */ 14 | public class Metric 15 | { 16 | private final String id; 17 | private final PushGateway gateway; 18 | private final Gauge metric; 19 | private final String job; 20 | private final CollectorRegistry registry = new CollectorRegistry(); 21 | 22 | public Metric(String gateWayUrl, String id, String name, String help, String job) 23 | { 24 | this.id = id; 25 | this.job = job; 26 | this.gateway = new PushGateway(gateWayUrl); 27 | this.metric = Gauge.build().name(name).help(help).labelNames("id").register(registry); 28 | } 29 | 30 | public void addValue(double value) 31 | { 32 | metric.labels(id).set(value); 33 | try { 34 | gateway.pushAdd(registry, job); 35 | } 36 | catch (IOException e) { 37 | e.printStackTrace(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/sbin/kafka-init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | data_path=( "/data1/kafka" "/data1/kafka/logs" "/data2/kafka" "/data2/kafka/logs" ) 4 | NODE_NAME=$HOSTNAME 5 | NODE_ID=${NODE_NAME:$((${#NODE_NAME}-1)):1} 6 | KAFKA_NODE_ID=$(($NODE_ID - 2)) 7 | KAFKA_HOME="/home/iir/opt/kafka" 8 | KAFKA_DIR="/home/iir/opt/kafka_2.11-1.1.1" 9 | KAFKA_CONFIG_FILE="$KAFKA_HOME/config/server.properties" 10 | 11 | # create a symbolic file and data directories 12 | create_dirs() 13 | { 14 | if [ $(id -u) != "0" ]; then 15 | echo "please run $0 in root." 16 | exit 0 17 | fi 18 | cd $KAFKA_DIR/.. && ln -s $KAFKA_DIR kafka && chown -h iir:iir kafka 19 | for path in "${data_path[@]}"; do 20 | if [ ! -e $path ]; then 21 | mkdir $path 22 | chown iir:iir $path 23 | fi 24 | done 25 | } 26 | 27 | # modify the configuration file 28 | modify_config() 29 | { 30 | if [ -w $KAFKA_CONFIG_FILE ]; then 31 | sed -i '21s/.*/broker.id='$KAFKA_NODE_ID'/' $KAFKA_CONFIG_FILE 32 | sed -i '31s/.*/listeners=PLAINTEXT:\/\/:'$((9192 + $KAFKA_NODE_ID))'/' $KAFKA_CONFIG_FILE 33 | else 34 | echo "no write permission on server.properties file" 35 | exit 0 36 | fi 37 | } 38 | 39 | create_dirs 40 | modify_config 41 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/utils/DateUtil.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.utils; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | 8 | /** 9 | * @version V1.0 10 | * @Package: cn.edu.ruc.iir.rainbow.benchmark 11 | * @ClassName: DateUtil 12 | * @Description: To sum some static useful functions 13 | * @author: Tao 14 | * @date: Create in 2017-07-28 7:11 15 | **/ 16 | public class DateUtil 17 | { 18 | private static AtomicInteger count = new AtomicInteger(0); 19 | 20 | public static String formatTime(Long time) 21 | { 22 | 23 | DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 24 | return format.format(new Date(time)); 25 | } 26 | 27 | public static String formatTime(Date time) 28 | { 29 | 30 | DateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 31 | return format.format(time); 32 | } 33 | 34 | public static String getCurTime() 35 | { 36 | SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");//set the style 37 | return df.format(new Date()) + count.getAndIncrement(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/Json.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class Json implements java.io.Serializable 7 | { 8 | private static final long serialVersionUID = 1L; 9 | 10 | private int state = 0; // 是否成功 11 | private Object datas = ""; // 其他信息 12 | private String msg = ""; // 其他信息 13 | private List errorList = new ArrayList(); // 提示信息 14 | 15 | public int getState() 16 | { 17 | return state; 18 | } 19 | 20 | public void setState(int state) 21 | { 22 | this.state = state; 23 | } 24 | 25 | public Object getDatas() 26 | { 27 | return datas; 28 | } 29 | 30 | public void setDatas(Object datas) 31 | { 32 | this.datas = datas; 33 | } 34 | 35 | public String getMsg() 36 | { 37 | return msg; 38 | } 39 | 40 | public void setMsg(String msg) 41 | { 42 | this.msg = msg; 43 | } 44 | 45 | public List getErrorList() 46 | { 47 | return errorList; 48 | } 49 | 50 | public void setErrorList(List errorList) 51 | { 52 | this.errorList = errorList; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/connection/JDBCRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.metaserver.connection; 15 | 16 | /** 17 | * 18 | */ 19 | public class JDBCRecord 20 | { 21 | private String[] stringList; 22 | 23 | public JDBCRecord(int size) 24 | { 25 | stringList = new String[size]; 26 | } 27 | 28 | public String get(int index) 29 | { 30 | if (index < stringList.length) { 31 | return stringList[index]; 32 | } 33 | else { 34 | return null; 35 | } 36 | } 37 | 38 | public void put(String string, int index) 39 | { 40 | stringList[index] = string; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/TopicFiber.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class TopicFiber 11 | { 12 | private final String topic; 13 | private final int fiber; 14 | 15 | public TopicFiber(String topic, int fiber) 16 | { 17 | this.topic = topic; 18 | this.fiber = fiber; 19 | } 20 | 21 | public String getTopic() 22 | { 23 | return topic; 24 | } 25 | 26 | public int getFiber() 27 | { 28 | return fiber; 29 | } 30 | 31 | @Override 32 | public int hashCode() 33 | { 34 | return Objects.hash(topic, fiber); 35 | } 36 | 37 | @Override 38 | public boolean equals(Object obj) 39 | { 40 | if (obj == this) { 41 | return true; 42 | } 43 | if (obj instanceof TopicFiber) { 44 | TopicFiber other = (TopicFiber) obj; 45 | return Objects.equals(topic, other.topic) 46 | && Objects.equals(fiber, other.fiber); 47 | } 48 | return false; 49 | } 50 | 51 | @Override 52 | public String toString() 53 | { 54 | return String.format("%s-%d", topic, fiber); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /paraflow-examples/src/main/java/cn/edu/ruc/iir/paraflow/examples/collector/MockDataSink.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.examples.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.collector.DataSink; 4 | 5 | import java.util.Objects; 6 | 7 | /** 8 | * paraflow 9 | * 10 | * @author guodong 11 | */ 12 | public class MockDataSink 13 | extends DataSink 14 | { 15 | private final String dbName; 16 | private final String tblName; 17 | 18 | public MockDataSink(String dbName, String tblName) 19 | { 20 | super("mock-sink"); 21 | this.dbName = dbName; 22 | this.tblName = tblName; 23 | } 24 | 25 | @Override 26 | public String getDb() 27 | { 28 | return dbName; 29 | } 30 | 31 | @Override 32 | public String getTbl() 33 | { 34 | return tblName; 35 | } 36 | 37 | @Override 38 | public DataType getType() 39 | { 40 | return DataType.Parquet; 41 | } 42 | 43 | @Override 44 | public int hashCode() 45 | { 46 | return Objects.hashCode("mock-sink"); 47 | } 48 | 49 | @Override 50 | public boolean equals(Object other) 51 | { 52 | return false; 53 | } 54 | 55 | @Override 56 | public String toString() 57 | { 58 | return "MockDataSink"; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/ParaFlowExceptionLevel.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | /** 4 | * ParaFlow 5 | * 6 | * @author guodong 7 | */ 8 | public enum ParaFlowExceptionLevel 9 | { 10 | FATAL("FATAL"), // FATAL exception: system exit immediately 11 | ERROR("ERROR"), // ERROR exception: log out and output it to stderr, system continues to run. 12 | // This is for the situation where system can still continue to run though it may run abnormally. 13 | WARN("WARN"), // WARN exception: log out and output it to stdout, system continues to run. 14 | // This is for the situation where system can still run normally with this exception. 15 | INFO("INFO"), // INFO exception: log out, system continues to run. 16 | // This is for FYI information thrown back to user during execution. 17 | DEBUG("DEBUG"); // DEBUG exception: output to stdout. this is for system test and debug. 18 | // This kind of exceptions are NOT expected in the production environment. 19 | 20 | private final String name; 21 | 22 | ParaFlowExceptionLevel(String name) 23 | { 24 | this.name = name; 25 | } 26 | 27 | @Override 28 | public String toString() 29 | { 30 | return this.name; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/Region.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class Region 11 | implements Model 12 | { 13 | private final long rowNumber; 14 | private final long regionKey; 15 | private final String name; 16 | private final String comment; 17 | 18 | public Region(long rowNumber, long regionKey, String name, String comment) 19 | { 20 | this.rowNumber = rowNumber; 21 | this.regionKey = regionKey; 22 | this.name = Objects.requireNonNull(name, "name is null"); 23 | this.comment = Objects.requireNonNull(comment, "comment is null"); 24 | } 25 | 26 | @Override 27 | public long getRowNumber() 28 | { 29 | return rowNumber; 30 | } 31 | 32 | @Override 33 | public String toLine() 34 | { 35 | return regionKey + "|" + 36 | name + "|" + 37 | comment; 38 | } 39 | 40 | public long getRegionKey() 41 | { 42 | return regionKey; 43 | } 44 | 45 | public String getName() 46 | { 47 | return name; 48 | } 49 | 50 | public String getComment() 51 | { 52 | return comment; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/utils/CollectorThroughputMetric.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector.utils; 2 | 3 | import io.prometheus.client.CollectorRegistry; 4 | import io.prometheus.client.Gauge; 5 | import io.prometheus.client.exporter.PushGateway; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * paraflow 11 | * 12 | * @author guodong 13 | */ 14 | public class CollectorThroughputMetric 15 | { 16 | private final CollectorRegistry registry = new CollectorRegistry(); 17 | private final Gauge throughputMetric = 18 | Gauge.build().name("collector_throughput") 19 | .help("Collector throughput (MB/s)") 20 | .labelNames("id") 21 | .register(registry); 22 | private final String id; 23 | private final PushGateway gateway; 24 | 25 | public CollectorThroughputMetric(String id, String gatewayUrl) 26 | { 27 | this.id = id; 28 | this.gateway = new PushGateway(gatewayUrl); 29 | } 30 | 31 | public void addValue(double value) 32 | { 33 | throughputMetric.labels(id).set(value); 34 | try { 35 | gateway.pushAdd(registry, "collector_metrics"); 36 | } 37 | catch (IOException e) { 38 | e.printStackTrace(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/CustomerColumn.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import io.airlift.tpch.TpchColumnType; 4 | 5 | import static io.airlift.tpch.TpchColumnTypes.DOUBLE; 6 | import static io.airlift.tpch.TpchColumnTypes.IDENTIFIER; 7 | import static io.airlift.tpch.TpchColumnTypes.varchar; 8 | 9 | /** 10 | * paraflow 11 | * 12 | * @author guodong 13 | */ 14 | public enum CustomerColumn 15 | implements Column 16 | { 17 | CUSTOMER_KEY("c_custkey", IDENTIFIER), 18 | NAME("c_name", varchar(25)), 19 | ADDRESS("c_address", varchar(40)), 20 | NATION_KEY("c_nationkey", IDENTIFIER), 21 | PHONE("c_phone", varchar(15)), 22 | ACCOUNT_BALANCE("c_acctbal", DOUBLE), 23 | MARKET_SEGMENT("c_mktsegment", varchar(10)), 24 | COMMENT("c_comment", varchar(117)); 25 | 26 | private final String columnName; 27 | private final TpchColumnType type; 28 | 29 | CustomerColumn(String columnName, TpchColumnType type) 30 | { 31 | this.columnName = columnName; 32 | this.type = type; 33 | } 34 | 35 | @Override 36 | public String getColumnName() 37 | { 38 | return columnName; 39 | } 40 | 41 | @Override 42 | public TpchColumnType getType() 43 | { 44 | return type; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/CollectorRuntime.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.collector.utils.CollectorConfig; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | 10 | /** 11 | * collector runtime 12 | * 13 | * @author guodong 14 | */ 15 | class CollectorRuntime 16 | { 17 | private static final ExecutorService executorService = Executors.newCachedThreadPool(); 18 | private static final Map flowTasks = new HashMap<>(); 19 | private static final long statsInterval = 3000L; 20 | private static ParaflowKafkaProducer kafkaProducer; 21 | 22 | CollectorRuntime(CollectorConfig conf) 23 | { 24 | kafkaProducer = new ParaflowKafkaProducer(conf, statsInterval); 25 | } 26 | 27 | static void destroy() 28 | { 29 | for (FlowTask task : flowTasks.values()) { 30 | task.close(); 31 | } 32 | executorService.shutdownNow(); 33 | } 34 | 35 | void run(DataFlow dataFlow) 36 | { 37 | FlowTask task = new FlowTask<>(dataFlow, kafkaProducer); 38 | flowTasks.put(dataFlow.getName(), task); 39 | executorService.submit((Runnable) task::execute); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/config/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=stdout, E 2 | 3 | # stdout 4 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target = System.out 6 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 7 | 8 | # DEBUG 9 | log4j.appender.D = org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.D.File = /home/iir/opt/paraflow/logs/debug.log 11 | log4j.appender.D.Append = true 12 | log4j.appender.D.Threshold = DEBUG 13 | log4j.appender.D.layout = org.apache.log4j.PatternLayout 14 | 15 | # INFO 16 | log4j.appender.I = org.apache.log4j.DailyRollingFileAppender 17 | log4j.appender.I.File = /home/iir/opt/paraflow/logs/info.log 18 | log4j.appender.I.Append = true 19 | log4j.appender.I.Threshold = INFO 20 | log4j.appender.I.layout = org.apache.log4j.PatternLayout 21 | 22 | # WARN 23 | log4j.appender.W = org.apache.log4j.DailyRollingFileAppender 24 | log4j.appender.W.File = /home/iir/opt/paraflow/logs/warn.log 25 | log4j.appender.W.Append = true 26 | log4j.appender.W.Threshold = WARN 27 | log4j.appender.W.layout = org.apache.log4j.PatternLayout 28 | 29 | # ERROR 30 | log4j.appender.E = org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.E.File = /home/iir/opt/paraflow/logs/error.log 32 | log4j.appender.E.Append = true 33 | log4j.appender.E.Threshold = ERROR 34 | log4j.appender.E.layout = org.apache.log4j.PatternLayout 35 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/config/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=stdout, E 2 | 3 | # stdout 4 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target = System.out 6 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 7 | 8 | # DEBUG 9 | log4j.appender.D = org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.D.File = /home/iir/opt/paraflow/logs/debug.log 11 | log4j.appender.D.Append = true 12 | log4j.appender.D.Threshold = DEBUG 13 | log4j.appender.D.layout = org.apache.log4j.PatternLayout 14 | 15 | # INFO 16 | log4j.appender.I = org.apache.log4j.DailyRollingFileAppender 17 | log4j.appender.I.File = /home/iir/opt/paraflow/logs/info.log 18 | log4j.appender.I.Append = true 19 | log4j.appender.I.Threshold = INFO 20 | log4j.appender.I.layout = org.apache.log4j.PatternLayout 21 | 22 | # WARN 23 | log4j.appender.W = org.apache.log4j.DailyRollingFileAppender 24 | log4j.appender.W.File = /home/iir/opt/paraflow/logs/warn.log 25 | log4j.appender.W.Append = true 26 | log4j.appender.W.Threshold = WARN 27 | log4j.appender.W.layout = org.apache.log4j.PatternLayout 28 | 29 | # ERROR 30 | log4j.appender.E = org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.E.File = /home/iir/opt/paraflow/logs/error.log 32 | log4j.appender.E.Append = true 33 | log4j.appender.E.Threshold = ERROR 34 | log4j.appender.E.layout = org.apache.log4j.PatternLayout 35 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/UpdateBlockPathAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.UpdateBlockPathException; 5 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 6 | import cn.edu.ruc.iir.paraflow.metaserver.proto.MetaProto; 7 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 8 | 9 | import java.util.Optional; 10 | 11 | /** 12 | * paraflow 13 | * 14 | * @author guodong 15 | */ 16 | public class UpdateBlockPathAction 17 | extends Action 18 | { 19 | @Override 20 | public ActionResponse act(ActionResponse input, Connection connection) 21 | throws ParaFlowException 22 | { 23 | Optional paramOp = input.getParam(); 24 | if (paramOp.isPresent()) { 25 | MetaProto.UpdateBlockPathParam param = (MetaProto.UpdateBlockPathParam) paramOp.get(); 26 | String updateBlockPathStatement = SQLTemplate.updateBlockPath(param.getOriginPath(), param.getNewPath()); 27 | int status = connection.executeUpdate(updateBlockPathStatement); 28 | if (status == 0) { 29 | throw new UpdateBlockPathException(); 30 | } 31 | } 32 | return null; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/UserCreationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class UserCreationException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = 3073077438904348727L; 8 | 9 | /** 10 | * get error message. 11 | * 12 | * @return error message 13 | */ 14 | @Override 15 | public String getMessage() 16 | { 17 | return "User creation error"; 18 | } 19 | 20 | @Override 21 | public StatusProto.ResponseStatus getResponseStatus() 22 | { 23 | return StatusProto.ResponseStatus.newBuilder() 24 | .setStatus(StatusProto.ResponseStatus.State.USER_CREATION_ERROR) 25 | .build(); 26 | } 27 | 28 | /** 29 | * get system hint message for user on how to deal with this exception 30 | * 31 | * @return hint message 32 | */ 33 | @Override 34 | public String getHint() 35 | { 36 | return "User may already exist, try again."; 37 | } 38 | 39 | /** 40 | * get exception level 41 | * 42 | * @return exception level 43 | */ 44 | @Override 45 | public ParaFlowExceptionLevel getLevel() 46 | { 47 | return ParaFlowExceptionLevel.ERROR; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/config/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=stdout, E 2 | 3 | # stdout 4 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.Target = System.out 6 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 7 | 8 | # DEBUG 9 | log4j.appender.D = org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.D.File = /home/iir/opt/paraflow/logs/debug.log 11 | log4j.appender.D.Append = true 12 | log4j.appender.D.Threshold = DEBUG 13 | log4j.appender.D.layout = org.apache.log4j.PatternLayout 14 | 15 | # INFO 16 | log4j.appender.I = org.apache.log4j.DailyRollingFileAppender 17 | log4j.appender.I.File = /home/iir/opt/paraflow/logs/info.log 18 | log4j.appender.I.Append = true 19 | log4j.appender.I.Threshold = INFO 20 | log4j.appender.I.layout = org.apache.log4j.PatternLayout 21 | 22 | # WARN 23 | log4j.appender.W = org.apache.log4j.DailyRollingFileAppender 24 | log4j.appender.W.File = /home/iir/opt/paraflow/logs/warn.log 25 | log4j.appender.W.Append = true 26 | log4j.appender.W.Threshold = WARN 27 | log4j.appender.W.layout = org.apache.log4j.PatternLayout 28 | 29 | # ERROR 30 | log4j.appender.E = org.apache.log4j.DailyRollingFileAppender 31 | log4j.appender.E.File = /home/iir/opt/paraflow/logs/error.log 32 | log4j.appender.E.Append = true 33 | log4j.appender.E.Threshold = ERROR 34 | log4j.appender.E.layout = org.apache.log4j.PatternLayout 35 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MetaTableCorruptedException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class MetaTableCorruptedException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = -6160322841066723567L; 8 | 9 | /** 10 | * get error message. 11 | * 12 | * @return error message 13 | */ 14 | @Override 15 | public String getMessage() 16 | { 17 | return "Meta table corrupted"; 18 | } 19 | 20 | @Override 21 | public StatusProto.ResponseStatus getResponseStatus() 22 | { 23 | return StatusProto.ResponseStatus.newBuilder() 24 | .setStatus(StatusProto.ResponseStatus.State.META_TABLE_CORRUPTED_FATAL) 25 | .build(); 26 | } 27 | 28 | /** 29 | * get system hint message for user on how to deal with this exception 30 | * 31 | * @return hint message 32 | */ 33 | @Override 34 | public String getHint() 35 | { 36 | return "Clean meta data and try again"; 37 | } 38 | 39 | /** 40 | * get exception level 41 | * 42 | * @return exception level 43 | */ 44 | @Override 45 | public ParaFlowExceptionLevel getLevel() 46 | { 47 | return ParaFlowExceptionLevel.FATAL; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/resources/paraflow.properties: -------------------------------------------------------------------------------- 1 | # Kafka producer configuration 2 | kafka.bootstrap.servers=127.0.0.1:9092 3 | kafka.acks=all 4 | kafka.retries=0 5 | kafka.batch.size=16384 6 | kafka.linger.ms=1 7 | kafka.buffer.memory=33554432 8 | # Default deserializers for key and value 9 | kafka.key.deserializer=org.apache.kafka.common.serialization.LongDeserializer 10 | kafka.value.deserializer=cn.edu.ruc.iir.paraflow.commons.message.DefaultMessageDe 11 | # Meta server 12 | metadata.server.host=127.0.0.1 13 | # metadata.server.host=10.77.40.27 14 | metadata.server.port=10012 15 | meta.client.shutdown.timeout=2 16 | # Consumer 17 | consumer.buffer.poll.timeout=3 18 | consumer.buffer.offer.timeout=3 19 | consumer.thread.num=1 20 | consumer.shutdown.timeout=5 21 | consumer.buffer.pool.size=10240 22 | consumer.group.id=1 23 | consumer.timeout.ms=-1 24 | consumer.poll.timeout.ms=60000 25 | consumer.pull.thread.num=1 26 | consumer.process.thread.num=1 27 | consumer.flush.thread.num=1 28 | #HDFS 29 | hdfs.warehouse=hdfs://127.0.0.1:9000/metadata 30 | #hdfs.warehouse=hdfs://10.77.40.27:9000/metadata 31 | #ORC FILE 32 | orc.file.stripe.size=67108864 33 | orc.file.buffer.size=131072 34 | orc.file.block.size=134217728 35 | //orc.file.stripe.size=10240 36 | //orc.file.buffer.size=1024 37 | //orc.file.block.size=20480 38 | # the dir containing core-site.xml and hdfs-site.xml 39 | hdfs.config.dir=/home/presto/opt/hadoop-2.7.3/etc/hadoop/ -------------------------------------------------------------------------------- /paraflow-connector/src/test/java/cn/edu/ruc/iir/paraflow/connector/TestHdfsFile.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.connector; 2 | 3 | import cn.edu.ruc.iir.paraflow.connector.impl.ParaflowPrestoConfig; 4 | import com.facebook.presto.spi.HostAddress; 5 | import org.apache.hadoop.fs.Path; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * @version V1.0 13 | * @Package: cn.edu.ruc.iir.paraflow.connector 14 | * @ClassName: TestHdfsFile 15 | * @Description: 16 | * @author: tao 17 | * @date: Create in 2019-09-14 18:50 18 | **/ 19 | public class TestHdfsFile 20 | { 21 | ParaflowPrestoConfig config; 22 | FSFactory fsFactory; 23 | 24 | @Before 25 | public void setUp() 26 | { 27 | config = new ParaflowPrestoConfig(); 28 | config.setHDFSWarehouse("hdfs://dbiir01:9000/paraflow"); 29 | config.setMetaserverHost("127.0.0.1"); 30 | config.setMetaserverPort(10012); 31 | fsFactory = new FSFactory(config); 32 | } 33 | 34 | @Test 35 | public void testFileLocation() 36 | { 37 | Path tablePath = new Path("hdfs://dbiir01:9000/paraflow/test/tpch/019355364443918028120303902"); 38 | List addresses = fsFactory.getBlockLocations(tablePath, 0, Long.MAX_VALUE); 39 | for (HostAddress s : addresses) { 40 | System.out.println(s.getHostText()); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/DatabaseCreationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class DatabaseCreationException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = -8371247825004879423L; 8 | 9 | /** 10 | * get error message. 11 | * 12 | * @return error message 13 | */ 14 | @Override 15 | public String getMessage() 16 | { 17 | return "Database creation exception"; 18 | } 19 | 20 | @Override 21 | public StatusProto.ResponseStatus getResponseStatus() 22 | { 23 | return StatusProto.ResponseStatus 24 | .newBuilder() 25 | .setStatus(StatusProto.ResponseStatus.State.DATABASE_CREATION_ERROR) 26 | .build(); 27 | } 28 | 29 | /** 30 | * get system hint message for user on how to deal with this exception 31 | * 32 | * @return hint message 33 | */ 34 | @Override 35 | public String getHint() 36 | { 37 | return "Check and try again"; 38 | } 39 | 40 | /** 41 | * get exception level 42 | * 43 | * @return exception level 44 | */ 45 | @Override 46 | public ParaFlowExceptionLevel getLevel() 47 | { 48 | return ParaFlowExceptionLevel.ERROR; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/function/Function.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.function; 15 | 16 | import com.fasterxml.jackson.annotation.JsonSubTypes; 17 | import com.fasterxml.jackson.annotation.JsonTypeInfo; 18 | 19 | /** 20 | * presto-root 21 | * 22 | * @author Jelly 23 | */ 24 | 25 | @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "className") 26 | @JsonSubTypes({@JsonSubTypes.Type(value = Function0.class, name = "function0")}) 27 | public interface Function 28 | { 29 | int apply(String v); 30 | 31 | long apply(int v); 32 | 33 | long apply(long v); 34 | 35 | @Override 36 | int hashCode(); 37 | 38 | @Override 39 | boolean equals(Object obj); 40 | 41 | @Override 42 | String toString(); 43 | } 44 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MessageDeSerializationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class MessageDeSerializationException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = -6499213164635187737L; 13 | 14 | /** 15 | * get error message. 16 | * 17 | * @return error message 18 | */ 19 | @Override 20 | public String getMessage() 21 | { 22 | return "Message deserialization exception"; 23 | } 24 | 25 | @Override 26 | public StatusProto.ResponseStatus getResponseStatus() 27 | { 28 | // todo set status as MSG_DESERIALIZE_ERROR 29 | return StatusProto.ResponseStatus.newBuilder().build(); 30 | } 31 | 32 | /** 33 | * get system hint message for user on how to deal with this exception 34 | * 35 | * @return hint message 36 | */ 37 | @Override 38 | public String getHint() 39 | { 40 | return "Check byte content and this one is ignored"; 41 | } 42 | 43 | /** 44 | * get exception level 45 | * 46 | * @return exception level 47 | */ 48 | @Override 49 | public ParaFlowExceptionLevel getLevel() 50 | { 51 | return ParaFlowExceptionLevel.WARN; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MetaInitException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class MetaInitException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = -4623517898557801064L; 13 | 14 | /** 15 | * get error message. 16 | * 17 | * @return error message 18 | */ 19 | @Override 20 | public String getMessage() 21 | { 22 | return "Metadata initialization error"; 23 | } 24 | 25 | @Override 26 | public StatusProto.ResponseStatus getResponseStatus() 27 | { 28 | return StatusProto.ResponseStatus.newBuilder() 29 | .setStatus(StatusProto.ResponseStatus.State.META_INIT_FATAL) 30 | .build(); 31 | } 32 | 33 | /** 34 | * get system hint message for user on how to deal with this exception 35 | * 36 | * @return hint message 37 | */ 38 | @Override 39 | public String getHint() 40 | { 41 | return "Try clean meta data and try again"; 42 | } 43 | 44 | /** 45 | * get exception level 46 | * 47 | * @return exception level 48 | */ 49 | @Override 50 | public ParaFlowExceptionLevel getLevel() 51 | { 52 | return ParaFlowExceptionLevel.FATAL; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ParaFlow 2 | 3 | ParaFlow is an interactive analysis system for OLAP developed at [DBIIR Lab @ RUC](http://iir.ruc.edu.cn). 4 | 5 | #### Install & Deploy 6 | ##### Hadoop 7 | Hadoop file system is required. 8 | ##### Zookeeper-3.4.13 9 | This is required by Kafka. 10 | what need to deploy is simply config the cluster ip and port; 11 | ##### Kafka-2.11_1.11 12 | ##### Postgresql-9.5 13 | ##### Presto-0.192 14 | ##### Paraflow 15 | 1. MetaServer(one node) 16 | 2. Loader [cn.edu.ruc.iir.paraflow.example.loader.BasicLoader] 17 | 18 | config the ./paraflow-loader.sh then: 19 | 20 | `./sbin/paraflow-loader.sh deploy` 21 | 3. Collector [cn.edu.ruc.iir.paraflow.example.loader.BasicCollector] 22 | 23 | config the ./paraflow-collector.sh then: 24 | `./sbin/paraflow-collector.sh deploy` 25 | 4. Presto connector 26 | 27 | #### Configuration 28 | ##### Initialization 29 | 1. Create user and database in pg for metadata. 30 | 31 | `CREATE USER paraflow WITH PASSWORD 'paraflow'`; 32 | `CREATE DATABASE paraflowmeta`; 33 | `GRANT ALL ON DATABASE paraflowmeta TO paraflow`. 34 | 35 | #### Startup 36 | 1. Start Zookeeper cluster 37 | 2. Start Kafka 38 | 3. Start PostgreSql 39 | 4. Start Paraflow MetaServer 40 | `./bin/paraflow-metaserver-start.sh [-daemon]` 41 | 5. Start Paraflow Loader 42 | `./sbin/paraflow-loader.sh start` 43 | 6. Start Paraflow Collector 44 | `./sbin/paraflow-collector.sh start` 45 | 7. Start Presto cluster or single node to execute queries; 46 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MetaTableCreationException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class MetaTableCreationException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = -4915701495568826208L; 13 | 14 | /** 15 | * get error message. 16 | * 17 | * @return error message 18 | */ 19 | @Override 20 | public String getMessage() 21 | { 22 | return "meta table creation exception"; 23 | } 24 | 25 | @Override 26 | public StatusProto.ResponseStatus getResponseStatus() 27 | { 28 | return StatusProto.ResponseStatus.newBuilder() 29 | .setStatus(StatusProto.ResponseStatus.State.META_TABLE_CREATION_FATAL) 30 | .build(); 31 | } 32 | 33 | /** 34 | * get system hint message for user on how to deal with this exception 35 | * 36 | * @return hint message 37 | */ 38 | @Override 39 | public String getHint() 40 | { 41 | return "Check and try again"; 42 | } 43 | 44 | /** 45 | * get exception level 46 | * 47 | * @return exception level 48 | */ 49 | @Override 50 | public ParaFlowExceptionLevel getLevel() 51 | { 52 | return ParaFlowExceptionLevel.FATAL; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/DataFlow.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | import cn.edu.ruc.iir.paraflow.commons.ParaflowFiberPartitioner; 5 | 6 | import java.util.concurrent.atomic.AtomicBoolean; 7 | 8 | /** 9 | * dataflow interface 10 | * 11 | * @author guodong 12 | */ 13 | public abstract class DataFlow 14 | { 15 | protected final String name; 16 | protected final DataSource dataSource; 17 | protected AtomicBoolean finished = new AtomicBoolean(false); 18 | 19 | public DataFlow(String name, DataSource dataSource) 20 | { 21 | this.name = name; 22 | this.dataSource = dataSource; 23 | } 24 | 25 | public String getName() 26 | { 27 | return this.name; 28 | } 29 | 30 | public void setFinished() 31 | { 32 | this.finished.set(true); 33 | } 34 | 35 | public boolean isFinished() 36 | { 37 | return this.finished.get(); 38 | } 39 | 40 | public abstract Message next(); 41 | 42 | public abstract FiberFlow keyBy(int idx); 43 | 44 | public abstract FiberFlow timeBy(int idx); 45 | 46 | public abstract FiberFlow sink(DataSink dataSink); 47 | 48 | public abstract DataSink getSink(); 49 | 50 | public abstract FiberFlow partitionBy(ParaflowFiberPartitioner partitioner); 51 | 52 | public abstract ParaflowFiberPartitioner getPartitioner(); 53 | } 54 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/Types.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector; 15 | 16 | import static com.google.common.base.Preconditions.checkArgument; 17 | import static java.lang.String.format; 18 | 19 | /** 20 | * @author jelly.guodong.jin@gmail.com 21 | */ 22 | public class Types 23 | { 24 | private Types() 25 | { 26 | } 27 | 28 | public static B checkType(A value, Class target, String name) 29 | { 30 | if (value == null) { 31 | throw new NullPointerException(format("%s is null", name)); 32 | } 33 | checkArgument(target.isInstance(value), 34 | "%s must be of type %s, not %s", 35 | name, 36 | target.getName(), 37 | value.getClass().getName()); 38 | return target.cast(value); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/utils/KafkaPartitioner.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector.utils; 2 | 3 | import org.apache.kafka.clients.producer.Partitioner; 4 | import org.apache.kafka.common.Cluster; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * paraflow 10 | * 11 | * @author guodong 12 | */ 13 | public abstract class KafkaPartitioner implements Partitioner 14 | { 15 | /** 16 | * Compute the partition for the given record. 17 | * 18 | * @param topic The topic name 19 | * @param key The key to partition on (or null if no key) 20 | * @param keyBytes The serialized key to partition on( or null if no key) 21 | * @param value The value to partition on or null 22 | * @param valueBytes The serialized value to partition on or null 23 | * @param cluster The current cluster metadata 24 | */ 25 | @Override 26 | public abstract int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster); 27 | 28 | /** 29 | * This is called when partitioner is closed. 30 | */ 31 | @Override 32 | public void close() 33 | { 34 | // nothing to do 35 | } 36 | 37 | /** 38 | * Configure this class with the given key-value pairs 39 | * 40 | * @param configs 41 | */ 42 | @Override 43 | public void configure(Map configs) 44 | { 45 | // nothing to do 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/ActionParamNotValidException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | //todo alice: add exception implementation 6 | public class ActionParamNotValidException extends ParaFlowException 7 | { 8 | /** 9 | * get error message. 10 | * 11 | * @return error message 12 | */ 13 | private static final long serialVersionUID = 6980171393859140122L; 14 | 15 | @Override 16 | public String getMessage() 17 | { 18 | return String.format("Action param not valid !"); 19 | } 20 | 21 | @Override 22 | public StatusProto.ResponseStatus getResponseStatus() 23 | { 24 | return StatusProto.ResponseStatus 25 | .newBuilder() 26 | .setStatus(StatusProto.ResponseStatus.State.ACTION_PARAM_INVALID_ERROR) 27 | .build(); 28 | } 29 | 30 | /** 31 | * get system hint message for user on how to deal with this exception 32 | * 33 | * @return hint message 34 | */ 35 | @Override 36 | public String getHint() 37 | { 38 | return "Action param not valid !"; 39 | } 40 | 41 | /** 42 | * get exception level 43 | * 44 | * @return exception level 45 | */ 46 | @Override 47 | public ParaFlowExceptionLevel getLevel() 48 | { 49 | return ParaFlowExceptionLevel.ERROR; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/CreateTblFuncAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.TblFuncCreationException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 8 | 9 | import java.util.Optional; 10 | 11 | public class CreateTblFuncAction extends Action 12 | { 13 | @Override 14 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 15 | { 16 | Optional tblIdOp = input.getProperties("tblId"); 17 | Optional funcIdOp = input.getProperties("funcId"); 18 | if (tblIdOp.isPresent() 19 | && funcIdOp.isPresent()) { 20 | //create tblfunc SQL 21 | String userStatement = SQLTemplate.createTblFunc( 22 | (long) tblIdOp.get(), 23 | (long) funcIdOp.get()); 24 | int status = connection.executeUpdate(userStatement); 25 | if (status == 0) { 26 | throw new TblFuncCreationException(); 27 | } 28 | } 29 | else { 30 | throw new ActionParamNotValidException(); 31 | } 32 | return input; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/ParaflowPlugin.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector; 15 | 16 | import com.facebook.presto.spi.Plugin; 17 | import com.facebook.presto.spi.connector.ConnectorFactory; 18 | import com.google.common.collect.ImmutableList; 19 | 20 | import static com.google.common.base.MoreObjects.firstNonNull; 21 | 22 | /** 23 | * presto-hdfs 24 | * 25 | * @author jelly.guodong.jin@gmail.com 26 | */ 27 | public class ParaflowPlugin 28 | implements Plugin 29 | { 30 | public static ClassLoader getClassLoader() 31 | { 32 | return firstNonNull(Thread.currentThread().getContextClassLoader(), ParaflowPlugin.class.getClassLoader()); 33 | } 34 | 35 | @Override 36 | public Iterable getConnectorFactories() 37 | { 38 | return ImmutableList.of(new ParaflowConnectorFactory()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/PrestoSelectTemplate3.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | import java.util.Random; 4 | 5 | /** 6 | * paraflow 7 | * 3. join with customer 8 | * 9 | * @author guodong 10 | */ 11 | public class PrestoSelectTemplate3 12 | extends QueryTemplate 13 | { 14 | private final long maxCustkey; 15 | private final String joinTable; 16 | private final Random random; 17 | 18 | public PrestoSelectTemplate3(QueryDistribution distribution, String table, String joinTable) 19 | { 20 | super(table); 21 | this.maxCustkey = distribution.getValue("max-custkey"); 22 | this.joinTable = joinTable; 23 | this.random = new Random(8833948812L); 24 | } 25 | 26 | @Override 27 | String makeQuery() 28 | { 29 | int custkey = random.nextInt((int) maxCustkey); 30 | return "SELECT c_name, c_address, c_phone, SUM(lo_quantity) AS sum_qty , AVG(lo_extendedprice) AS avg_price, avg(lo_discount) AS avg_disc, count(*) AS rs_num, min(lo_lineorderkey) AS min_lineorderkey, max(lo_lineorderkey) AS max_lineorderkey FROM " 31 | + table + ", " + joinTable 32 | + " WHERE " + table + ".lo_custkey=" + joinTable + ".c_custkey AND lo_custkey=" 33 | + custkey + " GROUP BY c_name, c_address, c_phone"; 34 | } 35 | 36 | @Override 37 | QueryGenerator.QueryType getType() 38 | { 39 | return QueryGenerator.QueryType.SELECT; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/generator/TimeGenerationPool.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.generator; 2 | 3 | /** 4 | * TimeGenerationPool generates timestamps based on user-defined frequency. 5 | * stride: how long we calculate the frequency. 6 | * frequency * stride: the number of records for every stride(like 1, 5, 10, etc.) seconds. 7 | * 8 | * @author guodong 9 | */ 10 | public class TimeGenerationPool 11 | { 12 | private long currentTime; 13 | private long frequency; 14 | private long stride; 15 | private long counter = 0; 16 | 17 | private TimeGenerationPool() 18 | { 19 | } 20 | 21 | public static final TimeGenerationPool INSTANCE() 22 | { 23 | return TimeGenerationPoolHolder.instance; 24 | } 25 | 26 | /** 27 | * Generation frequency. records/s 28 | */ 29 | public void init(long startTime, long stride, long frequency) 30 | { 31 | this.currentTime = startTime; 32 | this.stride = stride; 33 | this.frequency = frequency; 34 | } 35 | 36 | public synchronized long nextTime() 37 | { 38 | counter++; 39 | if (counter > frequency) { 40 | currentTime++; 41 | counter = 0; 42 | return currentTime; 43 | } 44 | return currentTime; 45 | } 46 | 47 | private static final class TimeGenerationPoolHolder 48 | { 49 | private static final TimeGenerationPool instance = new TimeGenerationPool(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/SQLExecutionException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | public class SQLExecutionException extends ParaFlowException 6 | { 7 | private static final long serialVersionUID = 5939773817133884884L; 8 | private final String sql; 9 | 10 | public SQLExecutionException(String sql) 11 | { 12 | this.sql = sql; 13 | } 14 | 15 | /** 16 | * get error message. 17 | * 18 | * @return error message 19 | */ 20 | @Override 21 | public String getMessage() 22 | { 23 | return String.format("SQL execution error [%s]", sql); 24 | } 25 | 26 | @Override 27 | public StatusProto.ResponseStatus getResponseStatus() 28 | { 29 | return StatusProto.ResponseStatus.newBuilder() 30 | .setStatus(StatusProto.ResponseStatus.State.SQL_EXECUTION_ERROR) 31 | .build(); 32 | } 33 | 34 | /** 35 | * get system hint message for user on how to deal with this exception 36 | * 37 | * @return hint message 38 | */ 39 | @Override 40 | public String getHint() 41 | { 42 | return "jdbc error"; 43 | } 44 | 45 | /** 46 | * get exception level 47 | * 48 | * @return exception level 49 | */ 50 | @Override 51 | public ParaFlowExceptionLevel getLevel() 52 | { 53 | return ParaFlowExceptionLevel.ERROR; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/model/Nation.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.model; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class Nation 11 | implements Model 12 | { 13 | private final long rowNumber; 14 | private final long nationKey; 15 | private final String name; 16 | private final long regionKey; 17 | private final String comment; 18 | 19 | public Nation(long rowNumber, long nationKey, String name, long regionKey, String comment) 20 | { 21 | this.rowNumber = rowNumber; 22 | this.nationKey = nationKey; 23 | this.name = Objects.requireNonNull(name, "name is null"); 24 | this.regionKey = regionKey; 25 | this.comment = Objects.requireNonNull(comment, "comment is null"); 26 | } 27 | 28 | @Override 29 | public long getRowNumber() 30 | { 31 | return rowNumber; 32 | } 33 | 34 | @Override 35 | public String toLine() 36 | { 37 | return nationKey + "|" + 38 | name + "|" + 39 | regionKey + "|" + 40 | comment; 41 | } 42 | 43 | public long getNationKey() 44 | { 45 | return nationKey; 46 | } 47 | 48 | public String getName() 49 | { 50 | return name; 51 | } 52 | 53 | public long getRegionKey() 54 | { 55 | return regionKey; 56 | } 57 | 58 | public String getComment() 59 | { 60 | return comment; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MessageSerializeException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public class MessageSerializeException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = 2583464045707403311L; 13 | private final String message; 14 | 15 | public MessageSerializeException(String message) 16 | { 17 | this.message = message; 18 | } 19 | 20 | /** 21 | * get error message. 22 | * 23 | * @return error message 24 | */ 25 | @Override 26 | public String getMessage() 27 | { 28 | return message; 29 | } 30 | 31 | @Override 32 | public StatusProto.ResponseStatus getResponseStatus() 33 | { 34 | // todo set status as MSG_SERIALIZE_ERROR 35 | return StatusProto.ResponseStatus.newBuilder().build(); 36 | } 37 | 38 | /** 39 | * get system hint message for user on how to deal with this exception 40 | * 41 | * @return hint message 42 | */ 43 | @Override 44 | public String getHint() 45 | { 46 | return "Check message and this one is ignored"; 47 | } 48 | 49 | /** 50 | * get exception level 51 | * 52 | * @return exception level 53 | */ 54 | @Override 55 | public ParaFlowExceptionLevel getLevel() 56 | { 57 | return ParaFlowExceptionLevel.WARN; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/CreateUserAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.UserCreationException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.proto.MetaProto; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | 10 | import java.util.Optional; 11 | 12 | public class CreateUserAction extends Action 13 | { 14 | @Override 15 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 16 | { 17 | Optional paramOp = input.getParam(); 18 | if (paramOp.isPresent()) { 19 | MetaProto.UserParam userParam = (MetaProto.UserParam) paramOp.get(); 20 | String userStatement = SQLTemplate.createUser( 21 | userParam.getUserName(), 22 | userParam.getPassword(), 23 | System.currentTimeMillis(), 24 | System.currentTimeMillis()); 25 | int status = connection.executeUpdate(userStatement); 26 | if (status == 0) { 27 | throw new UserCreationException(); 28 | } 29 | } 30 | else { 31 | throw new ActionParamNotValidException(); 32 | } 33 | return input; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/ActionResponse.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.metaserver.connection.ResultList; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Optional; 8 | 9 | public class ActionResponse 10 | { 11 | private ResultList responseResultList; 12 | private Object param = null; 13 | private Map properties = null; 14 | 15 | ResultList getResponseResultList() 16 | { 17 | return responseResultList; 18 | } 19 | 20 | void setResponseResultList(ResultList res) 21 | { 22 | this.responseResultList = res; 23 | } 24 | 25 | public Optional getParam() 26 | { 27 | if (param == null) { 28 | return Optional.empty(); 29 | } 30 | return Optional.of(param); 31 | } 32 | 33 | public void setParam(Object param) 34 | { 35 | this.param = param; 36 | } 37 | 38 | public void setProperties(String key, Object value) 39 | { 40 | if (properties == null) { 41 | properties = new HashMap<>(); 42 | } 43 | properties.put(key, value); 44 | } 45 | 46 | public Optional getProperties(String key) 47 | { 48 | if (properties == null) { 49 | return Optional.empty(); 50 | } 51 | if (properties.get(key) == null) { 52 | return Optional.empty(); 53 | } 54 | return Optional.of(properties.get(key)); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/GetUserNameAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.UserNotFoundException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.connection.ResultList; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | 10 | import java.util.Optional; 11 | 12 | public class GetUserNameAction extends Action 13 | { 14 | @Override 15 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 16 | { 17 | Optional paramOp = input.getParam(); 18 | Optional userIdOp = input.getProperties("userId"); 19 | if (paramOp.isPresent() && userIdOp.isPresent()) { 20 | long userId = (long) userIdOp.get(); 21 | String sqlStatement = SQLTemplate.findUserName(userId); 22 | ResultList resultList = connection.executeQuery(sqlStatement); 23 | if (!resultList.isEmpty()) { 24 | input.setProperties("userName", resultList.get(0).get(0)); 25 | } 26 | else { 27 | throw new UserNotFoundException(); 28 | } 29 | } 30 | else { 31 | throw new ActionParamNotValidException(); 32 | } 33 | return input; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/GetDatabaseIdAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.DatabaseNotFoundException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.connection.ResultList; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | 10 | import java.util.Optional; 11 | 12 | public class GetDatabaseIdAction extends Action 13 | { 14 | @Override 15 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 16 | { 17 | Optional paramOp = input.getParam(); 18 | Optional dbNameOp = input.getProperties("dbName"); 19 | if (paramOp.isPresent() && dbNameOp.isPresent()) { 20 | String sqlStatement = SQLTemplate.findDbId(dbNameOp.get().toString()); 21 | ResultList resultList = connection.executeQuery(sqlStatement); 22 | if (!resultList.isEmpty()) { 23 | input.setProperties("dbId", Long.parseLong(resultList.get(0).get(0))); 24 | } 25 | else { 26 | throw new DatabaseNotFoundException(dbNameOp.get().toString()); 27 | } 28 | } 29 | else { 30 | throw new ActionParamNotValidException(); 31 | } 32 | return input; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/QueryDistribution.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import static com.google.common.base.Preconditions.checkArgument; 7 | 8 | /** 9 | * paraflow 10 | * 11 | * @author guodong 12 | */ 13 | public class QueryDistribution 14 | { 15 | private final Map distributions; 16 | private long sizeLimit = -1L; // limit of query size generation. when reaching the specified number of queries, stop generation. 17 | private long timeLimit = -1L; // limit of query time generation. when reaching the specified time, stop generation. 18 | 19 | public QueryDistribution() 20 | { 21 | this.distributions = new HashMap<>(); 22 | } 23 | 24 | public void setSizeLimit(long sizeLimit) 25 | { 26 | checkArgument(sizeLimit >= 0, "size limit should not be negative"); 27 | this.sizeLimit = sizeLimit; 28 | } 29 | 30 | public void setTimeLimit(long timeLimit) 31 | { 32 | checkArgument(timeLimit >= 0, "time limit should not be negative"); 33 | this.timeLimit = timeLimit; 34 | } 35 | 36 | public long sizeLimit() 37 | { 38 | return sizeLimit; 39 | } 40 | 41 | public long timeLimit() 42 | { 43 | return timeLimit; 44 | } 45 | 46 | public void setDistribution(String key, long distributeValue) 47 | { 48 | distributions.put(key, distributeValue); 49 | } 50 | 51 | public long getValue(String key) 52 | { 53 | return distributions.getOrDefault(key, 0L); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /paraflow-loader/src/main/java/cn/edu/ruc/iir/paraflow/loader/ParaflowKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.loader; 2 | 3 | import org.apache.kafka.clients.consumer.Consumer; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.clients.consumer.KafkaConsumer; 6 | import org.apache.kafka.common.TopicPartition; 7 | import org.apache.kafka.common.serialization.ByteArrayDeserializer; 8 | 9 | import java.util.List; 10 | import java.util.Properties; 11 | 12 | public class ParaflowKafkaConsumer 13 | { 14 | private final Consumer consumer; 15 | 16 | public ParaflowKafkaConsumer(List topicPartitions, Properties config) 17 | { 18 | // set the consumer configuration properties for kafka record key and value serializers 19 | if (!config.containsKey(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG)) { 20 | config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); 21 | } 22 | if (!config.containsKey(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG)) { 23 | config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); 24 | } 25 | if (!config.containsKey(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG)) { 26 | throw new IllegalArgumentException(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG + " must be specified in the config"); 27 | } 28 | this.consumer = new KafkaConsumer<>(config); 29 | this.consumer.assign(topicPartitions); 30 | } 31 | 32 | public Consumer getConsumer() 33 | { 34 | return this.consumer; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/run/stop-all.sh: -------------------------------------------------------------------------------- 1 | !/bin/sh 2 | # stop-kafka-paraflow&metaServer&presto 3 | PREFIX="dbiir" 4 | master_num=1 5 | START=2 6 | END=9 7 | PARAFLOW_HOME="/home/iir/opt/paraflow" 8 | PARAFLOW_HEAP_OPTS="-Xmx16G -Xms8G" 9 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 10 | PARAFLOW_SBIN="/home/iir/opt/paraflow/sbin" 11 | PARAFLOW_BIN="/home/iir/opt/paraflow/bin" 12 | PRESTO_SBIN="/home/iir/opt/presto-server-0.192/sbin" 13 | PRESTO_BIN="/home/iir/opt/presto-server-0.192/bin" 14 | 15 | #stop the paraflow loaders 16 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/paraflow-loader.sh stop" 17 | echo "paraflow loader on dbiir0$master_num stopped" 18 | echo "loader is drawing data as a consumer from topic, at the same time, meta data sinking,relation data flushing to hdfs" 19 | 20 | #stop the paraflow collector 21 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/paraflow-collector.sh stop" 22 | echo "paraflow collector on dbiir0$master_num have stopped" 23 | 24 | # stop the Zookeeper 25 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/zookeeper-server.sh stop" 26 | echo " zookeeper02-06 cluster from dbiir0$master_num stopped" 27 | 28 | # stop the Kafka 29 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/kafka-server.sh stop" 30 | echo "kafka02-06 cluster from dbiir0$master_num stop" 31 | 32 | #stop the paraflow metaserver in daemon use & 33 | ssh $PREFIX"0"$master_num "$PARAFLOW_BIN/paraflow-metaserver-stop.sh &" 34 | echo " metaserver on dbiir0$master_num stopped in daemon" 35 | 36 | #stop the presto server cluster01-09 37 | ssh $PREFIX"0"$master_num "$PRESTO_SBIN/stop-all.sh" 38 | echo "presto server cluster on dbiir0$master_num - dbiir0$END stopped" 39 | 40 | echo "paraflow and its depencies entirely stop" 41 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/stop-all.sh: -------------------------------------------------------------------------------- 1 | !/bin/sh 2 | # stop-kafka-paraflow&metaServer&presto 3 | PREFIX="dbiir" 4 | master_num=1 5 | START=2 6 | END=9 7 | PARAFLOW_HOME="/home/iir/opt/paraflow" 8 | PARAFLOW_HEAP_OPTS="-Xmx16G -Xms8G" 9 | PARAFLOW_DIR="/home/iir/opt/paraflow-1.0-alpha1" 10 | PARAFLOW_SBIN="/home/iir/opt/paraflow/sbin" 11 | PARAFLOW_BIN="/home/iir/opt/paraflow/bin" 12 | PRESTO_SBIN="/home/iir/opt/presto-server-0.192/sbin" 13 | PRESTO_BIN="/home/iir/opt/presto-server-0.192/bin" 14 | 15 | #stop the paraflow loaders 16 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/paraflow-loader.sh stop" 17 | echo "paraflow loader on dbiir0$master_num stopped" 18 | echo "loader is drawing data as a consumer from topic, at the same time, meta data sinking,relation data flushing to hdfs" 19 | 20 | #stop the paraflow collector 21 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/paraflow-collector.sh stop" 22 | echo "paraflow collector on dbiir0$master_num have stopped" 23 | 24 | # stop the Zookeeper 25 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/zookeeper-server.sh stop" 26 | echo " zookeeper02-06 cluster from dbiir0$master_num stopped" 27 | 28 | # stop the Kafka 29 | ssh $PREFIX"0"$master_num "$PARAFLOW_SBIN/kafka-server.sh stop" 30 | echo "kafka02-06 cluster from dbiir0$master_num stop" 31 | 32 | #stop the paraflow metaserver in daemon use & 33 | ssh $PREFIX"0"$master_num "$PARAFLOW_BIN/paraflow-metaserver-stop.sh &" 34 | echo " metaserver on dbiir0$master_num stopped in daemon" 35 | 36 | #stop the presto server cluster01-09 37 | ssh $PREFIX"0"$master_num "$PRESTO_SBIN/stop-all.sh" 38 | echo "presto server cluster on dbiir0$master_num - dbiir0$END stopped" 39 | 40 | echo "paraflow and its depencies entirely stop" 41 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/MethodNotImplementedException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * paraflow 7 | * 8 | * @author guodong 9 | */ 10 | public final class MethodNotImplementedException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = 2492705799817499827L; 13 | private final String methodName; 14 | 15 | public MethodNotImplementedException(String methodName) 16 | { 17 | this.methodName = methodName; 18 | } 19 | 20 | /** 21 | * get error message. 22 | * 23 | * @return error message 24 | */ 25 | @Override 26 | public String getMessage() 27 | { 28 | return String.format("Method %s not implemented", methodName); 29 | } 30 | 31 | @Override 32 | public StatusProto.ResponseStatus getResponseStatus() 33 | { 34 | return StatusProto.ResponseStatus.newBuilder() 35 | .setStatus(StatusProto.ResponseStatus.State.METHOD_NOT_IMPLEMENTED_INFO) 36 | .build(); 37 | } 38 | 39 | /** 40 | * get system hint message for user on how to deal with this exception 41 | * 42 | * @return hint message 43 | */ 44 | @Override 45 | public String getHint() 46 | { 47 | return "Please define method implementation."; 48 | } 49 | 50 | /** 51 | * get exception level 52 | * 53 | * @return exception level 54 | */ 55 | @Override 56 | public ParaFlowExceptionLevel getLevel() 57 | { 58 | return ParaFlowExceptionLevel.INFO; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/ParaflowMetadataFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector; 15 | 16 | import cn.edu.ruc.iir.paraflow.connector.impl.ParaflowMetaDataReader; 17 | import com.google.inject.Inject; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | /** 22 | * @author jelly.guodong.jin@gmail.com 23 | */ 24 | public class ParaflowMetadataFactory 25 | { 26 | private final ParaflowConnectorId connectorId; 27 | private final ParaflowMetaDataReader metaDataQuery; 28 | 29 | @Inject 30 | public ParaflowMetadataFactory(ParaflowConnectorId connectorId, ParaflowMetaDataReader metaDataQuery) 31 | { 32 | this.connectorId = requireNonNull(connectorId, "connectorId is null"); 33 | this.metaDataQuery = requireNonNull(metaDataQuery, "metaServer is null"); 34 | } 35 | 36 | public ParaflowMetadata create() 37 | { 38 | return new ParaflowMetadata(metaDataQuery, connectorId); 39 | } 40 | 41 | public void shutdown() 42 | { 43 | metaDataQuery.shutdown(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/GetUserIdAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.UserNotFoundException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.connection.ResultList; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | 10 | import java.util.Optional; 11 | 12 | /** 13 | * paraflow 14 | * 15 | * @author guodong 16 | */ 17 | public class GetUserIdAction extends Action 18 | { 19 | @Override 20 | public ActionResponse act(ActionResponse input, Connection connection) 21 | throws ParaFlowException 22 | { 23 | Optional paramOp = input.getParam(); 24 | Optional userNameOp = input.getProperties("userName"); 25 | if (paramOp.isPresent() && userNameOp.isPresent()) { 26 | String userName = userNameOp.get().toString(); 27 | String sqlStatement = SQLTemplate.findUserId(userName); 28 | ResultList resultList = connection.executeQuery(sqlStatement); 29 | if (!resultList.isEmpty()) { 30 | input.setProperties("userId", Long.parseLong(resultList.get(0).get(0))); 31 | } 32 | else { 33 | throw new UserNotFoundException(); 34 | } 35 | return input; 36 | } 37 | else { 38 | throw new ActionParamNotValidException(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/application-dev.properties: -------------------------------------------------------------------------------- 1 | server.port=2018 2 | # 设置IO线程数, 它主要执行非阻塞的任务,它们会负责多个连接, 默认设置每个CPU核心一个线程 3 | server.undertow.io-threads=4 4 | # 阻塞任务线程池, 当执行类似servlet请求阻塞操作, undertow会从这个线程池中取得线程,它的值设置取决于系统的负载 5 | server.undertow.worker-threads=60 6 | # 以下的配置会影响buffer,这些buffer会用于服务器连接的IO操作,有点类似netty的池化内存管理 7 | # 每块buffer的空间大小,越小的空间被利用越充分 8 | server.undertow.buffer-size=1024 9 | # 每个区分配的buffer数量 , 所以pool的大小是buffer-size * buffers-per-region 10 | server.undertow.buffers-per-region=1024 11 | # 是否分配的直接内存 12 | server.undertow.direct-buffers=true 13 | # 中文乱码 14 | spring.http.encoding.force=true 15 | spring.http.encoding.charset=UTF-8 16 | spring.http.encoding.enabled=true 17 | server.tomcat.uri-encoding=UTF-8 18 | upload=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/UUID/ 19 | cloudPython=/data2/an_zhao/lab/ir_project/src/cloud.py 20 | featureURL=http://localhost:6175/retrival?input= 21 | hotPath=/data2/an_zhao/lab/ir_project/web_logs/new_log.txt 22 | Star.Data.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Data/ 23 | Index.Main.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Index/Main/ 24 | Scan.Main.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Main/ 25 | Index.Movie.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Index/Movie/ 26 | Scan.Movie.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Movie/ 27 | Index.Tv.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Index/Tv/ 28 | Scan.Tv.Path=/data2/an_zhao/lab/tomcat/apache-tomcat-8.5.27/webapps/ROOT/ImageRetrieval/Tv/ 29 | file.length=960 -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/utils/MetaConfig.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.utils; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ConfigFileNotFoundException; 4 | import cn.edu.ruc.iir.paraflow.commons.utils.ParaFlowConfig; 5 | 6 | /** 7 | * ParaFlow 8 | * 9 | * @author guodong 10 | */ 11 | public class MetaConfig 12 | { 13 | private ParaFlowConfig paraflowConfig; 14 | 15 | public MetaConfig() 16 | { 17 | } 18 | 19 | public static final MetaConfig INSTANCE() 20 | { 21 | return MetaConfigHolder.instance; 22 | } 23 | 24 | public void init() throws ConfigFileNotFoundException 25 | { 26 | paraflowConfig = new ParaFlowConfig("metaserver.conf"); 27 | paraflowConfig.build(); 28 | } 29 | 30 | public int getServerPort() 31 | { 32 | return Integer.parseInt(paraflowConfig.getProperty("server.port")); 33 | } 34 | 35 | public String getDBDriver() 36 | { 37 | return paraflowConfig.getProperty("db.driver"); 38 | } 39 | 40 | public String getDBHost() 41 | { 42 | return paraflowConfig.getProperty("db.host"); 43 | } 44 | 45 | public String getDBUser() 46 | { 47 | return paraflowConfig.getProperty("db.user"); 48 | } 49 | 50 | public String getDBPassword() 51 | { 52 | return paraflowConfig.getProperty("db.password"); 53 | } 54 | 55 | public String getHDFSWarehouse() 56 | { 57 | return paraflowConfig.getProperty("hdfs.warehouse"); 58 | } 59 | 60 | private static class MetaConfigHolder 61 | { 62 | public static final MetaConfig instance = new MetaConfig(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /paraflow-benchmark/src/main/java/cn/edu/ruc/iir/paraflow/benchmark/query/PrestoSelectTemplate1.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.benchmark.query; 2 | 3 | import java.util.Random; 4 | 5 | /** 6 | * paraflow 7 | * 1. filter with timestamp 8 | * 9 | * @author guodong 10 | */ 11 | public class PrestoSelectTemplate1 12 | extends QueryTemplate 13 | { 14 | private final int[] selection = {1, 2, 4, 6, 8, 10}; 15 | private final long maxTime; 16 | private final long minTime; 17 | private final long selectionSlice; 18 | private final Random random; 19 | private int selectIndex = 0; 20 | 21 | public PrestoSelectTemplate1(QueryDistribution distribution, String table) 22 | { 23 | super(table); 24 | this.maxTime = distribution.getValue("max-time"); 25 | this.minTime = distribution.getValue("min-time"); 26 | this.selectionSlice = (maxTime - minTime) / 100; 27 | this.random = new Random(2344449595L); 28 | } 29 | 30 | @Override 31 | String makeQuery() 32 | { 33 | long timePoint = random.nextInt((int) (maxTime - minTime)); 34 | long timeScale = selection[selectIndex++ % selection.length] * selectionSlice; 35 | return "SELECT SUM(lo_quantity) AS sum_qty , AVG(lo_extendedprice) AS avg_price, avg(lo_discount) AS avg_disc, count(*) AS rs_num, min(lo_lineorderkey) AS min_lineorderkey, max(lo_lineorderkey) AS max_lineorderkey FROM " 36 | + table 37 | + " WHERE lo_creation>" + (timePoint + minTime) 38 | + " AND lo_creation<" + (timePoint + minTime + timeScale); 39 | } 40 | 41 | @Override 42 | QueryGenerator.QueryType getType() 43 | { 44 | return QueryGenerator.QueryType.SELECT; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /paraflow-collector/src/main/java/cn/edu/ruc/iir/paraflow/collector/FiberFlow.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.collector; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.Message; 4 | import cn.edu.ruc.iir.paraflow.commons.ParaflowFiberPartitioner; 5 | 6 | /** 7 | * paraflow 8 | * 9 | * @author guodong 10 | */ 11 | public class FiberFlow 12 | extends DataFlow 13 | { 14 | private DataSink dataSink; 15 | private ParaflowFiberPartitioner partitioner; 16 | private int keyIdx; 17 | private int timeIdx; 18 | 19 | public FiberFlow(String name, DataSource dataSource) 20 | { 21 | super(name, dataSource); 22 | } 23 | 24 | public String getName() 25 | { 26 | return super.name; 27 | } 28 | 29 | @Override 30 | public Message next() 31 | { 32 | return super.dataSource.read(); 33 | } 34 | 35 | @Override 36 | public FiberFlow keyBy(int idx) 37 | { 38 | this.keyIdx = idx; 39 | return this; 40 | } 41 | 42 | @Override 43 | public FiberFlow timeBy(int idx) 44 | { 45 | this.timeIdx = idx; 46 | return this; 47 | } 48 | 49 | @Override 50 | public FiberFlow sink(DataSink dataSink) 51 | { 52 | this.dataSink = dataSink; 53 | return this; 54 | } 55 | 56 | @Override 57 | public DataSink getSink() 58 | { 59 | return dataSink; 60 | } 61 | 62 | @Override 63 | public FiberFlow partitionBy(ParaflowFiberPartitioner partitioner) 64 | { 65 | this.partitioner = partitioner; 66 | return this; 67 | } 68 | 69 | @Override 70 | public ParaflowFiberPartitioner getPartitioner() 71 | { 72 | return this.partitioner; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/java/cn/edu/ruc/iir/paraflow/http/server/model/DynamicJson.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.http.server.model; 2 | 3 | import java.util.Arrays; 4 | 5 | public class DynamicJson 6 | { 7 | private int rowCount; 8 | private int colCount; 9 | private Object data; 10 | private String[] column; 11 | 12 | public DynamicJson() 13 | { 14 | } 15 | 16 | public DynamicJson(int rowCount, int colCount, Object data, String[] column) 17 | { 18 | this.rowCount = rowCount; 19 | this.colCount = colCount; 20 | this.data = data; 21 | this.column = column; 22 | } 23 | 24 | public int getRowCount() 25 | { 26 | return rowCount; 27 | } 28 | 29 | public void setRowCount(int rowCount) 30 | { 31 | this.rowCount = rowCount; 32 | } 33 | 34 | public int getColCount() 35 | { 36 | return colCount; 37 | } 38 | 39 | public void setColCount(int colCount) 40 | { 41 | this.colCount = colCount; 42 | } 43 | 44 | public Object getData() 45 | { 46 | return data; 47 | } 48 | 49 | public void setData(Object data) 50 | { 51 | this.data = data; 52 | } 53 | 54 | public String[] getColumn() 55 | { 56 | return column; 57 | } 58 | 59 | public void setColumn(String[] column) 60 | { 61 | this.column = column; 62 | } 63 | 64 | @Override 65 | public String toString() 66 | { 67 | return "DynamicJson{" + 68 | "rowCount=" + rowCount + 69 | ", colCount=" + colCount + 70 | ", data=" + data + 71 | ", column=" + Arrays.toString(column) + 72 | '}'; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/ConfigFileNotFoundException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * ParaFlow 7 | * 8 | * @author guodong 9 | */ 10 | public final class ConfigFileNotFoundException extends ParaFlowException 11 | { 12 | private static final long serialVersionUID = 6980171393859140121L; 13 | 14 | private final String configPath; 15 | 16 | public ConfigFileNotFoundException(String configPath) 17 | { 18 | this.configPath = configPath; 19 | } 20 | 21 | /** 22 | * get error message. 23 | * 24 | * @return error message 25 | */ 26 | @Override 27 | public String getMessage() 28 | { 29 | return String.format("Configuration file %s not found", this.configPath); 30 | } 31 | 32 | @Override 33 | public StatusProto.ResponseStatus getResponseStatus() 34 | { 35 | return StatusProto.ResponseStatus 36 | .newBuilder() 37 | .setStatus(StatusProto.ResponseStatus.State.CONFIG_FILE_NOT_FOUND_FATAL) 38 | .build(); 39 | } 40 | 41 | /** 42 | * get system hint message for user on how to deal with this exception 43 | * 44 | * @return hint message 45 | */ 46 | @Override 47 | public String getHint() 48 | { 49 | return "Use default configuration properties set by user or in system instead."; 50 | } 51 | 52 | /** 53 | * get exception level 54 | * 55 | * @return exception level 56 | */ 57 | @Override 58 | public ParaFlowExceptionLevel getLevel() 59 | { 60 | return ParaFlowExceptionLevel.FATAL; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/RPCServerIOException.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 4 | 5 | /** 6 | * This exception is intended to be used in paraflow-metaserver. 7 | * Representing for gRPC server IOExpception when starting up. 8 | */ 9 | public final class RPCServerIOException extends ParaFlowException 10 | { 11 | private static final long serialVersionUID = 5621165837126253248L; 12 | 13 | private final int port; 14 | 15 | public RPCServerIOException(int port) 16 | { 17 | this.port = port; 18 | } 19 | 20 | /** 21 | * get error message. 22 | * 23 | * @return error message 24 | */ 25 | @Override 26 | public String getMessage() 27 | { 28 | return String.format("gRPC server cannot start at port %d", port); 29 | } 30 | 31 | @Override 32 | public StatusProto.ResponseStatus getResponseStatus() 33 | { 34 | return StatusProto.ResponseStatus 35 | .newBuilder() 36 | .setStatus(StatusProto.ResponseStatus.State.RPC_SERVER_IO_FATAL) 37 | .build(); 38 | } 39 | 40 | /** 41 | * get system hint message for user on how to deal with this exception 42 | * 43 | * @return hint message 44 | */ 45 | @Override 46 | public String getHint() 47 | { 48 | return "Server stops. Check if port is already used and try again."; 49 | } 50 | 51 | /** 52 | * get exception level 53 | * 54 | * @return exception level 55 | */ 56 | @Override 57 | public ParaFlowExceptionLevel getLevel() 58 | { 59 | return ParaFlowExceptionLevel.FATAL; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /paraflow-connector/src/main/java/cn/edu/ruc/iir/paraflow/connector/exception/ParaflowErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.connector.exception; 15 | 16 | import com.facebook.presto.spi.ErrorCode; 17 | import com.facebook.presto.spi.ErrorCodeSupplier; 18 | import com.facebook.presto.spi.ErrorType; 19 | 20 | import static com.facebook.presto.spi.ErrorType.EXTERNAL; 21 | 22 | /** 23 | * @author jelly.guodong.jin@gmail.com 24 | */ 25 | public enum ParaflowErrorCode 26 | implements ErrorCodeSupplier 27 | { 28 | CONNECTOR_INIT_ERROR(0, ErrorType.INTERNAL_ERROR), 29 | CONNECTOR_SHUTDOWN_ERROR(1, ErrorType.INTERNAL_ERROR), 30 | TABLE_NOT_FOUND(4, EXTERNAL), 31 | FUNCTION_UNSUPPORTED(8, EXTERNAL), 32 | HDFS_CURSOR_ERROR(6, EXTERNAL), 33 | PARAFLOW_HDFS_FILE_ERROR(37, EXTERNAL), 34 | PARAFLOW_CONFIG_ERROR(38, EXTERNAL), 35 | HDFS_SPLIT_NOT_OPEN(5, EXTERNAL); 36 | 37 | private final ErrorCode errorCode; 38 | 39 | ParaflowErrorCode(int code, ErrorType type) 40 | { 41 | errorCode = new ErrorCode(code + 0x0210_0000, name(), type); 42 | } 43 | 44 | @Override 45 | public ErrorCode toErrorCode() 46 | { 47 | return errorCode; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /paraflow-1.0-alpha1/sbin/zookeeper-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # init|start|stop|clean|destroy the kafka cluster 3 | 4 | echo $1" the zookeeper cluster." 5 | 6 | PREFIX="dbiir" 7 | START=2 8 | END=6 9 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 10 | ZK_HOME="/home/iir/opt/zookeeper" 11 | 12 | startup() 13 | { 14 | for ((i=$START; i<=$END; i++)) 15 | do 16 | echo "starting zookeeper on dbiir0"$i 17 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh start" 18 | done 19 | } 20 | 21 | stop() 22 | { 23 | for ((i=$START; i<=$END; i++)) 24 | do 25 | echo "stopping zookeeper on dbiir0"$i 26 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh stop" 27 | done 28 | } 29 | 30 | deploy() 31 | { 32 | if [ $(id -u) != "0" ]; then 33 | echo "please run $0 in root." 34 | exit 0 35 | fi 36 | for ((i=$START; i<=$END; i++)) 37 | do 38 | echo "deploy zookeeper on dbiir0"$i 39 | scp -r $ZK_DIR $PREFIX"0"$i:$ZK_DIR 40 | ssh $PREFIX"0"$i "chown iir:iir $ZK_DIR" 41 | ssh $PREFIX"0"$i "$ZK_DIR/sbin/zookeeper-init.sh" 42 | done 43 | } 44 | 45 | cleanup() 46 | { 47 | for ((i=$START; i<=$END; i++)) 48 | do 49 | echo "destroy kafka on dbiir0"$i 50 | ssh $PREFIX"0"$i "rm -rf /data1/zookeeper/logs" 51 | done 52 | } 53 | 54 | destroy() 55 | { 56 | for ((i=$START; i<=$END; i++)) 57 | do 58 | echo "destroy kafka on dbiir0"$i 59 | ssh $PREFIX"0"$i "rm -rf $ZK_HOME/ && rm -rf $ZK_HOME && rm -rf $ZK_DIR && rm -rf /data1/zookeeper" 60 | done 61 | } 62 | 63 | if [ "$1" == "start" ]; then 64 | startup 65 | elif [ "$1" == "stop" ]; then 66 | stop 67 | elif [ "$1" == "deploy" ]; then 68 | deploy 69 | elif [ "$1" == "clean" ]; then 70 | cleanup 71 | elif [ "$1" == "destroy" ]; then 72 | destroy 73 | else 74 | echo "Usage: $0 deploy|start|stop|clean|destroy" 75 | fi 76 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/sbin/zookeeper-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # init|start|stop|clean|destroy the kafka cluster 3 | 4 | echo $1" the zookeeper cluster." 5 | 6 | PREFIX="dbiir" 7 | START=2 8 | END=6 9 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 10 | ZK_HOME="/home/iir/opt/zookeeper" 11 | 12 | startup() 13 | { 14 | for ((i=$START; i<=$END; i++)) 15 | do 16 | echo "starting zookeeper on dbiir0"$i 17 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh start" 18 | done 19 | } 20 | 21 | stop() 22 | { 23 | for ((i=$START; i<=$END; i++)) 24 | do 25 | echo "stopping zookeeper on dbiir0"$i 26 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh stop" 27 | done 28 | } 29 | 30 | deploy() 31 | { 32 | if [ $(id -u) != "0" ]; then 33 | echo "please run $0 in root." 34 | exit 0 35 | fi 36 | for ((i=$START; i<=$END; i++)) 37 | do 38 | echo "deploy zookeeper on dbiir0"$i 39 | scp -r $ZK_DIR $PREFIX"0"$i:$ZK_DIR 40 | ssh $PREFIX"0"$i "chown iir:iir $ZK_DIR" 41 | ssh $PREFIX"0"$i "$ZK_DIR/sbin/zookeeper-init.sh" 42 | done 43 | } 44 | 45 | cleanup() 46 | { 47 | for ((i=$START; i<=$END; i++)) 48 | do 49 | echo "destroy kafka on dbiir0"$i 50 | ssh $PREFIX"0"$i "rm -rf /data1/zookeeper/logs" 51 | done 52 | } 53 | 54 | destroy() 55 | { 56 | for ((i=$START; i<=$END; i++)) 57 | do 58 | echo "destroy kafka on dbiir0"$i 59 | ssh $PREFIX"0"$i "rm -rf $ZK_HOME/ && rm -rf $ZK_HOME && rm -rf $ZK_DIR && rm -rf /data1/zookeeper" 60 | done 61 | } 62 | 63 | if [ "$1" == "start" ]; then 64 | startup 65 | elif [ "$1" == "stop" ]; then 66 | stop 67 | elif [ "$1" == "deploy" ]; then 68 | deploy 69 | elif [ "$1" == "clean" ]; then 70 | cleanup 71 | elif [ "$1" == "destroy" ]; then 72 | destroy 73 | else 74 | echo "Usage: $0 deploy|start|stop|clean|destroy" 75 | fi 76 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/connection/TransactionController.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.connection; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 4 | import cn.edu.ruc.iir.paraflow.metaserver.action.Action; 5 | import cn.edu.ruc.iir.paraflow.metaserver.action.ActionResponse; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | public class TransactionController 11 | { 12 | private final Connection connection; 13 | private final List actions; 14 | private boolean autoCommit = true; 15 | 16 | public TransactionController(Connection connection) 17 | { 18 | this.connection = connection; 19 | this.actions = new ArrayList<>(); 20 | } 21 | 22 | public void setAutoCommit(boolean autoCommit) 23 | { 24 | this.autoCommit = autoCommit; 25 | } 26 | 27 | public void addAction(Action action) 28 | { 29 | actions.add(action); 30 | } 31 | 32 | public ActionResponse commit() throws ParaFlowException 33 | { 34 | ActionResponse response = new ActionResponse(); 35 | commit(response); 36 | return response; 37 | } 38 | 39 | public ActionResponse commit(ActionResponse response) throws ParaFlowException 40 | { 41 | connection.setAutoCommit(autoCommit); 42 | for (Action action : actions) { 43 | response = action.act(response, this.connection); 44 | } 45 | if (!autoCommit) { 46 | connection.commit(); 47 | } 48 | return response; 49 | } 50 | 51 | public void close() 52 | { 53 | try { 54 | connection.close(); 55 | } 56 | catch (ParaFlowException e) { 57 | e.printStackTrace(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /paraflow-http-server/src/main/resources/static/tpl/tplTable.htm: -------------------------------------------------------------------------------- 1 | <% 2 | for( var i=0;i< tplData.length;i++){ 3 | var aItem = tplData[i]; 4 | %> 5 | 6 |
7 |
8 |

<%=aItem.name %>

9 |
10 |
11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | <% 21 | for( var j=0;j< aItem.columns.length;j++){ 22 | var aColumn = aItem.columns[j]; 23 | var k = j+1; 24 | %> 25 | 26 | 29 | 34 | 37 | 38 | <%} %> 39 | <% if ( aItem.columns.length<=0){ %> 40 | 41 | 44 | 45 | <%} %> 46 |
IdColumn NameData TypeComment
27 | <%=k %> 28 | 30 | <%=aColumn.name %> 31 | 32 | <%=aColumn.type %> 33 | 35 | 36 |
42 | no column. 43 |
47 |
48 |
49 | 50 | <%} %> 51 | 52 | <% if ( tplData.length<=0){ %> 53 |
54 |
55 |

no table.

56 |
57 |
58 | <%} %> 59 | -------------------------------------------------------------------------------- /paraflow-1.0-config/paraflow/paraflow-1.0-alpha1/sbin/zookeeper-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # init|start|stop|clean|destroy the kafka cluster 3 | 4 | echo $1" the zookeeper cluster." 5 | 6 | PREFIX="dbiir" 7 | START=2 8 | END=6 9 | ZK_DIR="/home/iir/opt/zookeeper-3.4.13" 10 | ZK_HOME="/home/iir/opt/zookeeper" 11 | 12 | startup() 13 | { 14 | for ((i=$START; i<=$END; i++)) 15 | do 16 | echo "starting zookeeper on dbiir0"$i 17 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh start" 18 | done 19 | } 20 | 21 | stop() 22 | { 23 | for ((i=$START; i<=$END; i++)) 24 | do 25 | echo "stopping zookeeper on dbiir0"$i 26 | ssh $PREFIX"0"$i "/home/iir/opt/zookeeper/bin/zkServer.sh stop" 27 | done 28 | } 29 | 30 | deploy() 31 | { 32 | if [ $(id -u) != "0" ]; then 33 | echo "please run $0 in root." 34 | exit 0 35 | fi 36 | for ((i=$START; i<=$END; i++)) 37 | do 38 | echo "deploy zookeeper on dbiir0"$i 39 | scp -r $ZK_DIR $PREFIX"0"$i:$ZK_DIR 40 | ssh $PREFIX"0"$i "chown iir:iir $ZK_DIR" 41 | ssh $PREFIX"0"$i "$ZK_DIR/sbin/zookeeper-init.sh" 42 | done 43 | } 44 | 45 | cleanup() 46 | { 47 | for ((i=$START; i<=$END; i++)) 48 | do 49 | echo "destroy kafka on dbiir0"$i 50 | ssh $PREFIX"0"$i "rm -rf /data1/zookeeper/logs" 51 | done 52 | } 53 | 54 | destroy() 55 | { 56 | for ((i=$START; i<=$END; i++)) 57 | do 58 | echo "destroy kafka on dbiir0"$i 59 | ssh $PREFIX"0"$i "rm -rf $ZK_HOME/ && rm -rf $ZK_HOME && rm -rf $ZK_DIR && rm -rf /data1/zookeeper" 60 | done 61 | } 62 | 63 | if [ "$1" == "start" ]; then 64 | startup 65 | elif [ "$1" == "stop" ]; then 66 | stop 67 | elif [ "$1" == "deploy" ]; then 68 | deploy 69 | elif [ "$1" == "clean" ]; then 70 | cleanup 71 | elif [ "$1" == "destroy" ]; then 72 | destroy 73 | else 74 | echo "Usage: $0 deploy|start|stop|clean|destroy" 75 | fi 76 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/CreateDatabaseAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.DatabaseCreationException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.proto.MetaProto; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | import cn.edu.ruc.iir.paraflow.metaserver.utils.Utils; 10 | 11 | import java.util.Optional; 12 | 13 | public class CreateDatabaseAction extends Action 14 | { 15 | @Override 16 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 17 | { 18 | Optional param = input.getParam(); 19 | Optional userIdOp = input.getProperties("userId"); 20 | if (param.isPresent() && userIdOp.isPresent()) { 21 | MetaProto.DbParam dbParam = (MetaProto.DbParam) param.get(); 22 | String locationUrl = dbParam.getLocationUrl(); 23 | if (locationUrl.isEmpty()) { 24 | locationUrl = Utils.formatDbUrl(dbParam.getDbName()); 25 | } 26 | String userStatement = SQLTemplate.createDatabase( 27 | dbParam.getDbName(), 28 | (long) userIdOp.get(), 29 | locationUrl); 30 | int status = connection.executeUpdate(userStatement); 31 | if (status == 0) { 32 | throw new DatabaseCreationException(); 33 | } 34 | } 35 | else { 36 | throw new ActionParamNotValidException(); 37 | } 38 | return input; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /paraflow-metaserver/src/main/java/cn/edu/ruc/iir/paraflow/metaserver/action/GetTableIdAction.java: -------------------------------------------------------------------------------- 1 | package cn.edu.ruc.iir.paraflow.metaserver.action; 2 | 3 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ActionParamNotValidException; 4 | import cn.edu.ruc.iir.paraflow.commons.exceptions.ParaFlowException; 5 | import cn.edu.ruc.iir.paraflow.commons.exceptions.TableNotFoundException; 6 | import cn.edu.ruc.iir.paraflow.metaserver.connection.Connection; 7 | import cn.edu.ruc.iir.paraflow.metaserver.connection.ResultList; 8 | import cn.edu.ruc.iir.paraflow.metaserver.utils.SQLTemplate; 9 | 10 | import java.util.Optional; 11 | 12 | /** 13 | * paraflow 14 | * 15 | * @author guodong 16 | */ 17 | public class GetTableIdAction extends Action 18 | { 19 | @Override 20 | public ActionResponse act(ActionResponse input, Connection connection) throws ParaFlowException 21 | { 22 | Optional paramOp = input.getParam(); 23 | Optional dbIdOp = input.getProperties("dbId"); 24 | Optional tblNameOp = input.getProperties("tblName"); 25 | if (paramOp.isPresent() 26 | && dbIdOp.isPresent() 27 | && tblNameOp.isPresent()) { 28 | long dbId = (long) dbIdOp.get(); 29 | String tblName = tblNameOp.get().toString(); 30 | String sqlStatement = SQLTemplate.findTblId(dbId, tblName); 31 | ResultList resultList = connection.executeQuery(sqlStatement); 32 | if (!resultList.isEmpty()) { 33 | input.setProperties("tblId", Long.parseLong(resultList.get(0).get(0))); 34 | } 35 | else { 36 | throw new TableNotFoundException(tblName); 37 | } 38 | } 39 | else { 40 | throw new ActionParamNotValidException(); 41 | } 42 | return input; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/TableRenameException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 15 | 16 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 17 | 18 | public class TableRenameException extends ParaFlowException 19 | { 20 | private static final long serialVersionUID = -8022436969319033821L; 21 | 22 | @Override 23 | public String getMessage() 24 | { 25 | return "table rename exception"; 26 | } 27 | 28 | @Override 29 | public StatusProto.ResponseStatus getResponseStatus() 30 | { 31 | return StatusProto.ResponseStatus.newBuilder() 32 | .setStatus(StatusProto.ResponseStatus.State.TABLE_RENAME_ERROR) 33 | .build(); 34 | } 35 | 36 | /** 37 | * get system hint message for user on how to deal with this exception 38 | * 39 | * @return hint message 40 | */ 41 | @Override 42 | public String getHint() 43 | { 44 | return "Check and try again"; 45 | } 46 | 47 | /** 48 | * get exception level 49 | * 50 | * @return exception level 51 | */ 52 | @Override 53 | public ParaFlowExceptionLevel getLevel() 54 | { 55 | return ParaFlowExceptionLevel.ERROR; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /paraflow-commons/src/main/java/cn/edu/ruc/iir/paraflow/commons/exceptions/ColumnRenameException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package cn.edu.ruc.iir.paraflow.commons.exceptions; 15 | 16 | import cn.edu.ruc.iir.paraflow.commons.proto.StatusProto; 17 | 18 | public class ColumnRenameException extends ParaFlowException 19 | { 20 | private static final long serialVersionUID = -7288943929416471257L; 21 | 22 | @Override 23 | public String getMessage() 24 | { 25 | return "Column rename exception"; 26 | } 27 | 28 | @Override 29 | public StatusProto.ResponseStatus getResponseStatus() 30 | { 31 | return StatusProto.ResponseStatus 32 | .newBuilder() 33 | .setStatus(StatusProto.ResponseStatus.State.COLUMN_RENAME_ERROR) 34 | .build(); 35 | } 36 | 37 | /** 38 | * get system hint message for user on how to deal with this exception 39 | * 40 | * @return hint message 41 | */ 42 | @Override 43 | public String getHint() 44 | { 45 | return "Check and try again"; 46 | } 47 | 48 | /** 49 | * get exception level 50 | * 51 | * @return exception level 52 | */ 53 | @Override 54 | public ParaFlowExceptionLevel getLevel() 55 | { 56 | return ParaFlowExceptionLevel.ERROR; 57 | } 58 | } 59 | --------------------------------------------------------------------------------