├── .gitignore ├── ldbc-converter ├── .gitignore ├── README.md ├── configurations │ ├── comment │ │ ├── comment_10.yaml │ │ ├── comment_2.yaml │ │ └── comment_5.yaml │ ├── forum.yaml │ ├── forum_person.yaml │ ├── person │ │ ├── person_10.yaml │ │ ├── person_2.yaml │ │ └── person_5.yaml │ └── post │ │ ├── post_10.yaml │ │ ├── post_2.yaml │ │ └── post_5.yaml ├── dataset-generator.py ├── generated-queries │ ├── ddl │ │ ├── analyze.sql │ │ ├── load.sql │ │ ├── schema_constraints.sql │ │ ├── schema_foreign_keys.sql │ │ └── schemas.sql │ ├── union-all │ │ ├── comment.sql │ │ ├── forum.sql │ │ ├── forum_person.sql │ │ ├── message.sql │ │ ├── person.sql │ │ └── post.sql │ └── workload │ │ ├── interactive-complex-2.sql │ │ ├── interactive-complex-3.sql │ │ ├── interactive-complex-4.sql │ │ ├── interactive-short-1.sql │ │ ├── interactive-short-2.sql │ │ ├── interactive-short-3.sql │ │ ├── interactive-short-4.sql │ │ ├── interactive-short-5-no-filter.sql │ │ ├── interactive-short-5.sql │ │ ├── interactive-short-6.sql │ │ ├── interactive-short-7.sql │ │ └── test.sql ├── generator-script.sh ├── ldbc.sh ├── original-dataset.png ├── original-queries │ ├── ddl │ │ ├── analyze.sql │ │ ├── load.sql │ │ ├── schema_constraints.sql │ │ ├── schema_foreign_keys.sql │ │ └── schemas.sql │ └── workload │ │ ├── interactive-complex-1.sql │ │ ├── interactive-complex-10.sql │ │ ├── interactive-complex-11.sql │ │ ├── interactive-complex-12.sql │ │ ├── interactive-complex-13.sql │ │ ├── interactive-complex-14.sql │ │ ├── interactive-complex-2.sql │ │ ├── interactive-complex-3.sql │ │ ├── interactive-complex-4.sql │ │ ├── interactive-complex-5.sql │ │ ├── interactive-complex-6.sql │ │ ├── interactive-complex-7.sql │ │ ├── interactive-complex-8.sql │ │ ├── interactive-complex-9.sql │ │ ├── interactive-short-1.sql │ │ ├── interactive-short-2.sql │ │ ├── interactive-short-3.sql │ │ ├── interactive-short-4.sql │ │ ├── interactive-short-5.sql │ │ ├── interactive-short-6.sql │ │ └── interactive-short-7.sql ├── plan-experiments │ ├── error.txt │ ├── extract_plan.sh │ ├── original │ │ └── short-2.txt │ ├── result.txt │ └── schemaless │ │ ├── complex-2.txt │ │ ├── complex-4.txt │ │ ├── pg_stats.txt │ │ ├── short-1.txt │ │ ├── short-2.txt │ │ ├── short-3.txt │ │ ├── short-4.txt │ │ ├── short-5.txt │ │ └── short-7.txt ├── query-generator.py ├── results │ ├── analyze.err │ ├── analyze.log │ ├── bench.log │ ├── constraints.err │ ├── constraints.log │ ├── create.err │ ├── create.log │ ├── foreign.err │ ├── foreign.log │ ├── load.err │ ├── load.log │ ├── settings.err │ └── settings.log ├── run-ldbc-example.sh ├── schemas │ ├── comment.yaml │ ├── forum.yaml │ ├── forum_person.yaml │ ├── organisation.yaml │ ├── person.yaml │ ├── place.yaml │ ├── post.yaml │ ├── tag.yaml │ └── tagClass.yaml └── utils.py └── tpch-converter ├── .gitignore ├── README.md ├── configurations └── tpch │ ├── lineitem_1.yaml │ ├── lineitem_2.yaml │ ├── lineitem_3.yaml │ ├── lineitem_4.yaml │ ├── lineitem_5.yaml │ ├── lineitem_6.yaml │ ├── lineitem_7.yaml │ ├── sample_lineitem_conf.yaml │ ├── sample_supplier_conf.yaml │ └── schemaless │ ├── lineitem_10.yaml │ ├── lineitem_11.yaml │ ├── lineitem_12.yaml │ ├── lineitem_13.yaml │ ├── lineitem_14.yaml │ ├── lineitem_5.yaml │ ├── lineitem_6.yaml │ ├── lineitem_7.yaml │ ├── lineitem_8.yaml │ └── lineitem_9.yaml ├── execution_time_calculator.py ├── generator.py ├── queryModifier.py ├── runner.sh ├── runner_test_only.sh ├── schemas └── tpch │ ├── lineitem.yaml │ └── supplier.yaml ├── sqlFilesGenerator.py ├── templates └── tpch │ ├── alter │ ├── customer.sql │ ├── lineitem.sql │ ├── nation.sql │ ├── orders.sql │ ├── part.sql │ ├── partsupp.sql │ ├── region.sql │ └── supplier.sql │ ├── analyze │ ├── customer.sql │ ├── lineitem.sql │ ├── nation.sql │ ├── orders.sql │ ├── part.sql │ ├── partsupp.sql │ ├── region.sql │ └── supplier.sql │ ├── index │ ├── customer.sql │ ├── lineitem.sql │ ├── nation.sql │ ├── orders.sql │ ├── part.sql │ ├── partsupp.sql │ ├── region.sql │ └── supplier.sql │ ├── load │ ├── customer.sql │ ├── lineitem.sql │ ├── nation.sql │ ├── orders.sql │ ├── part.sql │ ├── partsupp.sql │ ├── region.sql │ └── supplier.sql │ └── pkeys │ ├── customer.sql │ ├── lineitem.sql │ ├── nation.sql │ ├── orders.sql │ ├── part.sql │ ├── partsupp.sql │ ├── region.sql │ └── supplier.sql ├── test.sh └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | data/* 2 | results/* 3 | queries/* 4 | *.pyc 5 | execution_average_result.txt -------------------------------------------------------------------------------- /ldbc-converter/.gitignore: -------------------------------------------------------------------------------- 1 | original-dataset/* 2 | generated-dataset/* -------------------------------------------------------------------------------- /ldbc-converter/README.md: -------------------------------------------------------------------------------- 1 | # schemaless-data-generator 2 | 3 | For the design description, see [Schemaless Data Experiment on Orca](https://docs.google.com/document/d/1R7ENQvLVNHQ-DG-sga0tfgGWJIkobQT77HQFVKUvxH8/edit#). 4 | 5 | ## Setup 6 | 7 | 1. Create original-dataset folder 8 | 2. Copy dataset into the folder (dynamic, static folder) 9 | 10 | ![Original Dataset](./original-dataset.png) 11 | 12 | ## Folder Structure 13 | 14 | ``` 15 | ./configuration: yaml files 16 | ./schemas: schema yaml files 17 | ``` 18 | 19 | ## Bash files 20 | 21 | ``` 22 | runner.sh: move generated files and run the experiment. 23 | ``` 24 | 25 | ## Configuration Files 26 | ``` 27 | table_name: 28 | schema_file: 29 | data_file: 30 | output_folders_path: 31 | partitions: 32 | 33 | # for each partition 34 | : 35 | portion: 36 | column_removing: 37 | activate: 38 | columns: 39 | ``` 40 | 41 | ## Usage 42 | ``` 43 | python dataset-generator.py 44 | ``` -------------------------------------------------------------------------------- /ldbc-converter/configurations/comment/comment_10.yaml: -------------------------------------------------------------------------------- 1 | table_name: comment 2 | schema_file: ./schemas/comment.yaml 3 | data_file_path: ./original-dataset/dynamic/comment_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] 6 | p1: 7 | portion: 10 8 | column_removing: 9 | activate: false 10 | columns: [ 11 | ] 12 | p2: 13 | portion: 10 14 | column_removing: 15 | activate: true 16 | columns: [ 17 | m_browserused, 18 | m_content, 19 | m_length, 20 | ] 21 | p3: 22 | portion: 10 23 | column_removing: 24 | activate: true 25 | columns: [ 26 | m_length, 27 | m_creatorid, 28 | m_locationid, 29 | ] 30 | p4: 31 | portion: 10 32 | column_removing: 33 | activate: true 34 | columns: [ 35 | m_locationid, 36 | m_replyof_post, 37 | m_replyof_comment 38 | ] 39 | p5: 40 | portion: 10 41 | column_removing: 42 | activate: true 43 | columns: [ 44 | m_creatorid, 45 | m_locationid, 46 | m_replyof_post, 47 | m_replyof_comment 48 | ] 49 | p6: 50 | portion: 10 51 | column_removing: 52 | activate: true 53 | columns: [ 54 | m_content, 55 | m_length, 56 | m_creatorid, 57 | m_locationid, 58 | ] 59 | p7: 60 | portion: 10 61 | column_removing: 62 | activate: true 63 | columns: [ 64 | m_locationip, 65 | m_browserused, 66 | m_content, 67 | m_length, 68 | ] 69 | p8: 70 | portion: 10 71 | column_removing: 72 | activate: true 73 | columns: [ 74 | m_browserused, 75 | m_content, 76 | m_length, 77 | m_creatorid, 78 | m_locationid, 79 | ] 80 | p9: 81 | portion: 10 82 | column_removing: 83 | activate: true 84 | columns: [ 85 | m_creationdate, 86 | m_locationip, 87 | m_browserused, 88 | m_content, 89 | m_length, 90 | ] 91 | p10: 92 | portion: 10 93 | column_removing: 94 | activate: true 95 | columns: [ 96 | m_length, 97 | m_creatorid, 98 | m_locationid, 99 | m_replyof_post, 100 | m_replyof_comment 101 | ] 102 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/comment/comment_2.yaml: -------------------------------------------------------------------------------- 1 | table_name: comment 2 | schema_file: ./schemas/comment.yaml 3 | data_file_path: ./original-dataset/dynamic/comment_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2] 6 | p1: 7 | portion: 50 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | m_locationid, 12 | m_replyof_post, 13 | m_replyof_comment 14 | ] 15 | p2: 16 | portion: 50 17 | column_removing: 18 | activate: true 19 | columns: [ 20 | m_length, 21 | m_creatorid, 22 | m_locationid, 23 | ] 24 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/comment/comment_5.yaml: -------------------------------------------------------------------------------- 1 | table_name: comment 2 | schema_file: ./schemas/comment.yaml 3 | data_file_path: ./original-dataset/dynamic/comment_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5] 6 | p1: 7 | portion: 20 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | m_replyof_post, 12 | m_replyof_comment 13 | ] 14 | p2: 15 | portion: 20 16 | column_removing: 17 | activate: true 18 | columns: [ 19 | m_creatorid, 20 | m_locationid, 21 | ] 22 | p3: 23 | portion: 20 24 | column_removing: 25 | activate: true 26 | columns: [ 27 | m_content, 28 | m_length, 29 | ] 30 | p4: 31 | portion: 20 32 | column_removing: 33 | activate: true 34 | columns: [ 35 | m_browserused, 36 | m_content, 37 | m_length, 38 | ] 39 | p5: 40 | portion: 20 41 | column_removing: 42 | activate: true 43 | columns: [ 44 | m_length, 45 | m_creatorid, 46 | m_locationid, 47 | ] 48 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/forum.yaml: -------------------------------------------------------------------------------- 1 | table_name: forum 2 | schema_file: ./schemas/forum.yaml 3 | data_file_path: ./original-dataset/dynamic/forum_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2] 6 | p1: 7 | portion: 50 8 | column_removing: 9 | activate: true 10 | columns: [f_creationdate] 11 | p2: 12 | portion: 50 13 | column_removing: 14 | activate: true 15 | columns: [f_moderatorid] 16 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/forum_person.yaml: -------------------------------------------------------------------------------- 1 | table_name: forum_person 2 | schema_file: ./schemas/forum_person.yaml 3 | data_file_path: ./original-dataset/dynamic/forum_hasMember_person_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2] 6 | p1: 7 | portion: 50 8 | column_removing: 9 | activate: false 10 | p2: 11 | portion: 50 12 | column_removing: 13 | activate: true 14 | columns: [fp_joindate] 15 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/person/person_10.yaml: -------------------------------------------------------------------------------- 1 | table_name: person 2 | schema_file: ./schemas/person.yaml 3 | data_file_path: ./original-dataset/dynamic/person_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] 6 | p1: 7 | portion: 10 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | p_browserused, 12 | p_placeid 13 | ] 14 | p2: 15 | portion: 10 16 | column_removing: 17 | activate: true 18 | columns: [ 19 | p_locationip, 20 | p_browserused, 21 | ] 22 | p3: 23 | portion: 10 24 | column_removing: 25 | activate: true 26 | columns: [ 27 | p_creationdate, 28 | p_locationip, 29 | ] 30 | p4: 31 | portion: 10 32 | column_removing: 33 | activate: true 34 | columns: [ 35 | p_birthday, 36 | p_creationdate, 37 | ] 38 | p5: 39 | portion: 10 40 | column_removing: 41 | activate: true 42 | columns: [ 43 | p_gender, 44 | p_birthday, 45 | ] 46 | p6: 47 | portion: 10 48 | column_removing: 49 | activate: true 50 | columns: [ 51 | p_lastname, 52 | p_gender, 53 | ] 54 | p7: 55 | portion: 10 56 | column_removing: 57 | activate: true 58 | columns: [ 59 | p_firstname, 60 | p_lastname, 61 | ] 62 | p8: 63 | portion: 10 64 | column_removing: 65 | activate: true 66 | columns: [ 67 | p_locationip, 68 | p_browserused, 69 | p_placeid 70 | ] 71 | p9: 72 | portion: 10 73 | column_removing: 74 | activate: true 75 | columns: [ 76 | p_birthday, 77 | p_creationdate, 78 | p_locationip, 79 | ] 80 | p10: 81 | portion: 10 82 | column_removing: 83 | activate: true 84 | columns: [ 85 | p_lastname, 86 | p_gender, 87 | p_birthday, 88 | ] -------------------------------------------------------------------------------- /ldbc-converter/configurations/person/person_2.yaml: -------------------------------------------------------------------------------- 1 | table_name: person 2 | schema_file: ./schemas/person.yaml 3 | data_file_path: ./original-dataset/dynamic/person_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2] 6 | p1: 7 | portion: 50 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | p_browserused, 12 | p_placeid 13 | ] 14 | p2: 15 | portion: 50 16 | column_removing: 17 | activate: true 18 | columns: [ 19 | p_creationdate, 20 | p_locationip, 21 | ] 22 | 23 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/person/person_5.yaml: -------------------------------------------------------------------------------- 1 | table_name: person 2 | schema_file: ./schemas/person.yaml 3 | data_file_path: ./original-dataset/dynamic/person_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5] 6 | p1: 7 | portion: 20 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | p_browserused, 12 | p_placeid 13 | ] 14 | p2: 15 | portion: 20 16 | column_removing: 17 | activate: true 18 | columns: [ 19 | p_locationip, 20 | p_browserused 21 | ] 22 | p3: 23 | portion: 20 24 | column_removing: 25 | activate: true 26 | columns: [ 27 | p_creationdate, 28 | p_locationip 29 | ] 30 | p4: 31 | portion: 20 32 | column_removing: 33 | activate: true 34 | columns: [ 35 | p_birthday, 36 | p_creationdate 37 | ] 38 | p5: 39 | portion: 20 40 | column_removing: 41 | activate: true 42 | columns: [ 43 | p_gender, 44 | p_birthday 45 | ] 46 | 47 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/post/post_10.yaml: -------------------------------------------------------------------------------- 1 | table_name: post 2 | schema_file: ./schemas/post.yaml 3 | data_file_path: ./original-dataset/dynamic/post_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] 6 | p1: 7 | portion: 10 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | m_ps_forumid, 12 | m_locationid 13 | ] 14 | p2: 15 | portion: 10 16 | column_removing: 17 | activate: true 18 | columns: [ 19 | m_length, 20 | m_creatorid, 21 | ] 22 | p3: 23 | portion: 10 24 | column_removing: 25 | activate: true 26 | columns: [ 27 | m_ps_language, 28 | m_content, 29 | ] 30 | p4: 31 | portion: 10 32 | column_removing: 33 | activate: true 34 | columns: [ 35 | m_locationip, 36 | m_browserused, 37 | ] 38 | p5: 39 | portion: 10 40 | column_removing: 41 | activate: true 42 | columns: [ 43 | m_ps_imagefile, 44 | m_creationdate, 45 | ] 46 | p6: 47 | portion: 10 48 | column_removing: 49 | activate: true 50 | columns: [ 51 | m_creatorid, 52 | m_ps_forumid, 53 | m_locationid 54 | ] 55 | p7: 56 | portion: 10 57 | column_removing: 58 | activate: true 59 | columns: [ 60 | m_ps_language, 61 | m_content, 62 | m_length, 63 | ] 64 | p8: 65 | portion: 10 66 | column_removing: 67 | activate: true 68 | columns: [ 69 | m_creationdate, 70 | m_locationip, 71 | m_browserused, 72 | ] 73 | p9: 74 | portion: 10 75 | column_removing: 76 | activate: true 77 | columns: [ 78 | m_length, 79 | m_creatorid, 80 | m_ps_forumid, 81 | m_locationid 82 | ] 83 | p10: 84 | portion: 10 85 | column_removing: 86 | activate: true 87 | columns: [ 88 | m_locationip, 89 | m_browserused, 90 | m_ps_language, 91 | m_content, 92 | ] 93 | -------------------------------------------------------------------------------- /ldbc-converter/configurations/post/post_2.yaml: -------------------------------------------------------------------------------- 1 | table_name: post 2 | schema_file: ./schemas/post.yaml 3 | data_file_path: ./original-dataset/dynamic/post_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2] 6 | p1: 7 | portion: 50 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | m_creatorid, 12 | m_ps_forumid, 13 | m_locationid 14 | ] 15 | p2: 16 | portion: 50 17 | column_removing: 18 | activate: true 19 | columns: [ 20 | m_ps_language, 21 | m_content, 22 | m_length, 23 | ] -------------------------------------------------------------------------------- /ldbc-converter/configurations/post/post_5.yaml: -------------------------------------------------------------------------------- 1 | table_name: post 2 | schema_file: ./schemas/post.yaml 3 | data_file_path: ./original-dataset/dynamic/post_0_0.csv 4 | output_folders_path: ./generated-dataset/dynamic 5 | partitions: [p1, p2, p3, p4, p5] 6 | p1: 7 | portion: 20 8 | column_removing: 9 | activate: true 10 | columns: [ 11 | m_creatorid, 12 | m_ps_forumid, 13 | m_locationid 14 | ] 15 | p2: 16 | portion: 20 17 | column_removing: 18 | activate: true 19 | columns: [ 20 | m_content, 21 | m_length, 22 | m_creatorid, 23 | ] 24 | p3: 25 | portion: 20 26 | column_removing: 27 | activate: true 28 | columns: [ 29 | m_browserused, 30 | m_ps_language, 31 | m_content, 32 | ] 33 | p4: 34 | portion: 20 35 | column_removing: 36 | activate: true 37 | columns: [ 38 | m_creationdate, 39 | m_locationip, 40 | m_browserused, 41 | ] 42 | p5: 43 | portion: 20 44 | column_removing: 45 | activate: true 46 | columns: [ 47 | m_ps_imagefile, 48 | m_creationdate, 49 | m_locationip, 50 | ] 51 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/ddl/analyze.sql: -------------------------------------------------------------------------------- 1 | ANALYZE; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/ddl/load.sql: -------------------------------------------------------------------------------- 1 | -- Define the base path variable 2 | \set base_path '/home/schemaless-data-generator/ldbc-converter/generated-dataset' 3 | 4 | -- Populate forum table 5 | COPY forum_p1 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/forum_0_0_p1.csv' WITH DELIMITER '|'; 6 | COPY forum_p2 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/forum_0_0_p2.csv' WITH DELIMITER '|'; 7 | 8 | -- Populate forum_person table 9 | COPY forum_person_p1 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/forum_hasMember_person_0_0_p1.csv' WITH DELIMITER '|'; 10 | COPY forum_person_p2 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/forum_hasMember_person_0_0_p2.csv' WITH DELIMITER '|'; 11 | 12 | -- Populate forum_tag table 13 | COPY forum_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/forum_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 14 | 15 | -- Populate organisation table 16 | COPY organisation FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 17 | 18 | -- Populate person table 19 | COPY person_p1 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/person_0_0_p1.csv' WITH DELIMITER '|'; 20 | COPY person_p2 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/person_0_0_p2.csv' WITH DELIMITER '|'; 21 | COPY person_p3 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/person_0_0_p3.csv' WITH DELIMITER '|'; 22 | COPY person_p4 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/person_0_0_p4.csv' WITH DELIMITER '|'; 23 | COPY person_p5 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/person_0_0_p5.csv' WITH DELIMITER '|'; 24 | 25 | -- Populate person_email table 26 | COPY person_email FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_email_emailaddress_0_0.csv' WITH DELIMITER '|' CSV HEADER; 27 | 28 | -- Populate person_tag table 29 | COPY person_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_hasInterest_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 30 | 31 | -- Populate knows table 32 | COPY knows ( k_person1id, k_person2id, k_creationdate) FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_knows_person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 33 | COPY knows ( k_person2id, k_person1id, k_creationdate) FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_knows_person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 34 | 35 | -- Populate likes table 36 | COPY likes FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_likes_post_0_0.csv' WITH DELIMITER '|' CSV HEADER; 37 | COPY likes FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_likes_comment_0_0.csv' WITH DELIMITER '|' CSV HEADER; 38 | 39 | -- Populate person_language table 40 | COPY person_language FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_speaks_language_0_0.csv' WITH DELIMITER '|' CSV HEADER; 41 | 42 | -- Populate person_university table 43 | COPY person_university FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_studyAt_organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 44 | 45 | -- Populate person_company table 46 | COPY person_company FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_workAt_organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 47 | 48 | -- Populate place table 49 | COPY place FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/place_0_0.csv' WITH DELIMITER '|' CSV HEADER; 50 | 51 | -- Populate message_tag table 52 | COPY message_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/post_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 53 | COPY message_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/comment_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 54 | 55 | -- Populate tagclass table 56 | COPY tagclass FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/tagclass_0_0.csv' WITH DELIMITER '|' CSV HEADER; 57 | 58 | -- Populate tag table 59 | COPY tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 60 | 61 | CREATE TABLE country AS 62 | SELECT city.pl_placeid AS ctry_city, ctry.pl_name AS ctry_name 63 | FROM place city, place ctry 64 | WHERE city.pl_containerplaceid = ctry.pl_placeid 65 | AND ctry.pl_type = 'country'; 66 | 67 | -- Populate posts and comments tables, union them into message 68 | COPY post_p1 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/post_0_0_p1.csv' WITH DELIMITER '|'; 69 | COPY post_p2 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/post_0_0_p2.csv' WITH DELIMITER '|'; 70 | COPY post_p3 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/post_0_0_p3.csv' WITH DELIMITER '|'; 71 | COPY post_p4 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/post_0_0_p4.csv' WITH DELIMITER '|'; 72 | COPY post_p5 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/post_0_0_p5.csv' WITH DELIMITER '|'; 73 | COPY comment_p1 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/comment_0_0_p1.csv' WITH DELIMITER '|'; 74 | COPY comment_p2 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/comment_0_0_p2.csv' WITH DELIMITER '|'; 75 | COPY comment_p3 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/comment_0_0_p3.csv' WITH DELIMITER '|'; 76 | COPY comment_p4 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/comment_0_0_p4.csv' WITH DELIMITER '|' CSV HEADER; 77 | COPY comment_p5 FROM '/home/schemaless-data-generator/ldbc-converter/generated-dataset/dynamic/comment_0_0_p5.csv' WITH DELIMITER '|' CSV HEADER; 78 | 79 | -- Note: to distinguish between "post" and "comment" records: 80 | -- - m_c_replyof IS NULL for all "post" records 81 | -- - m_c_replyof IS NOT NULL for all "comment" records 82 | -- CREATE TABLE message AS 83 | -- SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 84 | -- FROM post 85 | -- UNION ALL 86 | -- SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 87 | -- FROM comment; 88 | 89 | -- DROP TABLE post; 90 | -- DROP TABLE comment; 91 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/ddl/schema_constraints.sql: -------------------------------------------------------------------------------- 1 | -- psql would automatically build index on pk constraints. so disable it 2 | 3 | ALTER TABLE message ADD PRIMARY KEY (m_messageid); 4 | ALTER TABLE forum_p1 ADD PRIMARY KEY (f_forumid); 5 | ALTER TABLE forum_p2 ADD PRIMARY KEY (f_forumid); 6 | ALTER TABLE organisation ADD PRIMARY KEY (o_organisationid); 7 | ALTER TABLE person_p1 ADD PRIMARY KEY (p_personid); 8 | ALTER TABLE person_p2 ADD PRIMARY KEY (p_personid); 9 | ALTER TABLE person_p3 ADD PRIMARY KEY (p_personid); 10 | ALTER TABLE person_p4 ADD PRIMARY KEY (p_personid); 11 | ALTER TABLE person_p5 ADD PRIMARY KEY (p_personid); 12 | ALTER TABLE place ADD PRIMARY KEY (pl_placeid); 13 | ALTER TABLE tagclass ADD PRIMARY KEY (tc_tagclassid); 14 | ALTER TABLE tag ADD PRIMARY KEY (t_tagid); 15 | ALTER TABLE forum_person_p1 ADD PRIMARY KEY (fp_forumid, fp_personid); 16 | ALTER TABLE forum_person_p2 ADD PRIMARY KEY (fp_forumid, fp_personid); 17 | ALTER TABLE forum_tag ADD PRIMARY KEY (ft_forumid, ft_tagid); 18 | 19 | ALTER TABLE person_email ADD PRIMARY KEY (pe_personid, pe_email); 20 | ALTER TABLE person_tag ADD PRIMARY KEY (pt_personid, pt_tagid); 21 | ALTER TABLE knows ADD PRIMARY KEY (k_person1id, k_person2id); 22 | ALTER TABLE likes ADD PRIMARY KEY (l_messageid, l_personid); 23 | ALTER TABLE person_language ADD PRIMARY KEY (plang_personid, plang_language); 24 | ALTER TABLE person_university ADD PRIMARY KEY (pu_personid, pu_organisationid); 25 | ALTER TABLE person_company ADD PRIMARY KEY (pc_personid, pc_organisationid); 26 | ALTER TABLE message_tag ADD PRIMARY KEY (mt_messageid, mt_tagid); 27 | 28 | 29 | -- create index on foreign keys // index join 30 | CREATE INDEX forum_person_forumid_p1 ON forum_person_p1 (fp_forumid); 31 | CREATE INDEX forum_person_personid_p1 ON forum_person_p1 (fp_personid); 32 | CREATE INDEX forum_person_forumid_p2 ON forum_person_p1 (fp_forumid); 33 | CREATE INDEX forum_person_personid_p2 ON forum_person_p2 (fp_personid); 34 | CREATE INDEX forum_tag_forumid ON forum_tag (ft_forumid); 35 | CREATE INDEX forum_tag_tagid ON forum_tag (ft_tagid); 36 | CREATE INDEX knows_person1id ON knows (k_person1id); 37 | CREATE INDEX knows_person2id ON knows (k_person2id); 38 | CREATE INDEX likes_personid ON likes (l_personid); 39 | CREATE INDEX likes_messageid ON likes (l_messageid); 40 | CREATE INDEX organisation_placeid ON organisation (o_placeid); 41 | CREATE INDEX person_placeid_p1 ON person_p1 (p_placeid); 42 | CREATE INDEX person_placeid_p2 ON person_p2 (p_placeid); 43 | CREATE INDEX person_placeid_p3 ON person_p3 (p_placeid); 44 | CREATE INDEX person_placeid_p4 ON person_p4 (p_placeid); 45 | CREATE INDEX person_placeid_p5 ON person_p5 (p_placeid); 46 | CREATE INDEX person_company_personid ON person_company (pc_personid); 47 | CREATE INDEX person_company_organisationid ON person_company (pc_organisationid); 48 | CREATE INDEX person_email_personid ON person_email (pe_personid); 49 | CREATE INDEX person_language_personid ON person_language (plang_personid); 50 | CREATE INDEX person_tag_personid ON person_tag (pt_personid); 51 | CREATE INDEX person_tag_tagid ON person_tag (pt_tagid); 52 | CREATE INDEX person_university_personid ON person_university (pu_personid); 53 | CREATE INDEX person_university_organisationid ON person_university (pu_organisationid); 54 | CREATE INDEX place_containerplaceid ON place (pl_containerplaceid); 55 | -- CREATE INDEX message_creatorid ON message (m_creatorid); 56 | -- CREATE INDEX message_locationid ON message (m_locationid); 57 | -- CREATE INDEX message_forumid ON message (m_ps_forumid); 58 | -- CREATE INDEX message_replyof ON message (m_c_replyof); 59 | CREATE INDEX message_tag_messageid ON message_tag (mt_messageid); 60 | CREATE INDEX message_tag_tagid ON message_tag (mt_tagid); 61 | CREATE INDEX tag_tagclassid ON tag (t_tagclassid); 62 | CREATE INDEX tagclass_subclassoftagclassid ON tagclass (tc_subclassoftagclassid); 63 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/ddl/schema_foreign_keys.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE forum_p1 ADD FOREIGN KEY (f_moderatorid) REFERENCES person_p1; 2 | ALTER TABLE forum_p1 ADD FOREIGN KEY (f_moderatorid) REFERENCES person_p2; 3 | ALTER TABLE forum_p1 ADD FOREIGN KEY (f_moderatorid) REFERENCES person_p3; 4 | ALTER TABLE forum_p1 ADD FOREIGN KEY (f_moderatorid) REFERENCES person_p4; 5 | ALTER TABLE forum_p1 ADD FOREIGN KEY (f_moderatorid) REFERENCES person_p5; 6 | 7 | 8 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_forumid) REFERENCES forum_p1; 9 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_forumid) REFERENCES forum_p2; 10 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_personid) REFERENCES person_p1; 11 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_personid) REFERENCES person_p2; 12 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_personid) REFERENCES person_p3; 13 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_personid) REFERENCES person_p4; 14 | ALTER TABLE forum_person_p1 ADD FOREIGN KEY (fp_personid) REFERENCES person_p5; 15 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_forumid) REFERENCES forum_p1; 16 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_forumid) REFERENCES forum_p2; 17 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_personid) REFERENCES person_p1; 18 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_personid) REFERENCES person_p2; 19 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_personid) REFERENCES person_p3; 20 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_personid) REFERENCES person_p4; 21 | ALTER TABLE forum_person_p2 ADD FOREIGN KEY (fp_personid) REFERENCES person_p5; 22 | 23 | 24 | ALTER TABLE forum_tag ADD FOREIGN KEY (ft_forumid) REFERENCES forum_p1; 25 | ALTER TABLE forum_tag ADD FOREIGN KEY (ft_forumid) REFERENCES forum_p2; 26 | 27 | ALTER TABLE forum_tag ADD FOREIGN KEY (ft_tagid) REFERENCES tag; 28 | 29 | ALTER TABLE knows ADD FOREIGN KEY (k_person1id) REFERENCES person; 30 | ALTER TABLE knows ADD FOREIGN KEY (k_person2id) REFERENCES person; 31 | ALTER TABLE likes ADD FOREIGN KEY (l_personid) REFERENCES person; 32 | ALTER TABLE likes ADD FOREIGN KEY (l_messageid) REFERENCES message; 33 | ALTER TABLE organisation ADD FOREIGN KEY (o_placeid) REFERENCES place; 34 | ALTER TABLE person_p1 ADD FOREIGN KEY (p_placeid) REFERENCES place; 35 | ALTER TABLE person_p2 ADD FOREIGN KEY (p_placeid) REFERENCES place; 36 | ALTER TABLE person_p3 ADD FOREIGN KEY (p_placeid) REFERENCES place; 37 | ALTER TABLE person_p4 ADD FOREIGN KEY (p_placeid) REFERENCES place; 38 | ALTER TABLE person_p5 ADD FOREIGN KEY (p_placeid) REFERENCES place; 39 | ALTER TABLE person_company ADD FOREIGN KEY (pc_personid) REFERENCES person; 40 | ALTER TABLE person_company ADD FOREIGN KEY (pc_organisationid) REFERENCES organisation; 41 | ALTER TABLE person_email ADD FOREIGN KEY (pe_personid) REFERENCES person; 42 | ALTER TABLE person_language ADD FOREIGN KEY (plang_personid) REFERENCES person; 43 | ALTER TABLE person_tag ADD FOREIGN KEY (pt_personid) REFERENCES person; 44 | ALTER TABLE person_tag ADD FOREIGN KEY (pt_tagid) REFERENCES tag; 45 | ALTER TABLE person_university ADD FOREIGN KEY (pu_personid) REFERENCES person; 46 | ALTER TABLE person_university ADD FOREIGN KEY (pu_organisationid) REFERENCES organisation; 47 | ALTER TABLE place ADD FOREIGN KEY (pl_containerplaceid) REFERENCES place; 48 | -- ALTER TABLE message ADD FOREIGN KEY (m_creatorid) REFERENCES person; 49 | -- ALTER TABLE message ADD FOREIGN KEY (m_locationid) REFERENCES place; 50 | -- ALTER TABLE message ADD FOREIGN KEY (m_ps_forumid) REFERENCES forum; 51 | -- ALTER TABLE message ADD FOREIGN KEY (m_c_replyof) REFERENCES message; 52 | ALTER TABLE message_tag ADD FOREIGN KEY (mt_messageid) REFERENCES message; 53 | ALTER TABLE message_tag ADD FOREIGN KEY (mt_tagid) REFERENCES tag; 54 | ALTER TABLE tag ADD FOREIGN KEY (t_tagclassid) REFERENCES tagclass; 55 | ALTER TABLE tagclass ADD FOREIGN KEY (tc_subclassoftagclassid) REFERENCES tagclass; 56 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/ddl/schemas.sql: -------------------------------------------------------------------------------- 1 | create table post_p1 ( 2 | m_messageid bigint not null, 3 | m_ps_imagefile varchar, 4 | m_creationdate timestamp with time zone not null, 5 | m_locationip varchar not null, 6 | m_browserused varchar not null, 7 | m_ps_language varchar, 8 | m_content text 9 | ); 10 | 11 | create table post_p2 ( 12 | m_messageid bigint not null, 13 | m_ps_imagefile varchar, 14 | m_creationdate timestamp with time zone not null, 15 | m_locationip varchar not null, 16 | m_browserused varchar not null, 17 | m_ps_language varchar, 18 | m_length int not null 19 | ); 20 | 21 | create table post_p3 ( 22 | m_messageid bigint not null, 23 | m_ps_imagefile varchar, 24 | m_creationdate timestamp with time zone not null, 25 | m_locationip varchar not null, 26 | m_browserused varchar not null, 27 | m_ps_language varchar, 28 | m_creatorid bigint not null 29 | ); 30 | 31 | create table post_p4 ( 32 | m_messageid bigint not null, 33 | m_ps_imagefile varchar, 34 | m_creationdate timestamp with time zone not null, 35 | m_locationip varchar not null, 36 | m_browserused varchar not null, 37 | m_ps_language varchar, 38 | m_ps_forumid bigint 39 | ); 40 | 41 | create table post_p5 ( 42 | m_messageid bigint not null, 43 | m_ps_imagefile varchar, 44 | m_creationdate timestamp with time zone not null, 45 | m_locationip varchar not null, 46 | m_browserused varchar not null, 47 | m_ps_language varchar, 48 | m_locationid bigint not null 49 | ); 50 | 51 | create table comment_p1 ( 52 | m_messageid bigint not null, 53 | m_creationdate timestamp with time zone not null, 54 | m_locationip varchar not null, 55 | m_browserused varchar not null, 56 | m_content text, 57 | m_length int not null 58 | ); 59 | 60 | create table comment_p2 ( 61 | m_messageid bigint not null, 62 | m_creationdate timestamp with time zone not null, 63 | m_locationip varchar not null, 64 | m_browserused varchar not null, 65 | m_content text, 66 | m_creatorid bigint not null 67 | ); 68 | 69 | create table comment_p3 ( 70 | m_messageid bigint not null, 71 | m_creationdate timestamp with time zone not null, 72 | m_locationip varchar not null, 73 | m_browserused varchar not null, 74 | m_content text, 75 | m_locationid bigint not null 76 | ); 77 | 78 | create table comment_p4 ( 79 | m_messageid bigint not null, 80 | m_creationdate timestamp with time zone not null, 81 | m_locationip varchar not null, 82 | m_browserused varchar not null, 83 | m_content text, 84 | m_replyof_post bigint 85 | ); 86 | 87 | create table comment_p5 ( 88 | m_messageid bigint not null, 89 | m_creationdate timestamp with time zone not null, 90 | m_locationip varchar not null, 91 | m_browserused varchar not null, 92 | m_content text, 93 | m_replyof_comment bigint 94 | ); 95 | 96 | create table forum_p1 ( 97 | f_forumid bigint not null, 98 | f_title varchar not null, 99 | f_moderatorid bigint not null 100 | ); 101 | 102 | create table forum_p2 ( 103 | f_forumid bigint not null, 104 | f_title varchar not null, 105 | f_creationdate timestamp with time zone not null 106 | ); 107 | 108 | create table forum_person_p1 ( 109 | fp_forumid bigint not null, 110 | fp_personid bigint not null, 111 | fp_joindate timestamp with time zone not null 112 | ); 113 | 114 | create table forum_person_p2 ( 115 | fp_forumid bigint not null, 116 | fp_personid bigint not null 117 | ); 118 | 119 | create table forum_tag ( 120 | ft_forumid bigint not null, 121 | ft_tagid bigint not null 122 | ); 123 | 124 | create table organisation ( 125 | o_organisationid bigint not null, 126 | o_type varchar not null, 127 | o_name varchar not null, 128 | o_url varchar not null, 129 | o_placeid bigint not null 130 | ); 131 | 132 | create table person_p1 ( 133 | p_personid bigint not null, 134 | p_firstname varchar not null, 135 | p_locationip varchar not null, 136 | p_browserused varchar not null, 137 | p_placeid bigint not null 138 | ); 139 | 140 | create table person_p2 ( 141 | p_personid bigint not null, 142 | p_lastname varchar not null, 143 | p_locationip varchar not null, 144 | p_browserused varchar not null, 145 | p_placeid bigint not null 146 | ); 147 | 148 | create table person_p3 ( 149 | p_personid bigint not null, 150 | p_gender varchar not null, 151 | p_locationip varchar not null, 152 | p_browserused varchar not null, 153 | p_placeid bigint not null 154 | ); 155 | 156 | create table person_p4 ( 157 | p_personid bigint not null, 158 | p_birthday date not null, 159 | p_locationip varchar not null, 160 | p_browserused varchar not null, 161 | p_placeid bigint not null 162 | ); 163 | 164 | create table person_p5 ( 165 | p_personid bigint not null, 166 | p_creationdate timestamp with time zone not null, 167 | p_locationip varchar not null, 168 | p_browserused varchar not null, 169 | p_placeid bigint not null 170 | ); 171 | 172 | create table person_email ( 173 | pe_personid bigint not null, 174 | pe_email varchar not null 175 | ); 176 | 177 | 178 | create table person_tag ( 179 | pt_personid bigint not null, 180 | pt_tagid bigint not null 181 | ); 182 | 183 | create table knows ( 184 | k_person1id bigint not null, 185 | k_person2id bigint not null, 186 | k_creationdate timestamp with time zone not null 187 | ); 188 | 189 | create table likes ( 190 | l_personid bigint not null, 191 | l_messageid bigint not null, 192 | l_creationdate timestamp with time zone not null 193 | ); 194 | 195 | create table person_language ( 196 | plang_personid bigint not null, 197 | plang_language varchar not null 198 | ); 199 | 200 | create table person_university ( 201 | pu_personid bigint not null, 202 | pu_organisationid bigint not null, 203 | pu_classyear int not null 204 | ); 205 | 206 | create table person_company ( 207 | pc_personid bigint not null, 208 | pc_organisationid bigint not null, 209 | pc_workfrom int not null 210 | ); 211 | 212 | create table place ( 213 | pl_placeid bigint not null, 214 | pl_name varchar not null, 215 | pl_url varchar not null, 216 | pl_type varchar not null, 217 | pl_containerplaceid bigint -- null for continents 218 | ); 219 | 220 | create table message_tag ( 221 | mt_messageid bigint not null, 222 | mt_tagid bigint not null 223 | ); 224 | 225 | create table tagclass ( 226 | tc_tagclassid bigint not null, 227 | tc_name varchar not null, 228 | tc_url varchar not null, 229 | tc_subclassoftagclassid bigint -- null for the root tagclass (Thing) 230 | ); 231 | 232 | create table tag ( 233 | t_tagid bigint not null, 234 | t_name varchar not null, 235 | t_url varchar not null, 236 | t_tagclassid bigint not null 237 | ); 238 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/comment.sql: -------------------------------------------------------------------------------- 1 | ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 2 | UNION ALL 3 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 4 | UNION ALL 5 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 6 | UNION ALL 7 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 8 | UNION ALL 9 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/forum.sql: -------------------------------------------------------------------------------- 1 | ((SELECT f_forumid, f_title, NULL as f_creationdate, f_moderatorid FROM forum_p1) 2 | UNION ALL 3 | (SELECT f_forumid, f_title, f_creationdate, NULL as f_moderatorid FROM forum_p2)) as forum -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/forum_person.sql: -------------------------------------------------------------------------------- 1 | ((SELECT fp_forumid, fp_personid, fp_joindate FROM forum_person_p1) 2 | UNION ALL 3 | (SELECT fp_forumid, fp_personid, NULL as fp_joindate FROM forum_person_p2)) as forum_person -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/message.sql: -------------------------------------------------------------------------------- 1 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 2 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 3 | UNION ALL 4 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 5 | UNION ALL 6 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 7 | UNION ALL 8 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 9 | UNION ALL 10 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 11 | UNION ALL 12 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 13 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 14 | UNION ALL 15 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 16 | UNION ALL 17 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 18 | UNION ALL 19 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 20 | UNION ALL 21 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/person.sql: -------------------------------------------------------------------------------- 1 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 2 | UNION ALL 3 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 4 | UNION ALL 5 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 6 | UNION ALL 7 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 8 | UNION ALL 9 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/union-all/post.sql: -------------------------------------------------------------------------------- 1 | ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 2 | UNION ALL 3 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 4 | UNION ALL 5 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 6 | UNION ALL 7 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 8 | UNION ALL 9 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-complex-2.sql: -------------------------------------------------------------------------------- 1 | /* Q2. Recent messages by your friends 2 | \set personId 10995116278009 3 | \set maxDate '\'2010-10-16\'' 4 | */ 5 | EXPLAIN (ANALYZE, FORMAT JSON) 6 | select p_personid, p_firstname, p_lastname, m_messageid, COALESCE(m_ps_imagefile, m_content), m_creationdate 7 | from 8 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 9 | UNION ALL 10 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 11 | UNION ALL 12 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 13 | UNION ALL 14 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 15 | UNION ALL 16 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person, 17 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 18 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 19 | UNION ALL 20 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 21 | UNION ALL 22 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 23 | UNION ALL 24 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 25 | UNION ALL 26 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 27 | UNION ALL 28 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 29 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 30 | UNION ALL 31 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 32 | UNION ALL 33 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 34 | UNION ALL 35 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 36 | UNION ALL 37 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, 38 | knows 39 | where 40 | p_personid = m_creatorid and 41 | m_creationdate <= to_timestamp(1354060800000/1000)::date and 42 | k_person1id = 17592186052613 and 43 | k_person2id = p_personid 44 | order by m_creationdate desc, m_messageid asc 45 | limit 20 46 | ; 47 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-complex-3.sql: -------------------------------------------------------------------------------- 1 | /* Q3. Friends and friends of friends that have been to given countries 2 | \set personId 6597069766734 3 | \set countryXName '\'Angola\'' 4 | \set countryYName '\'Colombia\'' 5 | \set startDate '\'2010-06-01\''::date 6 | \set durationDays 28 7 | */ 8 | SET client_min_messages = debug5; 9 | EXPLAIN (ANALYZE, FORMAT JSON) 10 | select p_personid, p_firstname, p_lastname, ct1, ct2, total as totalcount 11 | from 12 | ( select k_person2id 13 | from knows 14 | where 15 | k_person1id = 17592186055119 16 | union 17 | select k2.k_person2id 18 | from knows k1, knows k2 19 | where 20 | k1.k_person1id = 17592186055119 and k1.k_person2id = k2.k_person1id and k2.k_person2id <> 17592186055119 21 | ) f, ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 22 | UNION ALL 23 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 24 | UNION ALL 25 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 26 | UNION ALL 27 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 28 | UNION ALL 29 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person, place p1, place p2, 30 | ( 31 | select chn.m_c_creatorid, ct1, ct2, ct1 + ct2 as total 32 | from 33 | ( 34 | select m_creatorid as m_c_creatorid, count(*) as ct1 from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 35 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 36 | UNION ALL 37 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 38 | UNION ALL 39 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 40 | UNION ALL 41 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 42 | UNION ALL 43 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 44 | UNION ALL 45 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 46 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 47 | UNION ALL 48 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 49 | UNION ALL 50 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 51 | UNION ALL 52 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 53 | UNION ALL 54 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, place 55 | where 56 | m_locationid = pl_placeid and pl_name = 'Laos' and 57 | m_creationdate >= to_timestamp(1306886400000/1000)::date and m_creationdate < (to_timestamp(1306886400000/1000)::date + INTERVAL '1 days' * 42) 58 | group by m_c_creatorid 59 | ) chn, 60 | ( 61 | select m_creatorid as m_c_creatorid, count(*) as ct2 from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 62 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 63 | UNION ALL 64 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 65 | UNION ALL 66 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 67 | UNION ALL 68 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 69 | UNION ALL 70 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 71 | UNION ALL 72 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 73 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 74 | UNION ALL 75 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 76 | UNION ALL 77 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 78 | UNION ALL 79 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 80 | UNION ALL 81 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, place 82 | where 83 | m_locationid = pl_placeid and pl_name = 'Scotland' and 84 | m_creationdate >= to_timestamp(1306886400000/1000)::date and m_creationdate < (to_timestamp(1306886400000/1000)::date + INTERVAL '1 days' * 42) 85 | group by m_creatorid --m_c_creatorid 86 | ) ind 87 | where CHN.m_c_creatorid = IND.m_c_creatorid 88 | ) cpc 89 | where 90 | f.k_person2id = p_personid and p_placeid = p1.pl_placeid and 91 | p1.pl_containerplaceid = p2.pl_placeid and p2.pl_name <> 'Laos' and p2.pl_name <> 'Laos' and 92 | f.k_person2id = cpc.m_c_creatorid 93 | order by totalcount desc, p_personid asc 94 | limit 20 95 | ; 96 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-complex-4.sql: -------------------------------------------------------------------------------- 1 | /* Q4. New topics 2 | \set personId 10995116277918 3 | \set startDate '\'2010-10-01\''::date 4 | \set durationDays 31 5 | */ 6 | SET client_min_messages = debug5; 7 | EXPLAIN (ANALYZE, FORMAT JSON) 8 | select t_name, count(*) as postCount 9 | from tag, (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 10 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 11 | UNION ALL 12 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 13 | UNION ALL 14 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 15 | UNION ALL 16 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 17 | UNION ALL 18 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 19 | UNION ALL 20 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 21 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 22 | UNION ALL 23 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 24 | UNION ALL 25 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 26 | UNION ALL 27 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 28 | UNION ALL 29 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, message_tag recent, knows 30 | where 31 | m_messageid = mt_messageid and 32 | mt_tagid = t_tagid and 33 | m_creatorid = k_person2id and 34 | m_c_replyof IS NULL and -- post, not comment 35 | k_person1id = 21990232559429 and 36 | m_creationdate >= to_timestamp(1335830400000/1000)::date and 37 | m_creationdate < (to_timestamp(1335830400000/1000)::date + INTERVAL '1 days' * 37) and 38 | not exists ( 39 | select * from 40 | (select distinct mt_tagid from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 41 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 42 | UNION ALL 43 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 44 | UNION ALL 45 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 46 | UNION ALL 47 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 48 | UNION ALL 49 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 50 | UNION ALL 51 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 52 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 53 | UNION ALL 54 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 55 | UNION ALL 56 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 57 | UNION ALL 58 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 59 | UNION ALL 60 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, message_tag, knows 61 | where 62 | k_person1id = 21990232559429 and 63 | k_person2id = m_creatorid and 64 | m_c_replyof IS NULL and -- post, not comment 65 | mt_messageid = m_messageid and 66 | m_creationdate < to_timestamp(1335830400000/1000)::date) tags 67 | where tags.mt_tagid = recent.mt_tagid) 68 | group by t_name 69 | order by postCount desc, t_name asc 70 | limit 10 71 | ; 72 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-1.sql: -------------------------------------------------------------------------------- 1 | EXPLAIN (ANALYZE, FORMAT JSON) 2 | select p_firstname, p_lastname, p_birthday, p_locationip, p_browserused, p_placeid, p_gender, p_creationdate 3 | from ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 4 | UNION ALL 5 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 6 | UNION ALL 7 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 8 | UNION ALL 9 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 10 | UNION ALL 11 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 12 | where p_personid = 35184372099695; 13 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-2.sql: -------------------------------------------------------------------------------- 1 | SET client_min_messages = debug5; 2 | SET optimizer = on; 3 | 4 | EXPLAIN (ANALYZE, FORMAT JSON) 5 | with recursive cposts(m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof, m_creatorid) AS ( 6 | select m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof, m_creatorid 7 | from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 8 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 9 | UNION ALL 10 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 11 | UNION ALL 12 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 13 | UNION ALL 14 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 15 | UNION ALL 16 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 17 | UNION ALL 18 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 19 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 20 | UNION ALL 21 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 22 | UNION ALL 23 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 24 | UNION ALL 25 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 26 | UNION ALL 27 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message 28 | where m_creatorid = 933 29 | order by m_creationdate desc 30 | limit 10 31 | ), parent(postid,replyof,orig_postid,creator) AS ( 32 | select m_messageid, m_c_replyof, m_messageid, m_creatorid from cposts 33 | UNION ALL 34 | select m_messageid, m_c_replyof, orig_postid, m_creatorid 35 | from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 36 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 37 | UNION ALL 38 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 39 | UNION ALL 40 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 41 | UNION ALL 42 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 43 | UNION ALL 44 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 45 | UNION ALL 46 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 47 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 48 | UNION ALL 49 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 50 | UNION ALL 51 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 52 | UNION ALL 53 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 54 | UNION ALL 55 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, parent 56 | where m_messageid=replyof 57 | ) 58 | select p1.m_messageid, COALESCE(m_ps_imagefile, m_content, ''), p1.m_creationdate, 59 | p2.m_messageid, p2.p_personid, p2.p_firstname, p2.p_lastname 60 | from 61 | (select m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof from cposts 62 | ) p1 63 | left join 64 | (select orig_postid, postid as m_messageid, p_personid, p_firstname, p_lastname 65 | from parent, 66 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 67 | UNION ALL 68 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 69 | UNION ALL 70 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 71 | UNION ALL 72 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 73 | UNION ALL 74 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 75 | where replyof is null and creator = p_personid 76 | )p2 77 | on p2.orig_postid = p1.m_messageid 78 | order by m_creationdate desc, p2.m_messageid desc; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-3.sql: -------------------------------------------------------------------------------- 1 | EXPLAIN (ANALYZE, FORMAT JSON) 2 | select p_personid, p_firstname, p_lastname, k_creationdate 3 | from knows, 4 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 5 | UNION ALL 6 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 7 | UNION ALL 8 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 9 | UNION ALL 10 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 11 | UNION ALL 12 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 13 | where k_person1id = 933 and k_person2id = p_personid 14 | order by k_creationdate desc, p_personid asc; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-4.sql: -------------------------------------------------------------------------------- 1 | EXPLAIN (ANALYZE, FORMAT JSON) 2 | select COALESCE(m_ps_imagefile, m_content, ''), m_creationdate 3 | from (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 4 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 5 | UNION ALL 6 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 7 | UNION ALL 8 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 9 | UNION ALL 10 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 11 | UNION ALL 12 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 13 | UNION ALL 14 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 15 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 16 | UNION ALL 17 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 18 | UNION ALL 19 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 20 | UNION ALL 21 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 22 | UNION ALL 23 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message 24 | where m_messageid = 2199029886840; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-5-no-filter.sql: -------------------------------------------------------------------------------- 1 | EXPLAIN (ANALYZE, FORMAT JSON) 2 | select p_personid, p_firstname, p_lastname 3 | from 4 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 5 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 6 | UNION ALL 7 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 8 | UNION ALL 9 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 10 | UNION ALL 11 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 12 | UNION ALL 13 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 14 | UNION ALL 15 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 16 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 17 | UNION ALL 18 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 19 | UNION ALL 20 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 21 | UNION ALL 22 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 23 | UNION ALL 24 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, 25 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 26 | UNION ALL 27 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 28 | UNION ALL 29 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 30 | UNION ALL 31 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 32 | UNION ALL 33 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 34 | where m_creatorid = p_personid; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-5.sql: -------------------------------------------------------------------------------- 1 | EXPLAIN (ANALYZE, FORMAT JSON) 2 | select p_personid, p_firstname, p_lastname 3 | from 4 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 5 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 6 | UNION ALL 7 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 8 | UNION ALL 9 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 10 | UNION ALL 11 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 12 | UNION ALL 13 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 14 | UNION ALL 15 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 16 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 17 | UNION ALL 18 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 19 | UNION ALL 20 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 21 | UNION ALL 22 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 23 | UNION ALL 24 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, 25 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 26 | UNION ALL 27 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 28 | UNION ALL 29 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 30 | UNION ALL 31 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 32 | UNION ALL 33 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 34 | where m_messageid = 824635044686 and m_creatorid = p_personid; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-6.sql: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE chain(parent, child) as( 2 | SELECT m_c_replyof, m_messageid FROM (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 3 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 4 | UNION ALL 5 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 6 | UNION ALL 7 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 8 | UNION ALL 9 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 10 | UNION ALL 11 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 12 | UNION ALL 13 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 14 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 15 | UNION ALL 16 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 17 | UNION ALL 18 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 19 | UNION ALL 20 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 21 | UNION ALL 22 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message where m_messageid = 824635044686 23 | UNION ALL 24 | SELECT p.m_c_replyof, p.m_messageid FROM 25 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 26 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 27 | UNION ALL 28 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 29 | UNION ALL 30 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 31 | UNION ALL 32 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 33 | UNION ALL 34 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 35 | UNION ALL 36 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 37 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 38 | UNION ALL 39 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 40 | UNION ALL 41 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 42 | UNION ALL 43 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 44 | UNION ALL 45 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as p, chain c where p.m_messageid = c.parent 46 | ) 47 | select f_forumid, f_title, p_personid, p_firstname, p_lastname 48 | from 49 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 50 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 51 | UNION ALL 52 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 53 | UNION ALL 54 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 55 | UNION ALL 56 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 57 | UNION ALL 58 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 59 | UNION ALL 60 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 61 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 62 | UNION ALL 63 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 64 | UNION ALL 65 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 66 | UNION ALL 67 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 68 | UNION ALL 69 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as message, 70 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 71 | UNION ALL 72 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 73 | UNION ALL 74 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 75 | UNION ALL 76 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 77 | UNION ALL 78 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person, 79 | ((SELECT f_forumid, f_title, NULL as f_creationdate, f_moderatorid FROM forum_p1) 80 | UNION ALL 81 | (SELECT f_forumid, f_title, f_creationdate, NULL as f_moderatorid FROM forum_p2)) as forum 82 | where m_messageid = (select coalesce(min(parent), 824635044686) from chain) 83 | and m_ps_forumid = f_forumid and f_moderatorid = p_personid; 84 | -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/interactive-short-7.sql: -------------------------------------------------------------------------------- 1 | SET client_min_messages = debug5; 2 | EXPLAIN (ANALYZE, FORMAT JSON) 3 | select p2.m_messageid, p2.m_content, p2.m_creationdate, p_personid, p_firstname, p_lastname, 4 | (case when exists ( 5 | select 1 from knows 6 | where p1.m_creatorid = k_person1id and p2.m_creatorid = k_person2id) 7 | then TRUE 8 | else FALSE 9 | end) 10 | from 11 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 12 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 13 | UNION ALL 14 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 15 | UNION ALL 16 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 17 | UNION ALL 18 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 19 | UNION ALL 20 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 21 | UNION ALL 22 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 23 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 24 | UNION ALL 25 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 26 | UNION ALL 27 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 28 | UNION ALL 29 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 30 | UNION ALL 31 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as p1, 32 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 33 | FROM ((SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p1) 34 | UNION ALL 35 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, m_length, NULL as m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p2) 36 | UNION ALL 37 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, m_creatorid, NULL as m_ps_forumid, NULL as m_locationid FROM post_p3) 38 | UNION ALL 39 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, m_ps_forumid, NULL as m_locationid FROM post_p4) 40 | UNION ALL 41 | (SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, NULL as m_content, NULL as m_length, NULL as m_creatorid, NULL as m_ps_forumid, m_locationid FROM post_p5)) as post 42 | UNION ALL 43 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 44 | FROM ((SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p1) 45 | UNION ALL 46 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, m_creatorid, NULL as m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p2) 47 | UNION ALL 48 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, m_locationid, NULL as m_replyof_post, NULL as m_replyof_comment FROM comment_p3) 49 | UNION ALL 50 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, m_replyof_post, NULL as m_replyof_comment FROM comment_p4) 51 | UNION ALL 52 | (SELECT m_messageid, m_creationdate, m_locationip, m_browserused, m_content, NULL as m_length, NULL as m_creatorid, NULL as m_locationid, NULL as m_replyof_post, m_replyof_comment FROM comment_p5)) as comment) as p2, 53 | ((SELECT p_personid, p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p1) 54 | UNION ALL 55 | (SELECT p_personid, NULL as p_firstname, p_lastname, NULL as p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p2) 56 | UNION ALL 57 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, p_gender, NULL as p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p3) 58 | UNION ALL 59 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, p_birthday, NULL as p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p4) 60 | UNION ALL 61 | (SELECT p_personid, NULL as p_firstname, NULL as p_lastname, NULL as p_gender, NULL as p_birthday, p_creationdate, p_locationip, p_browserused, p_placeid FROM person_p5)) as person 62 | where 63 | p1.m_messageid = 824635044682 and p2.m_c_replyof = p1.m_messageid and p2.m_creatorid = p_personid 64 | order by p2.m_creationdate desc, p2.m_creatorid asc; -------------------------------------------------------------------------------- /ldbc-converter/generated-queries/workload/test.sql: -------------------------------------------------------------------------------- 1 | SET client_min_messages = debug5; -------------------------------------------------------------------------------- /ldbc-converter/generator-script.sh: -------------------------------------------------------------------------------- 1 | rm -rf generated-dataset/dynamic/person 2 | mkdir generated-dataset/dynamic/person 3 | 4 | python dataset-generator.py configurations/person/person_2.yaml json 5 | mv generated-dataset/dynamic/person_0_0.json generated-dataset/dynamic/person/person_0_0_2.json 6 | 7 | python dataset-generator.py configurations/person/person_5.yaml json 8 | mv generated-dataset/dynamic/person_0_0.json generated-dataset/dynamic/person/person_0_0_5.json 9 | 10 | python dataset-generator.py configurations/person/person_10.yaml json 11 | mv generated-dataset/dynamic/person_0_0.json generated-dataset/dynamic/person/person_0_0_10.json 12 | 13 | rm -rf generated-dataset/dynamic/post 14 | mkdir generated-dataset/dynamic/post 15 | 16 | python dataset-generator.py configurations/post/post_2.yaml json 17 | mv generated-dataset/dynamic/post_0_0.json generated-dataset/dynamic/post/post_0_0_2.json 18 | 19 | python dataset-generator.py configurations/post/post_5.yaml json 20 | mv generated-dataset/dynamic/post_0_0.json generated-dataset/dynamic/post/post_0_0_5.json 21 | 22 | python dataset-generator.py configurations/post/post_10.yaml json 23 | mv generated-dataset/dynamic/post_0_0.json generated-dataset/dynamic/post/post_0_0_10.json 24 | 25 | rm -rf generated-dataset/dynamic/comment 26 | mkdir generated-dataset/dynamic/comment 27 | 28 | python dataset-generator.py configurations/comment/comment_2.yaml json 29 | mv generated-dataset/dynamic/comment_0_0.json generated-dataset/dynamic/comment/comment_0_0_2.json 30 | 31 | python dataset-generator.py configurations/comment/comment_5.yaml json 32 | mv generated-dataset/dynamic/comment_0_0.json generated-dataset/dynamic/comment/comment_0_0_5.json 33 | 34 | python dataset-generator.py configurations/comment/comment_10.yaml json 35 | mv generated-dataset/dynamic/comment_0_0.json generated-dataset/dynamic/comment/comment_0_0_10.json -------------------------------------------------------------------------------- /ldbc-converter/ldbc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | RESULTS=$1 4 | HOST=$2 5 | DBNAME=$3 6 | USER=$4 7 | PWD=$5 8 | # delay between stats collections (iostat, vmstat, ...) 9 | DELAY=15 10 | 11 | # DSS queries timeout (5 minutes or something like that) 12 | DSS_TIMEOUT=300000 # 5 minutes in seconds 13 | 14 | # log 15 | LOGFILE=bench.log 16 | 17 | function benchmark_run() { 18 | 19 | mkdir -p $RESULTS 20 | 21 | # store the settings 22 | psql -h $HOST -U $USER postgres -c "select name,setting from pg_settings" > $RESULTS/settings.log 2> $RESULTS/settings.err 23 | 24 | print_log "preparing LDBC database" 25 | 26 | # create database, populate it with data and set up foreign keys 27 | psql -h $HOST -U $USER $DBNAME < ./original-queries/ddl/schemas.sql > $RESULTS/create.log 2> $RESULTS/create.err 28 | 29 | print_log " loading data" 30 | 31 | psql -h $HOST -U $USER $DBNAME < ./original-queries/ddl/load.sql > $RESULTS/load.log 2> $RESULTS/load.err 32 | 33 | print_log " schema constraints" 34 | 35 | psql -h $HOST -U $USER $DBNAME < ./original-queries/ddl/schema_constraints.sql > $RESULTS/constraints.log 2> $RESULTS/constraints.err 36 | 37 | # print_log " creating foreign keys" 38 | 39 | # psql -h $HOST -U $USER $DBNAME < ./original-queries/ddl/schema_foreign_keys.sql > $RESULTS/foreign.log 2> $RESULTS/foreign.err 40 | 41 | print_log " analyzing" 42 | 43 | psql -h $HOST -U $USER $DBNAME < ./original-queries/ddl/analyze.sql > $RESULTS/analyze.log 2> $RESULTS/analyze.err 44 | } 45 | 46 | function print_log() { 47 | 48 | local message=$1 49 | 50 | echo `date +"%Y-%m-%d %H:%M:%S"` "["`date +%s`"] : $message" >> $RESULTS/$LOGFILE; 51 | 52 | } 53 | 54 | mkdir $RESULTS; 55 | 56 | export PGPASSWORD=$PWD 57 | 58 | # run the benchmark 59 | benchmark_run $RESULTS $DBNAME $USER 60 | -------------------------------------------------------------------------------- /ldbc-converter/original-dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/ldbc-converter/original-dataset.png -------------------------------------------------------------------------------- /ldbc-converter/original-queries/ddl/analyze.sql: -------------------------------------------------------------------------------- 1 | ANALYZE; -------------------------------------------------------------------------------- /ldbc-converter/original-queries/ddl/load.sql: -------------------------------------------------------------------------------- 1 | -- Populate forum table 2 | COPY forum FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/forum_0_0.csv' WITH DELIMITER '|' CSV HEADER; 3 | 4 | -- Populate forum_person table 5 | COPY forum_person FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/forum_hasMember_person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 6 | 7 | -- Populate forum_tag table 8 | COPY forum_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/forum_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 9 | 10 | -- Populate organisation table 11 | COPY organisation FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 12 | 13 | -- Populate person table 14 | COPY person FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 15 | 16 | -- Populate person_email table 17 | COPY person_email FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_email_emailaddress_0_0.csv' WITH DELIMITER '|' CSV HEADER; 18 | 19 | -- Populate person_tag table 20 | COPY person_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_hasInterest_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 21 | 22 | -- Populate knows table 23 | COPY knows ( k_person1id, k_person2id, k_creationdate) FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_knows_person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 24 | COPY knows ( k_person2id, k_person1id, k_creationdate) FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_knows_person_0_0.csv' WITH DELIMITER '|' CSV HEADER; 25 | 26 | -- Populate likes table 27 | COPY likes FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_likes_post_0_0.csv' WITH DELIMITER '|' CSV HEADER; 28 | COPY likes FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_likes_comment_0_0.csv' WITH DELIMITER '|' CSV HEADER; 29 | 30 | -- Populate person_language table 31 | COPY person_language FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_speaks_language_0_0.csv' WITH DELIMITER '|' CSV HEADER; 32 | 33 | -- Populate person_university table 34 | COPY person_university FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_studyAt_organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 35 | 36 | -- Populate person_company table 37 | COPY person_company FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/person_workAt_organisation_0_0.csv' WITH DELIMITER '|' CSV HEADER; 38 | 39 | -- Populate place table 40 | COPY place FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/place_0_0.csv' WITH DELIMITER '|' CSV HEADER; 41 | 42 | -- Populate message_tag table 43 | COPY message_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/post_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 44 | COPY message_tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/comment_hasTag_tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 45 | 46 | -- Populate tagclass table 47 | COPY tagclass FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/tagclass_0_0.csv' WITH DELIMITER '|' CSV HEADER; 48 | 49 | -- Populate tag table 50 | COPY tag FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/static/tag_0_0.csv' WITH DELIMITER '|' CSV HEADER; 51 | 52 | CREATE TABLE country AS 53 | SELECT city.pl_placeid AS ctry_city, ctry.pl_name AS ctry_name 54 | FROM place city, place ctry 55 | WHERE city.pl_containerplaceid = ctry.pl_placeid 56 | AND ctry.pl_type = 'country'; 57 | 58 | -- Populate posts and comments tables, union them into message 59 | COPY post FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/post_0_0.csv' WITH DELIMITER '|' CSV HEADER; 60 | COPY comment FROM '/home/schemaless-data-generator/ldbc-converter/original-dataset/dynamic/comment_0_0.csv' WITH DELIMITER '|' CSV HEADER; 61 | 62 | -- Note: to distinguish between "post" and "comment" records: 63 | -- - m_c_replyof IS NULL for all "post" records 64 | -- - m_c_replyof IS NOT NULL for all "comment" records 65 | CREATE TABLE message AS 66 | SELECT m_messageid, m_ps_imagefile, m_creationdate, m_locationip, m_browserused, m_ps_language, m_content, m_length, m_creatorid, m_locationid, m_ps_forumid, NULL AS m_c_replyof 67 | FROM post 68 | UNION ALL 69 | SELECT m_messageid, NULL, m_creationdate, m_locationip, m_browserused, NULL, m_content, m_length, m_creatorid, m_locationid, NULl, coalesce(m_replyof_post, m_replyof_comment) 70 | FROM comment; 71 | 72 | DROP TABLE post; 73 | DROP TABLE comment; 74 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/ddl/schema_constraints.sql: -------------------------------------------------------------------------------- 1 | -- psql would automatically build index on pk constraints. so disable it 2 | 3 | ALTER TABLE message ADD PRIMARY KEY (m_messageid); 4 | ALTER TABLE forum ADD PRIMARY KEY (f_forumid); 5 | ALTER TABLE organisation ADD PRIMARY KEY (o_organisationid); 6 | ALTER TABLE person ADD PRIMARY KEY (p_personid); 7 | ALTER TABLE place ADD PRIMARY KEY (pl_placeid); 8 | ALTER TABLE tagclass ADD PRIMARY KEY (tc_tagclassid); 9 | ALTER TABLE tag ADD PRIMARY KEY (t_tagid); 10 | ALTER TABLE forum_person ADD PRIMARY KEY (fp_forumid, fp_personid); 11 | ALTER TABLE forum_tag ADD PRIMARY KEY (ft_forumid, ft_tagid); 12 | 13 | ALTER TABLE person_email ADD PRIMARY KEY (pe_personid, pe_email); 14 | ALTER TABLE person_tag ADD PRIMARY KEY (pt_personid, pt_tagid); 15 | ALTER TABLE knows ADD PRIMARY KEY (k_person1id, k_person2id); 16 | ALTER TABLE likes ADD PRIMARY KEY (l_messageid, l_personid); 17 | ALTER TABLE person_language ADD PRIMARY KEY (plang_personid, plang_language); 18 | ALTER TABLE person_university ADD PRIMARY KEY (pu_personid, pu_organisationid); 19 | ALTER TABLE person_company ADD PRIMARY KEY (pc_personid, pc_organisationid); 20 | ALTER TABLE message_tag ADD PRIMARY KEY (mt_messageid, mt_tagid); 21 | 22 | 23 | -- create index on foreign keys // index join 24 | CREATE INDEX forum_person_forumid ON forum_person (fp_forumid); 25 | CREATE INDEX forum_person_personid ON forum_person (fp_personid); 26 | CREATE INDEX forum_tag_forumid ON forum_tag (ft_forumid); 27 | CREATE INDEX forum_tag_tagid ON forum_tag (ft_tagid); 28 | CREATE INDEX knows_person1id ON knows (k_person1id); 29 | CREATE INDEX knows_person2id ON knows (k_person2id); 30 | CREATE INDEX likes_personid ON likes (l_personid); 31 | CREATE INDEX likes_messageid ON likes (l_messageid); 32 | CREATE INDEX organisation_placeid ON organisation (o_placeid); 33 | CREATE INDEX person_placeid ON person (p_placeid); 34 | CREATE INDEX person_company_personid ON person_company (pc_personid); 35 | CREATE INDEX person_company_organisationid ON person_company (pc_organisationid); 36 | CREATE INDEX person_email_personid ON person_email (pe_personid); 37 | CREATE INDEX person_language_personid ON person_language (plang_personid); 38 | CREATE INDEX person_tag_personid ON person_tag (pt_personid); 39 | CREATE INDEX person_tag_tagid ON person_tag (pt_tagid); 40 | CREATE INDEX person_university_personid ON person_university (pu_personid); 41 | CREATE INDEX person_university_organisationid ON person_university (pu_organisationid); 42 | CREATE INDEX place_containerplaceid ON place (pl_containerplaceid); 43 | CREATE INDEX message_creatorid ON message (m_creatorid); 44 | CREATE INDEX message_locationid ON message (m_locationid); 45 | CREATE INDEX message_forumid ON message (m_ps_forumid); 46 | CREATE INDEX message_replyof ON message (m_c_replyof); 47 | CREATE INDEX message_tag_messageid ON message_tag (mt_messageid); 48 | CREATE INDEX message_tag_tagid ON message_tag (mt_tagid); 49 | CREATE INDEX tag_tagclassid ON tag (t_tagclassid); 50 | CREATE INDEX tagclass_subclassoftagclassid ON tagclass (tc_subclassoftagclassid); 51 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/ddl/schema_foreign_keys.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE forum ADD FOREIGN KEY (f_moderatorid) REFERENCES person; 2 | ALTER TABLE forum_person ADD FOREIGN KEY (fp_forumid) REFERENCES forum; 3 | ALTER TABLE forum_person ADD FOREIGN KEY (fp_personid) REFERENCES person; 4 | ALTER TABLE forum_tag ADD FOREIGN KEY (ft_forumid) REFERENCES forum; 5 | ALTER TABLE forum_tag ADD FOREIGN KEY (ft_tagid) REFERENCES tag; 6 | ALTER TABLE knows ADD FOREIGN KEY (k_person1id) REFERENCES person; 7 | ALTER TABLE knows ADD FOREIGN KEY (k_person2id) REFERENCES person; 8 | ALTER TABLE likes ADD FOREIGN KEY (l_personid) REFERENCES person; 9 | ALTER TABLE likes ADD FOREIGN KEY (l_messageid) REFERENCES message; 10 | ALTER TABLE organisation ADD FOREIGN KEY (o_placeid) REFERENCES place; 11 | ALTER TABLE person ADD FOREIGN KEY (p_placeid) REFERENCES place; 12 | ALTER TABLE person_company ADD FOREIGN KEY (pc_personid) REFERENCES person; 13 | ALTER TABLE person_company ADD FOREIGN KEY (pc_organisationid) REFERENCES organisation; 14 | ALTER TABLE person_email ADD FOREIGN KEY (pe_personid) REFERENCES person; 15 | ALTER TABLE person_language ADD FOREIGN KEY (plang_personid) REFERENCES person; 16 | ALTER TABLE person_tag ADD FOREIGN KEY (pt_personid) REFERENCES person; 17 | ALTER TABLE person_tag ADD FOREIGN KEY (pt_tagid) REFERENCES tag; 18 | ALTER TABLE person_university ADD FOREIGN KEY (pu_personid) REFERENCES person; 19 | ALTER TABLE person_university ADD FOREIGN KEY (pu_organisationid) REFERENCES organisation; 20 | ALTER TABLE place ADD FOREIGN KEY (pl_containerplaceid) REFERENCES place; 21 | ALTER TABLE message ADD FOREIGN KEY (m_creatorid) REFERENCES person; 22 | ALTER TABLE message ADD FOREIGN KEY (m_locationid) REFERENCES place; 23 | ALTER TABLE message ADD FOREIGN KEY (m_ps_forumid) REFERENCES forum; 24 | ALTER TABLE message ADD FOREIGN KEY (m_c_replyof) REFERENCES message; 25 | ALTER TABLE message_tag ADD FOREIGN KEY (mt_messageid) REFERENCES message; 26 | ALTER TABLE message_tag ADD FOREIGN KEY (mt_tagid) REFERENCES tag; 27 | ALTER TABLE tag ADD FOREIGN KEY (t_tagclassid) REFERENCES tagclass; 28 | ALTER TABLE tagclass ADD FOREIGN KEY (tc_subclassoftagclassid) REFERENCES tagclass; 29 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/ddl/schemas.sql: -------------------------------------------------------------------------------- 1 | drop view if exists country; 2 | drop table if exists forum_person; 3 | drop table if exists forum_tag; 4 | drop table if exists person_email; 5 | drop table if exists person_tag; 6 | drop table if exists person_language; 7 | drop table if exists person_university; 8 | drop table if exists person_company; 9 | drop table if exists message_tag; 10 | drop table if exists organisation; 11 | drop table if exists knows; 12 | drop table if exists likes; 13 | drop table if exists tag; 14 | drop table if exists tagclass; 15 | drop table if exists message; 16 | drop table if exists forum; 17 | drop table if exists person; 18 | drop table if exists place; 19 | 20 | create table post ( 21 | m_messageid bigint not null, 22 | m_ps_imagefile varchar, 23 | m_creationdate timestamp with time zone not null, 24 | m_locationip varchar not null, 25 | m_browserused varchar not null, 26 | m_ps_language varchar, 27 | m_content text, 28 | m_length int not null, 29 | m_creatorid bigint not null, 30 | m_ps_forumid bigint, 31 | m_locationid bigint not null 32 | ); 33 | 34 | create table comment ( 35 | m_messageid bigint not null, 36 | m_creationdate timestamp with time zone not null, 37 | m_locationip varchar not null, 38 | m_browserused varchar not null, 39 | m_content text, 40 | m_length int not null, 41 | m_creatorid bigint not null, 42 | m_locationid bigint not null, 43 | m_replyof_post bigint, 44 | m_replyof_comment bigint 45 | ); 46 | 47 | create table forum ( 48 | f_forumid bigint not null, 49 | f_title varchar not null, 50 | f_creationdate timestamp with time zone not null, 51 | f_moderatorid bigint not null 52 | ); 53 | 54 | create table forum_person ( 55 | fp_forumid bigint not null, 56 | fp_personid bigint not null, 57 | fp_joindate timestamp with time zone not null 58 | ); 59 | 60 | create table forum_tag ( 61 | ft_forumid bigint not null, 62 | ft_tagid bigint not null 63 | ); 64 | 65 | create table organisation ( 66 | o_organisationid bigint not null, 67 | o_type varchar not null, 68 | o_name varchar not null, 69 | o_url varchar not null, 70 | o_placeid bigint not null 71 | ); 72 | 73 | create table person ( 74 | p_personid bigint not null, 75 | p_firstname varchar not null, 76 | p_lastname varchar not null, 77 | p_gender varchar not null, 78 | p_birthday date not null, 79 | p_creationdate timestamp with time zone not null, 80 | p_locationip varchar not null, 81 | p_browserused varchar not null, 82 | p_placeid bigint not null 83 | ); 84 | 85 | create table person_email ( 86 | pe_personid bigint not null, 87 | pe_email varchar not null 88 | ); 89 | 90 | 91 | create table person_tag ( 92 | pt_personid bigint not null, 93 | pt_tagid bigint not null 94 | ); 95 | 96 | create table knows ( 97 | k_person1id bigint not null, 98 | k_person2id bigint not null, 99 | k_creationdate timestamp with time zone not null 100 | ); 101 | 102 | create table likes ( 103 | l_personid bigint not null, 104 | l_messageid bigint not null, 105 | l_creationdate timestamp with time zone not null 106 | ); 107 | 108 | create table person_language ( 109 | plang_personid bigint not null, 110 | plang_language varchar not null 111 | ); 112 | 113 | create table person_university ( 114 | pu_personid bigint not null, 115 | pu_organisationid bigint not null, 116 | pu_classyear int not null 117 | ); 118 | 119 | create table person_company ( 120 | pc_personid bigint not null, 121 | pc_organisationid bigint not null, 122 | pc_workfrom int not null 123 | ); 124 | 125 | create table place ( 126 | pl_placeid bigint not null, 127 | pl_name varchar not null, 128 | pl_url varchar not null, 129 | pl_type varchar not null, 130 | pl_containerplaceid bigint -- null for continents 131 | ); 132 | 133 | create table message_tag ( 134 | mt_messageid bigint not null, 135 | mt_tagid bigint not null 136 | ); 137 | 138 | create table tagclass ( 139 | tc_tagclassid bigint not null, 140 | tc_name varchar not null, 141 | tc_url varchar not null, 142 | tc_subclassoftagclassid bigint -- null for the root tagclass (Thing) 143 | ); 144 | 145 | create table tag ( 146 | t_tagid bigint not null, 147 | t_name varchar not null, 148 | t_url varchar not null, 149 | t_tagclassid bigint not null 150 | ); 151 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-1.sql: -------------------------------------------------------------------------------- 1 | /* Q1. Transitive friends with certain name 2 | \set personId 4398046511333 3 | \set firstName '\'Jose\'' 4 | */ 5 | select 6 | id, 7 | p_lastname, 8 | min (dist) as dist, 9 | p_birthday, 10 | p_creationdate, 11 | p_gender, 12 | p_browserused, 13 | p_locationip, 14 | (select array_agg(pe_email) from person_email where pe_personid = id group by pe_personid) as emails, 15 | (select array_agg(plang_language) from person_language where plang_personid = id group by plang_personid) as languages, 16 | p1.pl_name, 17 | (select array_agg(ARRAY[o2.o_name, pu_classyear::text, p2.pl_name]) from person_university, organisation o2, place p2 where pu_personid = id and pu_organisationid = o2.o_organisationid and o2.o_placeid = p2.pl_placeid group by pu_personid) as university, 18 | (select array_agg(ARRAY[o3.o_name, pc_workfrom::text, p3.pl_name]) from person_company, organisation o3, place p3 where pc_personid = id and pc_organisationid = o3.o_organisationid and o3.o_placeid = p3.pl_placeid group by pc_personid) as company 19 | from 20 | ( 21 | select k_person2id as id, 1 as dist from knows, person where k_person1id = :personId and p_personid = k_person2id and p_firstname = :firstName 22 | union all 23 | select b.k_person2id as id, 2 as dist from knows a, knows b, person 24 | where a.k_person1id = :personId 25 | and b.k_person1id = a.k_person2id 26 | and p_personid = b.k_person2id 27 | and p_firstname = :firstName 28 | and p_personid != :personId -- excluding start person 29 | union all 30 | select c.k_person2id as id, 3 as dist from knows a, knows b, knows c, person 31 | where a.k_person1id = :personId 32 | and b.k_person1id = a.k_person2id 33 | and b.k_person2id = c.k_person1id 34 | and p_personid = c.k_person2id 35 | and p_firstname = :firstName 36 | and p_personid != :personId -- excluding start person 37 | ) tmp, person, place p1 38 | where 39 | p_personid = id and 40 | p_placeid = p1.pl_placeid 41 | group by id, p_lastname, p_birthday, p_creationdate, p_gender, p_browserused, p_locationip, p1.pl_name 42 | order by dist, p_lastname, id 43 | LIMIT 20 44 | ; 45 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-10.sql: -------------------------------------------------------------------------------- 1 | /* Q10. Friend recommendation 2 | \set personId 4398046511333 3 | \set month 5 4 | */ 5 | select p_personid, p_firstname, p_lastname, 6 | ( select count(distinct m_messageid) 7 | from message, message_tag pt1 8 | where 9 | m_creatorid = p_personid and 10 | m_c_replyof IS NULL and -- post, not comment 11 | m_messageid = mt_messageid and 12 | exists (select * from person_tag where pt_personid = :personId and pt_tagid = pt1.mt_tagid) 13 | ) - 14 | ( select count(*) 15 | from message 16 | where 17 | m_creatorid = p_personid and 18 | m_c_replyof IS NULL and -- post, not comment 19 | not exists (select * from person_tag, message_tag where pt_personid = :personId and pt_tagid = mt_tagid and mt_messageid = m_messageid) 20 | ) as score, 21 | p_gender, pl_name 22 | from person, place, 23 | ( select distinct k2.k_person2id 24 | from knows k1, knows k2 25 | where 26 | k1.k_person1id = :personId and k1.k_person2id = k2.k_person1id and k2.k_person2id <> :personId and 27 | not exists (select * from knows where k_person1id = :personId and k_person2id = k2.k_person2id) 28 | ) f 29 | where 30 | p_placeid = pl_placeid and 31 | p_personid = f.k_person2id and 32 | ( 33 | (extract(month from p_birthday) = :month and (case when extract(day from p_birthday) >= 21 then true else false end)) 34 | or 35 | (extract(month from p_birthday) = :month % 12 + 1 and (case when extract(day from p_birthday) < 22 then true else false end)) 36 | ) 37 | order by score desc, p_personid 38 | limit 10 39 | ; 40 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-11.sql: -------------------------------------------------------------------------------- 1 | /* Q11. Job referral 2 | \set personId 10995116277918 3 | \set countryName '\'Hungary\'' 4 | \set workFromYear 2011 5 | */ 6 | select p_personid,p_firstname, p_lastname, o_name, pc_workfrom 7 | from person, person_company, organisation, place, 8 | ( select k_person2id 9 | from knows 10 | where 11 | k_person1id = :personId 12 | union 13 | select k2.k_person2id 14 | from knows k1, knows k2 15 | where 16 | k1.k_person1id = :personId and k1.k_person2id = k2.k_person1id and k2.k_person2id <> :personId 17 | ) f 18 | where 19 | p_personid = f.k_person2id and 20 | p_personid = pc_personid and 21 | pc_organisationid = o_organisationid and 22 | pc_workfrom < :workFromYear and 23 | o_placeid = pl_placeid and 24 | pl_name = :countryName 25 | order by pc_workfrom, p_personid, o_name desc 26 | limit 10 27 | ; 28 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-12.sql: -------------------------------------------------------------------------------- 1 | /* Q12. Expert search 2 | \set personId 10995116278009 3 | \set tagClassName '\'Monarch\'' 4 | */ 5 | with recursive extended_tags(s_subtagclassid,s_supertagclassid) as ( 6 | select tc_tagclassid, tc_tagclassid from tagclass 7 | UNION 8 | select tc.tc_tagclassid, t.s_supertagclassid from tagclass tc, extended_tags t 9 | where tc.tc_subclassoftagclassid=t.s_subtagclassid 10 | ) 11 | select p_personid, p_firstname, p_lastname, array_agg(distinct t_name), count(*) as replyCount 12 | from person, message p1, knows, message p2, message_tag, 13 | (select distinct t_tagid, t_name from tag where (t_tagclassid in ( 14 | select distinct s_subtagclassid from extended_tags k, tagclass 15 | where tc_tagclassid = k.s_supertagclassid and tc_name = :tagClassName) 16 | )) selected_tags 17 | where 18 | k_person1id = :personId and 19 | k_person2id = p_personid and 20 | p_personid = p1.m_creatorid and 21 | p1.m_c_replyof = p2.m_messageid and 22 | p2.m_c_replyof is null and 23 | p2.m_messageid = mt_messageid and 24 | mt_tagid = t_tagid 25 | group by p_personid, p_firstname, p_lastname 26 | order by replyCount desc, p_personid asc 27 | limit 20 28 | ; 29 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-13.sql: -------------------------------------------------------------------------------- 1 | /* Q13. Single shortest path 2 | \set person1Id 8796093022390 3 | \set person2Id 8796093022357 4 | */ 5 | WITH RECURSIVE search_graph(link, depth, path) AS ( 6 | SELECT :person1Id::bigint, 0, ARRAY[:person1Id::bigint]::bigint[] 7 | UNION ALL 8 | (WITH sg(link,depth) as (select * from search_graph) -- Note: sg is only the diff produced in the previous iteration 9 | SELECT distinct k_person2id, x.depth+1, array_append(path, k_person2id) 10 | FROM knows, sg x 11 | WHERE 1=1 12 | and x.link = k_person1id 13 | and k_person2id <> ALL (path) 14 | -- stop if we have reached person2 in the previous iteration 15 | and not exists(select * from sg y where y.link = :person2Id::bigint) 16 | -- skip reaching persons reached in the previous iteration 17 | and not exists(select * from sg y where y.link = k_person2id) 18 | ) 19 | ) 20 | select max(depth) from ( 21 | select depth from search_graph where link = :person2Id::bigint 22 | union select -1) tmp; 23 | ; 24 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-14.sql: -------------------------------------------------------------------------------- 1 | /* Q14. Trusted connection paths 2 | \set person1Id 8796093022357 3 | \set person2Id 8796093022390 4 | */ 5 | WITH start_node(v) AS ( 6 | SELECT :person1Id::bigint 7 | ) 8 | select * from ( 9 | WITH RECURSIVE 10 | search_graph(link, depth, path) AS ( 11 | (SELECT v::bigint, 0, ARRAY[]::bigint[][] from start_node) 12 | UNION ALL 13 | (WITH sg(link, depth) as (select * from search_graph) 14 | SELECT distinct k_person2id, x.depth + 1, path || ARRAY[[x.link, k_person2id]] 15 | FROM knows, sg x 16 | WHERE x.link = k_person1id and not exists(select * from sg y where y.link = :person2Id::bigint) and not exists(select * from sg y where y.link=k_person2id) 17 | ) 18 | ), 19 | paths(pid, path) AS ( 20 | SELECT row_number() OVER (), path FROM search_graph where link = :person2Id::bigint 21 | ), 22 | edges(id, e) AS ( 23 | SELECT pid, array_agg(path[d1][d2]) 24 | FROM paths, generate_subscripts(path, 1) d1, generate_subscripts(path, 2) d2 25 | GROUP BY pid, d1 26 | ), 27 | unique_edges(e) AS ( 28 | SELECT DISTINCT e from edges 29 | ), 30 | weights(we, score) as ( 31 | select e, sum(score) from ( 32 | select e, mid1, mid2, max(score) as score from ( 33 | select e, 1 as score, p1.m_messageid as mid1, p2.m_messageid as mid2 from unique_edges, message p1, message p2 where (p1.m_creatorid=e[1] and p2.m_creatorid=e[2] and p2.m_c_replyof=p1.m_messageid and p1.m_c_replyof is null) 34 | union all 35 | select e, 1 as score, p1.m_messageid as mid1, p2.m_messageid as mid2 from unique_edges, message p1, message p2 where (p1.m_creatorid=e[2] and p2.m_creatorid=e[1] and p2.m_c_replyof=p1.m_messageid and p1.m_c_replyof is null) 36 | union all 37 | select e, 0.5 as score, p1.m_messageid as mid1, p2.m_messageid as mid2 from unique_edges, message p1, message p2 where (p1.m_creatorid=e[1] and p2.m_creatorid=e[2] and p2.m_c_replyof=p1.m_messageid and p1.m_c_replyof is not null) 38 | union all 39 | select e, 0.5 as score, p1.m_messageid as mid1, p2.m_messageid as mid2 from unique_edges, message p1, message p2 where (p1.m_creatorid=e[2] and p2.m_creatorid=e[1] and p2.m_c_replyof=p1.m_messageid and p1.m_c_replyof is not null) 40 | ) pps group by e, mid1, mid2 41 | ) tmp 42 | group by e 43 | ), 44 | weightedpaths(path, score) as ( 45 | select path, coalesce(sum(score), 0) from paths, edges left join weights on we=e where pid=id group by id, path 46 | ) 47 | select path, score from weightedpaths order by score desc) 48 | x order by score desc; 49 | ; 50 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-2.sql: -------------------------------------------------------------------------------- 1 | /* Q2. Recent messages by your friends 2 | \set personId 10995116278009 3 | \set maxDate '\'2010-10-16\'' 4 | */ 5 | EXPLAIN (ANALYZE, FORMAT JSON) 6 | select p_personid, p_firstname, p_lastname, m_messageid, COALESCE(m_ps_imagefile, m_content), m_creationdate 7 | from person, message, knows 8 | where 9 | p_personid = m_creatorid and 10 | m_creationdate <= to_timestamp(1354060800000/1000)::date and 11 | k_person1id = 17592186052613 and 12 | k_person2id = p_personid 13 | order by m_creationdate desc, m_messageid asc 14 | limit 20 15 | ; 16 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-3.sql: -------------------------------------------------------------------------------- 1 | /* Q3. Friends and friends of friends that have been to given countries 2 | \set personId 6597069766734 3 | \set countryXName '\'Angola\'' 4 | \set countryYName '\'Colombia\'' 5 | \set startDate '\'2010-06-01\''::date 6 | \set durationDays 28 7 | */ 8 | select p_personid, p_firstname, p_lastname, ct1, ct2, total as totalcount 9 | from 10 | ( select k_person2id 11 | from knows 12 | where 13 | k_person1id = :personId 14 | union 15 | select k2.k_person2id 16 | from knows k1, knows k2 17 | where 18 | k1.k_person1id = :personId and k1.k_person2id = k2.k_person1id and k2.k_person2id <> :personId 19 | ) f, person, place p1, place p2, 20 | ( 21 | select chn.m_c_creatorid, ct1, ct2, ct1 + ct2 as total 22 | from 23 | ( 24 | select m_creatorid as m_c_creatorid, count(*) as ct1 from message, place 25 | where 26 | m_locationid = pl_placeid and pl_name = :countryXName and 27 | m_creationdate >= :startDate and m_creationdate < (:startDate + INTERVAL '1 days' * :durationDays) 28 | group by m_c_creatorid 29 | ) chn, 30 | ( 31 | select m_creatorid as m_c_creatorid, count(*) as ct2 from message, place 32 | where 33 | m_locationid = pl_placeid and pl_name = :countryYName and 34 | m_creationdate >= :startDate and m_creationdate < (:startDate + INTERVAL '1 days' * :durationDays) 35 | group by m_creatorid --m_c_creatorid 36 | ) ind 37 | where CHN.m_c_creatorid = IND.m_c_creatorid 38 | ) cpc 39 | where 40 | f.k_person2id = p_personid and p_placeid = p1.pl_placeid and 41 | p1.pl_containerplaceid = p2.pl_placeid and p2.pl_name <> :countryXName and p2.pl_name <> :countryYName and 42 | f.k_person2id = cpc.m_c_creatorid 43 | order by totalcount desc, p_personid asc 44 | limit 20 45 | ; 46 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-4.sql: -------------------------------------------------------------------------------- 1 | /* Q4. New topics 2 | \set personId 10995116277918 3 | \set startDate '\'2010-10-01\''::date 4 | \set durationDays 31 5 | */ 6 | EXPLAIN (ANALYZE, FORMAT JSON) 7 | select m_creationdate, count(*) as postCount 8 | from tag, message, message_tag recent, knows 9 | where 10 | m_messageid = mt_messageid and 11 | mt_tagid = t_tagid and 12 | m_creatorid = k_person2id and 13 | m_c_replyof IS NULL and -- post, not comment 14 | k_person1id = 21990232559429 and 15 | m_creationdate >= to_timestamp(1335830400000/1000)::date and 16 | m_creationdate < (to_timestamp(1335830400000/1000)::date + INTERVAL '1 days' * 37) and 17 | not exists ( 18 | select * from 19 | (select distinct mt_tagid from message, message_tag, knows 20 | where 21 | k_person1id = 21990232559429 and 22 | k_person2id = m_creatorid and 23 | m_c_replyof IS NULL and -- post, not comment 24 | mt_messageid = m_messageid and 25 | m_creationdate < to_timestamp(1335830400000/1000)::date) tags 26 | where tags.mt_tagid = recent.mt_tagid) 27 | group by m_creationdate 28 | order by postCount desc, m_creationdate asc 29 | limit 10 30 | ; 31 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-5.sql: -------------------------------------------------------------------------------- 1 | /* Q5 New groups 2 | \set personId 6597069766734 3 | \set minDate '\'2010-11-01\''::date 4 | */ 5 | select f_title, count(m_messageid) AS postCount 6 | from ( 7 | select f_title, f_forumid, f.k_person2id 8 | from forum, forum_person, 9 | ( select k_person2id 10 | from knows 11 | where 12 | k_person1id = :personId 13 | union 14 | select k2.k_person2id 15 | from knows k1, knows k2 16 | where 17 | k1.k_person1id = :personId and k1.k_person2id = k2.k_person1id and k2.k_person2id <> :personId 18 | ) f 19 | where f_forumid = fp_forumid and fp_personid = f.k_person2id and 20 | fp_joindate >= :minDate 21 | ) tmp left join message 22 | on tmp.f_forumid = m_ps_forumid and m_creatorid = tmp.k_person2id 23 | group by f_forumid, f_title 24 | order by postCount desc, f_forumid asc 25 | limit 20 26 | ; 27 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-6.sql: -------------------------------------------------------------------------------- 1 | /* Q6. Tag co-occurrence 2 | \set personId 4398046511333 3 | \set tagName '\'Carl_Gustaf_Emil_Mannerheim\'' 4 | */ 5 | select t_name, count(*) as postcount 6 | from tag, message_tag, message, 7 | ( select k_person2id 8 | from knows 9 | where 10 | k_person1id = :personId 11 | union 12 | select k2.k_person2id 13 | from knows k1, knows k2 14 | where 15 | k1.k_person1id = :personId and k1.k_person2id = k2.k_person1id and k2.k_person2id <> :personId 16 | ) f 17 | where 18 | m_creatorid = f.k_person2id and 19 | m_c_replyof IS NULL and -- post, not comment 20 | m_messageid = mt_messageid and 21 | mt_tagid = t_tagid and 22 | t_name <> :tagName and 23 | exists (select * from tag, message_tag where mt_messageid = m_messageid and mt_tagid = t_tagid and t_name = :tagName) 24 | group by t_name 25 | order by postCount desc, t_name asc 26 | limit 10 27 | ; 28 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-7.sql: -------------------------------------------------------------------------------- 1 | /* Q7. Recent likers 2 | \set personId 4398046511268 3 | */ 4 | select p_personid, p_firstname, p_lastname, l.l_creationdate, m_messageid, 5 | COALESCE(m_ps_imagefile, m_content), 6 | CAST(floor(EXTRACT(EPOCH FROM (l.l_creationdate - m_creationdate))) AS INTEGER) / 60 as minutesLatency, 7 | (case when exists (select 1 from knows where k_person1id = :personId and k_person2id = p_personid) then 0 else 1 end) as isnew 8 | from 9 | (select l_personid, max(l_creationdate) as l_creationdate 10 | from likes, message 11 | where 12 | m_messageid = l_messageid and 13 | m_creatorid = :personId 14 | group by l_personid 15 | order by l_creationdate desc 16 | limit 20 17 | ) tmp, message, person, likes as l 18 | where 19 | p_personid = tmp.l_personid and 20 | tmp.l_personid = l.l_personid and 21 | tmp.l_creationdate = l.l_creationdate and 22 | l.l_messageid = m_messageid 23 | order by l_creationdate desc, p_personid asc 24 | ; 25 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-8.sql: -------------------------------------------------------------------------------- 1 | /* Q8. Recent replies 2 | \set personId 143 3 | */ 4 | select p1.m_creatorid, p_firstname, p_lastname, p1.m_creationdate, p1.m_messageid, p1.m_content 5 | from message p1, message p2, person 6 | where 7 | p1.m_c_replyof = p2.m_messageid and 8 | p2.m_creatorid = :personId and 9 | p_personid = p1.m_creatorid 10 | order by p1.m_creationdate desc, p1.m_messageid asc 11 | limit 20 12 | ; 13 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-complex-9.sql: -------------------------------------------------------------------------------- 1 | /* Q9. Recent messages by friends or friends of friends 2 | \set personId 4398046511268 3 | \set maxDate '\'2010-11-16\''::date 4 | */ 5 | select p_personid, p_firstname, p_lastname, 6 | m_messageid, COALESCE(m_ps_imagefile, m_content), m_creationdate 7 | from 8 | ( select k_person2id 9 | from knows 10 | where k_person1id = :personId 11 | union 12 | select k2.k_person2id 13 | from knows k1, knows k2 14 | where k1.k_person1id = :personId 15 | and k1.k_person2id = k2.k_person1id 16 | and k2.k_person2id <> :personId 17 | ) f, person, message 18 | where 19 | p_personid = m_creatorid and p_personid = f.k_person2id and 20 | m_creationdate < :maxDate 21 | order by m_creationdate desc, m_messageid asc 22 | limit 20 23 | ; 24 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-1.sql: -------------------------------------------------------------------------------- 1 | /* IS1. Profile of a person 2 | \set personId 10995116277794 3 | */ 4 | select p_firstname, p_lastname, p_birthday, p_locationip, p_browserused, p_placeid, p_gender, p_creationdate 5 | from person 6 | where p_personid = :personId; 7 | ; 8 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-2.sql: -------------------------------------------------------------------------------- 1 | /* IS2. Recent messages of a person 2 | \set personId 10995116277795 3 | */ 4 | EXPLAIN (ANALYZE, FORMAT JSON) 5 | with recursive cposts(m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof, m_creatorid) AS ( 6 | select m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof, m_creatorid 7 | from message 8 | where m_creatorid = 933 9 | order by m_creationdate desc 10 | limit 10 11 | ), parent(postid,replyof,orig_postid,creator) AS ( 12 | select m_messageid, m_c_replyof, m_messageid, m_creatorid from cposts 13 | UNION ALL 14 | select m_messageid, m_c_replyof, orig_postid, m_creatorid 15 | from message,parent 16 | where m_messageid=replyof 17 | ) 18 | select p1.m_messageid, COALESCE(m_ps_imagefile, m_content, ''), p1.m_creationdate, 19 | p2.m_messageid, p2.p_personid, p2.p_firstname, p2.p_lastname 20 | from 21 | (select m_messageid, m_content, m_ps_imagefile, m_creationdate, m_c_replyof from cposts 22 | ) p1 23 | left join 24 | (select orig_postid, postid as m_messageid, p_personid, p_firstname, p_lastname 25 | from parent, person 26 | where replyof is null and creator = p_personid 27 | )p2 28 | on p2.orig_postid = p1.m_messageid 29 | order by m_creationdate desc, p2.m_messageid desc; 30 | ; 31 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-3.sql: -------------------------------------------------------------------------------- 1 | /* IS3. Friends of a person 2 | \set personId 10995116277794 3 | */ 4 | select p_personid, p_firstname, p_lastname, k_creationdate 5 | from knows, person 6 | where k_person1id = :personId and k_person2id = p_personid 7 | order by k_creationdate desc, p_personid asc; 8 | ; 9 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-4.sql: -------------------------------------------------------------------------------- 1 | /* IS4. Content of a message 2 | \set messageId 206158431836 3 | */ 4 | select COALESCE(m_ps_imagefile, m_content, ''), m_creationdate 5 | from message 6 | where m_messageid = :messageId; 7 | ; 8 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-5.sql: -------------------------------------------------------------------------------- 1 | /* IS5. Creator of a message 2 | \set messageId 206158431836 3 | */ 4 | select p_personid, p_firstname, p_lastname 5 | from message, person 6 | where m_messageid = :messageId and m_creatorid = p_personid; 7 | ; 8 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-6.sql: -------------------------------------------------------------------------------- 1 | /* IS6. Forum of a message 2 | \set messageId 206158431836 3 | */ 4 | WITH RECURSIVE chain(parent, child) as( 5 | SELECT m_c_replyof, m_messageid FROM message where m_messageid = :messageId 6 | UNION ALL 7 | SELECT p.m_c_replyof, p.m_messageid FROM message p, chain c where p.m_messageid = c.parent 8 | ) 9 | select f_forumid, f_title, p_personid, p_firstname, p_lastname 10 | from message, person, forum 11 | where m_messageid = (select coalesce(min(parent), :messageId) from chain) 12 | and m_ps_forumid = f_forumid and f_moderatorid = p_personid; 13 | ; 14 | -------------------------------------------------------------------------------- /ldbc-converter/original-queries/workload/interactive-short-7.sql: -------------------------------------------------------------------------------- 1 | /* IS7. Replies of a message 2 | \set messageId 206158432794 3 | */ 4 | select p2.m_messageid, p2.m_content, p2.m_creationdate, p_personid, p_firstname, p_lastname, 5 | (case when exists ( 6 | select 1 from knows 7 | where p1.m_creatorid = k_person1id and p2.m_creatorid = k_person2id) 8 | then TRUE 9 | else FALSE 10 | end) 11 | from message p1, message p2, person 12 | where 13 | p1.m_messageid = :messageId and p2.m_c_replyof = p1.m_messageid and p2.m_creatorid = p_personid 14 | order by p2.m_creationdate desc, p2.m_creatorid asc; 15 | ; 16 | -------------------------------------------------------------------------------- /ldbc-converter/plan-experiments/extract_plan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | FILENAME=$1 4 | 5 | rm -rf result.txt 6 | 7 | psql -h 127.0.0.1 -U gpadmin -d ldbc-schemaless -f $FILENAME >> result.txt -------------------------------------------------------------------------------- /ldbc-converter/query-generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | from pathlib import Path 6 | import utils 7 | import sys 8 | import yaml 9 | 10 | def generateUnionQuery(configuration_yaml, schema_yaml): 11 | # Get the table name 12 | target_table_name = configuration_yaml['table_name'] 13 | 14 | # Geneate a union query for the table 15 | # Make empty string 16 | union_query = '' 17 | # Append '(' 18 | union_query += '(' 19 | # Iterate over the partitions 20 | partitions = configuration_yaml['partitions'] 21 | for i, partition in enumerate(partitions): 22 | # Get schema columns 23 | schema_columns = schema_yaml['columns'] 24 | # Get columns not removed 25 | columns_wo_removing = utils.getSchemaColumnsWithoutRemovedColumns(schema_columns, configuration_yaml[partition]) 26 | # Append '(SELECT' 27 | union_query += '(SELECT ' 28 | # Iterate over the schema column 29 | for j, schema_column in enumerate(schema_columns): 30 | # Check if the column is removed. 31 | if schema_column not in columns_wo_removing: 32 | # If so, append ‘NULL as ’ 33 | union_query += ('NULL as ' + schema_column) 34 | else: 35 | union_query += schema_column 36 | # Append comma is not the last column 37 | if j < len(schema_columns) - 1: 38 | union_query += ', ' 39 | # Append ‘FROM ’ 40 | partition_table_name = utils.getPartitionTableBasedName(target_table_name, partition) 41 | union_query += (' FROM ' + partition_table_name) 42 | # If not the last partition, append ‘) \n UNION \n’ 43 | if i < len(partitions) - 1: 44 | union_query += ') \n UNION ALL \n' 45 | # Else, append ‘) \ 46 | else: 47 | union_query += ')' 48 | # Append 'as
' 49 | union_query += (') as ' + target_table_name) 50 | return union_query 51 | 52 | def exportQuery(output_queries_path, query_file_name, query): 53 | query_file_path = os.path.join(output_queries_path, query_file_name + '.sql') 54 | with open(query_file_path, 'w') as query_file: 55 | query_file.write(query) 56 | 57 | 58 | configuration_file_path = sys.argv[1] 59 | 60 | # Get configuration file 61 | with open(configuration_file_path, "r") as configuration_file: 62 | configuration_yaml = yaml.load(configuration_file) 63 | schema_file_path = configuration_yaml['schema_file'] 64 | table_name = configuration_yaml['table_name'] 65 | with open(schema_file_path, "r") as schema_file: 66 | schema_yaml = yaml.load(schema_file) 67 | union_query = generateUnionQuery(configuration_yaml, schema_yaml) 68 | exportQuery('generated-queries/union-all/', table_name, union_query) 69 | 70 | -------------------------------------------------------------------------------- /ldbc-converter/results/analyze.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/ldbc-converter/results/analyze.err -------------------------------------------------------------------------------- /ldbc-converter/results/analyze.log: -------------------------------------------------------------------------------- 1 | ANALYZE 2 | -------------------------------------------------------------------------------- /ldbc-converter/results/bench.log: -------------------------------------------------------------------------------- 1 | 2023-08-06 08:57:04 [1691312224] : preparing LDBC database 2 | 2023-08-06 08:57:04 [1691312224] : loading data 3 | 2023-08-06 08:57:04 [1691312224] : schema constraints 4 | 2023-08-06 08:57:04 [1691312224] : creating foreign keys 5 | 2023-08-06 08:57:04 [1691312224] : analyzing 6 | 2023-08-06 08:57:25 [1691312245] : preparing LDBC database 7 | 2023-08-06 08:57:25 [1691312245] : loading data 8 | 2023-08-06 08:57:25 [1691312245] : schema constraints 9 | 2023-08-06 08:57:25 [1691312245] : creating foreign keys 10 | 2023-08-06 08:57:25 [1691312245] : analyzing 11 | 2023-08-06 08:57:43 [1691312263] : preparing LDBC database 12 | 2023-08-06 08:57:43 [1691312263] : loading data 13 | 2023-08-06 08:57:43 [1691312263] : schema constraints 14 | 2023-08-06 08:57:43 [1691312263] : creating foreign keys 15 | 2023-08-06 08:57:43 [1691312263] : analyzing 16 | 2023-08-06 08:58:13 [1691312293] : preparing LDBC database 17 | 2023-08-06 08:58:13 [1691312293] : loading data 18 | 2023-08-06 08:58:13 [1691312293] : schema constraints 19 | 2023-08-06 08:58:13 [1691312293] : creating foreign keys 20 | 2023-08-06 08:58:13 [1691312293] : analyzing 21 | 2023-08-06 08:58:31 [1691312311] : preparing LDBC database 22 | 2023-08-06 08:58:31 [1691312311] : loading data 23 | 2023-08-06 08:58:31 [1691312311] : schema constraints 24 | 2023-08-06 08:58:31 [1691312311] : creating foreign keys 25 | 2023-08-06 08:58:31 [1691312311] : analyzing 26 | 2023-08-06 08:59:22 [1691312362] : preparing LDBC database 27 | 2023-08-06 08:59:22 [1691312362] : loading data 28 | 2023-08-06 08:59:22 [1691312362] : schema constraints 29 | 2023-08-06 08:59:23 [1691312363] : creating foreign keys 30 | 2023-08-06 08:59:23 [1691312363] : analyzing 31 | 2023-08-06 09:02:54 [1691312574] : preparing LDBC database 32 | 2023-08-06 09:02:54 [1691312574] : loading data 33 | 2023-08-06 09:02:54 [1691312574] : schema constraints 34 | 2023-08-06 09:02:55 [1691312575] : creating foreign keys 35 | 2023-08-06 09:02:55 [1691312575] : analyzing 36 | 2023-08-06 09:04:18 [1691312658] : preparing LDBC database 37 | 2023-08-06 09:04:18 [1691312658] : loading data 38 | 2023-08-06 09:04:18 [1691312658] : schema constraints 39 | 2023-08-06 09:04:19 [1691312659] : creating foreign keys 40 | 2023-08-06 09:04:19 [1691312659] : analyzing 41 | 2023-08-06 09:05:25 [1691312725] : preparing LDBC database 42 | 2023-08-06 09:05:26 [1691312726] : loading data 43 | 2023-08-06 09:05:38 [1691312738] : schema constraints 44 | 2023-08-06 09:05:42 [1691312742] : creating foreign keys 45 | 2023-08-06 09:05:42 [1691312742] : analyzing 46 | 2023-08-06 09:22:24 [1691313744] : preparing LDBC database 47 | 2023-08-06 09:22:24 [1691313744] : loading data 48 | 2023-08-06 09:22:43 [1691313763] : schema constraints 49 | 2023-08-06 09:23:16 [1691313796] : creating foreign keys 50 | 2023-08-06 09:23:17 [1691313797] : analyzing 51 | 2023-08-06 09:28:40 [1691314120] : preparing LDBC database 52 | 2023-08-06 09:28:41 [1691314121] : loading data 53 | 2023-08-06 09:29:00 [1691314140] : schema constraints 54 | 2023-08-06 09:29:36 [1691314176] : analyzing 55 | 2023-08-06 09:33:03 [1691314383] : preparing LDBC database 56 | 2023-08-06 09:33:04 [1691314384] : loading data 57 | 2023-08-06 09:33:19 [1691314399] : schema constraints 58 | 2023-08-06 09:33:55 [1691314435] : analyzing 59 | 2023-08-06 11:45:51 [1691322351] : preparing LDBC database 60 | 2023-08-06 11:45:51 [1691322351] : loading data 61 | 2023-08-06 11:46:11 [1691322371] : schema constraints 62 | 2023-08-06 11:46:47 [1691322407] : analyzing 63 | 2023-08-06 12:12:02 [1691323922] : preparing LDBC database 64 | 2023-08-06 12:12:03 [1691323923] : loading data 65 | 2023-08-06 12:12:19 [1691323939] : schema constraints 66 | 2023-08-06 12:12:54 [1691323974] : analyzing 67 | 2023-08-06 12:19:33 [1691324373] : preparing LDBC database 68 | 2023-08-06 12:19:34 [1691324374] : loading data 69 | 2023-08-06 12:19:53 [1691324393] : schema constraints 70 | 2023-08-06 12:20:30 [1691324430] : analyzing 71 | 2023-08-06 12:33:57 [1691325237] : preparing LDBC database 72 | 2023-08-06 12:34:07 [1691325247] : preparing LDBC database 73 | 2023-08-06 12:34:07 [1691325247] : loading data 74 | 2023-08-06 12:34:25 [1691325265] : schema constraints 75 | 2023-08-06 12:35:02 [1691325302] : analyzing 76 | 2023-08-06 12:37:24 [1691325444] : preparing LDBC database 77 | 2023-08-06 12:37:25 [1691325445] : loading data 78 | 2023-08-06 12:37:45 [1691325465] : schema constraints 79 | 2023-08-06 12:38:22 [1691325502] : analyzing 80 | 2023-08-06 12:43:14 [1691325794] : preparing LDBC database 81 | 2023-08-06 12:43:14 [1691325794] : loading data 82 | 2023-08-06 12:43:35 [1691325815] : schema constraints 83 | 2023-08-06 12:44:10 [1691325850] : analyzing 84 | 2023-08-06 13:26:38 [1691328398] : preparing LDBC database 85 | 2023-08-06 13:26:38 [1691328398] : loading data 86 | 2023-08-06 13:26:39 [1691328399] : schema constraints 87 | 2023-08-06 13:26:39 [1691328399] : analyzing 88 | 2023-08-06 13:28:16 [1691328496] : preparing LDBC database 89 | 2023-08-06 13:28:16 [1691328496] : loading data 90 | 2023-08-06 13:28:39 [1691328519] : schema constraints 91 | 2023-08-06 13:29:39 [1691328579] : analyzing 92 | -------------------------------------------------------------------------------- /ldbc-converter/results/constraints.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/ldbc-converter/results/constraints.err -------------------------------------------------------------------------------- /ldbc-converter/results/constraints.log: -------------------------------------------------------------------------------- 1 | ALTER TABLE 2 | ALTER TABLE 3 | ALTER TABLE 4 | ALTER TABLE 5 | ALTER TABLE 6 | ALTER TABLE 7 | ALTER TABLE 8 | ALTER TABLE 9 | ALTER TABLE 10 | ALTER TABLE 11 | ALTER TABLE 12 | ALTER TABLE 13 | ALTER TABLE 14 | ALTER TABLE 15 | ALTER TABLE 16 | ALTER TABLE 17 | ALTER TABLE 18 | CREATE INDEX 19 | CREATE INDEX 20 | CREATE INDEX 21 | CREATE INDEX 22 | CREATE INDEX 23 | CREATE INDEX 24 | CREATE INDEX 25 | CREATE INDEX 26 | CREATE INDEX 27 | CREATE INDEX 28 | CREATE INDEX 29 | CREATE INDEX 30 | CREATE INDEX 31 | CREATE INDEX 32 | CREATE INDEX 33 | CREATE INDEX 34 | CREATE INDEX 35 | CREATE INDEX 36 | CREATE INDEX 37 | CREATE INDEX 38 | CREATE INDEX 39 | CREATE INDEX 40 | CREATE INDEX 41 | CREATE INDEX 42 | CREATE INDEX 43 | CREATE INDEX 44 | CREATE INDEX 45 | -------------------------------------------------------------------------------- /ldbc-converter/results/create.err: -------------------------------------------------------------------------------- 1 | NOTICE: view "country" does not exist, skipping 2 | NOTICE: table "forum_person" does not exist, skipping 3 | NOTICE: table "forum_tag" does not exist, skipping 4 | NOTICE: table "person_email" does not exist, skipping 5 | NOTICE: table "person_tag" does not exist, skipping 6 | NOTICE: table "person_language" does not exist, skipping 7 | NOTICE: table "person_university" does not exist, skipping 8 | NOTICE: table "person_company" does not exist, skipping 9 | NOTICE: table "message_tag" does not exist, skipping 10 | NOTICE: table "organisation" does not exist, skipping 11 | NOTICE: table "knows" does not exist, skipping 12 | NOTICE: table "likes" does not exist, skipping 13 | NOTICE: table "tag" does not exist, skipping 14 | NOTICE: table "tagclass" does not exist, skipping 15 | NOTICE: table "message" does not exist, skipping 16 | NOTICE: table "forum" does not exist, skipping 17 | NOTICE: table "person" does not exist, skipping 18 | NOTICE: table "place" does not exist, skipping 19 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'm_messageid' as the Greenplum Database data distribution key for this table. 20 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 21 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'm_messageid' as the Greenplum Database data distribution key for this table. 22 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 23 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'f_forumid' as the Greenplum Database data distribution key for this table. 24 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 25 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'fp_forumid' as the Greenplum Database data distribution key for this table. 26 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 27 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'ft_forumid' as the Greenplum Database data distribution key for this table. 28 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 29 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'o_organisationid' as the Greenplum Database data distribution key for this table. 30 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 31 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'p_personid' as the Greenplum Database data distribution key for this table. 32 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 33 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'pe_personid' as the Greenplum Database data distribution key for this table. 34 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 35 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'pt_personid' as the Greenplum Database data distribution key for this table. 36 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 37 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'k_person1id' as the Greenplum Database data distribution key for this table. 38 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 39 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'l_personid' as the Greenplum Database data distribution key for this table. 40 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 41 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'plang_personid' as the Greenplum Database data distribution key for this table. 42 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 43 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'pu_personid' as the Greenplum Database data distribution key for this table. 44 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 45 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'pc_personid' as the Greenplum Database data distribution key for this table. 46 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 47 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'pl_placeid' as the Greenplum Database data distribution key for this table. 48 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 49 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'mt_messageid' as the Greenplum Database data distribution key for this table. 50 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 51 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'tc_tagclassid' as the Greenplum Database data distribution key for this table. 52 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 53 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 't_tagid' as the Greenplum Database data distribution key for this table. 54 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 55 | -------------------------------------------------------------------------------- /ldbc-converter/results/create.log: -------------------------------------------------------------------------------- 1 | DROP VIEW 2 | DROP TABLE 3 | DROP TABLE 4 | DROP TABLE 5 | DROP TABLE 6 | DROP TABLE 7 | DROP TABLE 8 | DROP TABLE 9 | DROP TABLE 10 | DROP TABLE 11 | DROP TABLE 12 | DROP TABLE 13 | DROP TABLE 14 | DROP TABLE 15 | DROP TABLE 16 | DROP TABLE 17 | DROP TABLE 18 | DROP TABLE 19 | CREATE TABLE 20 | CREATE TABLE 21 | CREATE TABLE 22 | CREATE TABLE 23 | CREATE TABLE 24 | CREATE TABLE 25 | CREATE TABLE 26 | CREATE TABLE 27 | CREATE TABLE 28 | CREATE TABLE 29 | CREATE TABLE 30 | CREATE TABLE 31 | CREATE TABLE 32 | CREATE TABLE 33 | CREATE TABLE 34 | CREATE TABLE 35 | CREATE TABLE 36 | CREATE TABLE 37 | -------------------------------------------------------------------------------- /ldbc-converter/results/foreign.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/ldbc-converter/results/foreign.err -------------------------------------------------------------------------------- /ldbc-converter/results/foreign.log: -------------------------------------------------------------------------------- 1 | ALTER TABLE 2 | ALTER TABLE 3 | ALTER TABLE 4 | ALTER TABLE 5 | ALTER TABLE 6 | ALTER TABLE 7 | ALTER TABLE 8 | ALTER TABLE 9 | ALTER TABLE 10 | ALTER TABLE 11 | ALTER TABLE 12 | ALTER TABLE 13 | ALTER TABLE 14 | ALTER TABLE 15 | -------------------------------------------------------------------------------- /ldbc-converter/results/load.err: -------------------------------------------------------------------------------- 1 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. 2 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. 3 | NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'm_messageid' as the Greenplum Database data distribution key for this table. 4 | HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. 5 | -------------------------------------------------------------------------------- /ldbc-converter/results/load.log: -------------------------------------------------------------------------------- 1 | COPY 90492 2 | COPY 1611869 3 | COPY 309766 4 | COPY 7955 5 | COPY 9892 6 | COPY 20975 7 | COPY 229166 8 | COPY 180623 9 | COPY 180623 10 | COPY 751677 11 | COPY 1438418 12 | COPY 21784 13 | COPY 7949 14 | COPY 21654 15 | COPY 1460 16 | COPY 713258 17 | COPY 2698393 18 | COPY 71 19 | COPY 16080 20 | SELECT 1343 21 | COPY 1003605 22 | COPY 2052169 23 | SELECT 3055774 24 | DROP TABLE 25 | DROP TABLE 26 | -------------------------------------------------------------------------------- /ldbc-converter/results/settings.err: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/ldbc-converter/results/settings.err -------------------------------------------------------------------------------- /ldbc-converter/run-ldbc-example.sh: -------------------------------------------------------------------------------- 1 | DBNAME=$1 2 | 3 | dropdb $DBNAME 4 | createdb $DBNAME 5 | 6 | bash ldbc.sh results 127.0.0.1 $DBNAME gpadmin gpadmin -------------------------------------------------------------------------------- /ldbc-converter/schemas/comment.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | m_messageid, 3 | m_creationdate, 4 | m_locationip, 5 | m_browserused, 6 | m_content, 7 | m_length, 8 | m_creatorid, 9 | m_locationid, 10 | m_replyof_post, 11 | m_replyof_comment 12 | ] 13 | m_messageid: bigint not null 14 | m_creationdate: timestamp with time zone not null 15 | m_locationip: varchar not null 16 | m_browserused: varchar not null 17 | m_content: text 18 | m_length: int not null 19 | m_creatorid: bigint not null 20 | m_locationid: bigint not null 21 | m_replyof_post: bigint 22 | m_replyof_comment: bigint 23 | primary_keys: [] 24 | create_option: 25 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/forum.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | f_forumid, 3 | f_title, 4 | f_creationdate, 5 | f_moderatorid 6 | ] 7 | f_forumid: bigint not null 8 | f_title: varchar not null 9 | f_creationdate: timestamp with time zone not null 10 | f_moderatorid: bigint not null 11 | primary_keys: [f_forumid] 12 | create_option: 13 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/forum_person.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | fp_forumid, 3 | fp_personid, 4 | fp_joindate 5 | ] 6 | fp_forumid: bigint not null 7 | fp_personid: bigint not null 8 | fp_joindate: timestamp with time zone not null 9 | primary_keys: [fp_forumid, fp_personid] 10 | create_option: 11 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/organisation.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | o_organisationid, 3 | o_type, 4 | o_name, 5 | o_url, 6 | o_placeid 7 | ] 8 | o_organisationid: bigint not null 9 | o_type: varchar not null 10 | o_name: varchar not null 11 | o_url: varchar not null 12 | o_placeid: bigint not null 13 | primary_keys: [o_organisationid] 14 | create_option: 15 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/person.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | p_personid, 3 | p_firstname, 4 | p_lastname, 5 | p_gender, 6 | p_birthday, 7 | p_creationdate, 8 | p_locationip, 9 | p_browserused, 10 | p_placeid 11 | ] 12 | p_personid: bigint not null 13 | p_firstname: varchar not null 14 | p_lastname: varchar not null 15 | p_gender: varchar not null 16 | p_birthday: date not null 17 | p_creationdate: timestamp with time zone not null 18 | p_locationip: varchar not null 19 | p_browserused: varchar not null 20 | p_placeid: bigint not null 21 | primary_keys: [p_personid] 22 | create_option: 23 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/place.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | pl_placeid, 3 | pl_name, 4 | pl_url, 5 | pl_type, 6 | pl_containerplaceid 7 | ] 8 | pl_placeid: bigint not null 9 | pl_name: varchar not null 10 | pl_url: varchar not null 11 | pl_type: varchar not null 12 | pl_containerplaceid: bigint 13 | primary_keys: [pl_placeid] 14 | create_option: 15 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/post.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | m_messageid, 3 | m_ps_imagefile, 4 | m_creationdate, 5 | m_locationip, 6 | m_browserused, 7 | m_ps_language, 8 | m_content, 9 | m_length, 10 | m_creatorid, 11 | m_ps_forumid, 12 | m_locationid 13 | ] 14 | m_messageid: bigint not null 15 | m_ps_imagefile: varchar 16 | m_creationdate: timestamp with time zone not null 17 | m_locationip: varchar not null 18 | m_browserused: varchar not null 19 | m_ps_language: varchar 20 | m_content: text 21 | m_length: int not null 22 | m_creatorid: bigint not null 23 | m_ps_forumid: bigint 24 | m_locationid: bigint not null 25 | primary_keys: [] 26 | create_option: 27 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/tag.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | t_tagid, 3 | t_name, 4 | t_url, 5 | t_tagclassid 6 | ] 7 | t_tagid: bigint not null 8 | t_name: varchar not null 9 | t_url: varchar not null 10 | t_tagclassid: bigint not null 11 | primary_keys: [t_tagid] 12 | create_option: 13 | -------------------------------------------------------------------------------- /ldbc-converter/schemas/tagClass.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | tc_tagclassid, 3 | tc_name, 4 | tc_url, 5 | tc_subclassoftagclassid 6 | ] 7 | tc_tagclassid: bigint not null 8 | tc_name: varchar not null 9 | tc_url: varchar not null 10 | tc_subclassoftagclassid: bigint 11 | primary_keys: [tc_tagclassid] 12 | create_option: 13 | -------------------------------------------------------------------------------- /ldbc-converter/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | def getOutputFolderPath(configuration_yaml): 5 | output_folders_path = configuration_yaml['output_folders_path'] 6 | output_folder_path = os.path.join(output_folders_path, '') 7 | return output_folder_path 8 | 9 | def getUnifiedOutputFilePath(configuration_yaml, format = 'csv'): 10 | file_name = getUnionFileBaseName(configuration_yaml) + '.' + format 11 | output_folder_path = getOutputFolderPath(configuration_yaml) 12 | output_file_path = os.path.join(output_folder_path, file_name) 13 | return output_file_path 14 | 15 | def getUnionFileBaseName(configuration_yaml): 16 | data_file_path = configuration_yaml['data_file_path'] 17 | base_file_name = Path(data_file_path).stem 18 | return base_file_name 19 | 20 | def getPartitionOutputFilePath(configuration_yaml, partition, format = 'csv'): 21 | file_name = getPartitionFileBaseName(configuration_yaml, partition) + '.' + format 22 | output_folder_path = getOutputFolderPath(configuration_yaml) 23 | output_file_path = os.path.join(output_folder_path, file_name) 24 | return output_file_path 25 | 26 | def getPartitionTableBasedName(table_name, partition): 27 | return table_name + '_' + partition 28 | 29 | def getPartitionFileBaseName(configuration_yaml, partition): 30 | data_file_path = configuration_yaml['data_file_path'] 31 | base_file_name = Path(data_file_path).stem 32 | return base_file_name + '_' + partition 33 | 34 | def getSchemaColumnsWithoutRemovedColumns(schema_columns, partition_configuration): 35 | # If column_removing is active 36 | column_removing_configuration = partition_configuration['column_removing'] 37 | if column_removing_configuration['activate']: 38 | columns_to_remove = column_removing_configuration['columns'] 39 | column_indicies = [schema_columns.index(column_to_remove) for column_to_remove in columns_to_remove] 40 | new_schema_columns = [column for i, column in enumerate(schema_columns) if i not in column_indicies] 41 | return new_schema_columns 42 | else: # If no to-be-removed columns, just return the original one 43 | return schema_columns 44 | 45 | def getRemovedColumns(partition_configuration): 46 | column_removing_configuration = partition_configuration['column_removing'] 47 | if column_removing_configuration['activate']: 48 | return column_removing_configuration['columns'] 49 | else: # If no to-be-removed columns, just return the original one 50 | return [] 51 | 52 | def getTypeConversionColumns(partition_configuration): 53 | type_conversion_configuration = partition_configuration['type_conversion'] 54 | if type_conversion_configuration['activate']: 55 | return type_conversion_configuration['columns'] 56 | else: 57 | return [] 58 | 59 | def getConversionType(partition_configuration, column_name): 60 | type_conversion_configuration = partition_configuration['type_conversion'] 61 | type_conversion_columns = type_conversion_configuration['columns'] 62 | type_columns = type_conversion_configuration['types'] 63 | column_index = type_conversion_columns.index(column_name) 64 | return type_columns[column_index] -------------------------------------------------------------------------------- /tpch-converter/.gitignore: -------------------------------------------------------------------------------- 1 | data/* 2 | results/* 3 | queries/* 4 | ./*.pyc 5 | execution_average_result.txt -------------------------------------------------------------------------------- /tpch-converter/README.md: -------------------------------------------------------------------------------- 1 | # schemaless-data-generator 2 | 3 | For the design description, see [Schemaless Data Experiment on Orca](https://docs.google.com/document/d/1R7ENQvLVNHQ-DG-sga0tfgGWJIkobQT77HQFVKUvxH8/edit#). 4 | 5 | ## Folder Structure 6 | 7 | ``` 8 | ./configuration: yaml files 9 | ./schemas: schema yaml files 10 | ./data: data csv files (not included in the git project) 11 | ./templates: template sql files for data setup sql file generation 12 | ./results: generated csv files and SQL files 13 | ``` 14 | 15 | ## Bash files 16 | 17 | ``` 18 | runner.sh: move generated files and run the experiment. 19 | ``` 20 | 21 | ## Usage 22 | ``` 23 | // to generate files 24 | python generator.py 25 | // to run TPCH benchmark using generated dataset 26 | bash runner.sh 27 | // mode means repetitive running or just single run. On repetitive running by r option 28 | // e.g, bash runner.sh tpch ./results/test_supplier r 29 | ``` 30 | 31 | ## TODOs 32 | 33 | In sqlFileGenerator.py, Replace this into class, and make configuration yaml and sceham yaml as a class member. 34 | 35 | How to handle table alias?? for example, SELECT .. FROM supplier as s; . We cannot just substitute supplier 36 | 37 | How to handler supplier_no or agg_lineitem?? (simple way? read one more character in the front and the back.) -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_1.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_1 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1] 9 | p1: 10 | portion: 100 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_2.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_2 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: true 13 | columns: [L_EXTENDEDPRICE, L_DISCOUNT] 14 | type_conversion: 15 | activate: false 16 | p2: 17 | portion: 50 18 | column_removing: 19 | activate: true 20 | columns: [L_SHIPDATE] 21 | type_conversion: 22 | activate: true 23 | columns: [L_QUANTITY] 24 | types: [CHAR(20)] -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_3.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_3 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 25 17 | column_removing: 18 | activate: true 19 | columns: [L_SHIPMODE, L_COMMENT] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 25 24 | column_removing: 25 | activate: true 26 | columns: [L_SHIPMODE] 27 | type_conversion: 28 | activate: true 29 | columns: [L_EXTENDEDPRICE] 30 | types: [CHAR(20)] -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_4.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_4 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 25 17 | column_removing: 18 | activate: true 19 | columns: [L_SHIPMODE] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 25 24 | column_removing: 25 | activate: true 26 | columns: [L_COMMENT] 27 | type_conversion: 28 | activate: false 29 | columns: [] 30 | types: [] -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_5.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_5 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] 9 | p1: 10 | portion: 10 11 | column_removing: 12 | activate: true 13 | columns: [ 14 | L_EXTENDEDPRICE, 15 | L_DISCOUNT, 16 | L_TAX, 17 | L_RETURNFLAG, 18 | L_LINESTATUS, 19 | L_SHIPDATE, 20 | L_COMMITDATE, 21 | L_RECEIPTDATE, 22 | L_SHIPINSTRUCT, 23 | ] 24 | type_conversion: 25 | activate: false 26 | p2: 27 | portion: 10 28 | column_removing: 29 | activate: true 30 | columns: [ 31 | L_QUANTITY, 32 | L_DISCOUNT, 33 | L_TAX, 34 | L_RETURNFLAG, 35 | L_LINESTATUS, 36 | L_SHIPDATE, 37 | L_COMMITDATE, 38 | L_RECEIPTDATE, 39 | L_SHIPINSTRUCT, 40 | ] 41 | type_conversion: 42 | activate: false 43 | p3: 44 | portion: 10 45 | column_removing: 46 | activate: true 47 | columns: [ 48 | L_QUANTITY, 49 | L_EXTENDEDPRICE, 50 | L_TAX, 51 | L_RETURNFLAG, 52 | L_LINESTATUS, 53 | L_SHIPDATE, 54 | L_COMMITDATE, 55 | L_RECEIPTDATE, 56 | L_SHIPINSTRUCT, 57 | ] 58 | type_conversion: 59 | activate: false 60 | p4: 61 | portion: 10 62 | column_removing: 63 | activate: true 64 | columns: [ 65 | L_QUANTITY, 66 | L_EXTENDEDPRICE, 67 | L_DISCOUNT, 68 | L_RETURNFLAG, 69 | L_LINESTATUS, 70 | L_SHIPDATE, 71 | L_COMMITDATE, 72 | L_RECEIPTDATE, 73 | L_SHIPINSTRUCT, 74 | ] 75 | type_conversion: 76 | activate: false 77 | p5: 78 | portion: 10 79 | column_removing: 80 | activate: true 81 | columns: [ 82 | L_QUANTITY, 83 | L_EXTENDEDPRICE, 84 | L_DISCOUNT, 85 | L_TAX, 86 | L_LINESTATUS, 87 | L_SHIPDATE, 88 | L_COMMITDATE, 89 | L_RECEIPTDATE, 90 | L_SHIPINSTRUCT, 91 | ] 92 | type_conversion: 93 | activate: false 94 | p6: 95 | portion: 10 96 | column_removing: 97 | activate: true 98 | columns: [ 99 | L_QUANTITY, 100 | L_EXTENDEDPRICE, 101 | L_DISCOUNT, 102 | L_TAX, 103 | L_RETURNFLAG, 104 | L_SHIPDATE, 105 | L_COMMITDATE, 106 | L_RECEIPTDATE, 107 | L_SHIPINSTRUCT, 108 | ] 109 | type_conversion: 110 | activate: false 111 | p7: 112 | portion: 10 113 | column_removing: 114 | activate: true 115 | columns: [ 116 | L_QUANTITY, 117 | L_EXTENDEDPRICE, 118 | L_DISCOUNT, 119 | L_TAX, 120 | L_RETURNFLAG, 121 | L_LINESTATUS, 122 | L_COMMITDATE, 123 | L_RECEIPTDATE, 124 | L_SHIPINSTRUCT, 125 | ] 126 | type_conversion: 127 | activate: false 128 | p8: 129 | portion: 10 130 | column_removing: 131 | activate: true 132 | columns: [ 133 | L_QUANTITY, 134 | L_EXTENDEDPRICE, 135 | L_DISCOUNT, 136 | L_TAX, 137 | L_RETURNFLAG, 138 | L_LINESTATUS, 139 | L_SHIPDATE, 140 | L_RECEIPTDATE, 141 | L_SHIPINSTRUCT, 142 | ] 143 | type_conversion: 144 | activate: false 145 | p9: 146 | portion: 10 147 | column_removing: 148 | activate: true 149 | columns: [ 150 | L_QUANTITY, 151 | L_EXTENDEDPRICE, 152 | L_DISCOUNT, 153 | L_TAX, 154 | L_RETURNFLAG, 155 | L_LINESTATUS, 156 | L_SHIPDATE, 157 | L_COMMITDATE, 158 | L_SHIPINSTRUCT, 159 | ] 160 | type_conversion: 161 | activate: false 162 | p10: 163 | portion: 10 164 | column_removing: 165 | activate: true 166 | columns: [ 167 | L_QUANTITY, 168 | L_EXTENDEDPRICE, 169 | L_DISCOUNT, 170 | L_TAX, 171 | L_RETURNFLAG, 172 | L_LINESTATUS, 173 | L_SHIPDATE, 174 | L_COMMITDATE, 175 | L_RECEIPTDATE, 176 | ] 177 | type_conversion: 178 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_6.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_6 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10] 9 | p1: 10 | portion: 10 11 | column_removing: 12 | activate: true 13 | columns: [ 14 | L_QUANTITY, 15 | L_EXTENDEDPRICE, 16 | L_DISCOUNT, 17 | L_TAX, 18 | L_RETURNFLAG, 19 | L_LINESTATUS, 20 | L_SHIPDATE, 21 | L_COMMITDATE, 22 | L_RECEIPTDATE, 23 | L_SHIPINSTRUCT, 24 | L_SHIPMODE, 25 | L_COMMENT 26 | ] 27 | type_conversion: 28 | activate: false 29 | p2: 30 | portion: 10 31 | column_removing: 32 | activate: true 33 | columns: [ 34 | L_PARTKEY, 35 | L_EXTENDEDPRICE, 36 | L_DISCOUNT, 37 | L_TAX, 38 | L_RETURNFLAG, 39 | L_LINESTATUS, 40 | L_SHIPDATE, 41 | L_COMMITDATE, 42 | L_RECEIPTDATE, 43 | L_SHIPINSTRUCT, 44 | L_SHIPMODE, 45 | L_COMMENT 46 | ] 47 | type_conversion: 48 | activate: false 49 | p3: 50 | portion: 10 51 | column_removing: 52 | activate: true 53 | columns: [ 54 | L_PARTKEY, 55 | L_QUANTITY, 56 | L_DISCOUNT, 57 | L_TAX, 58 | L_RETURNFLAG, 59 | L_LINESTATUS, 60 | L_SHIPDATE, 61 | L_COMMITDATE, 62 | L_RECEIPTDATE, 63 | L_SHIPINSTRUCT, 64 | L_SHIPMODE, 65 | L_COMMENT 66 | ] 67 | type_conversion: 68 | activate: false 69 | p4: 70 | portion: 10 71 | column_removing: 72 | activate: true 73 | columns: [ 74 | L_PARTKEY, 75 | L_QUANTITY, 76 | L_EXTENDEDPRICE, 77 | L_TAX, 78 | L_RETURNFLAG, 79 | L_LINESTATUS, 80 | L_SHIPDATE, 81 | L_COMMITDATE, 82 | L_RECEIPTDATE, 83 | L_SHIPINSTRUCT, 84 | L_SHIPMODE, 85 | L_COMMENT 86 | ] 87 | type_conversion: 88 | activate: false 89 | p5: 90 | portion: 10 91 | column_removing: 92 | activate: true 93 | columns: [ 94 | L_PARTKEY, 95 | L_QUANTITY, 96 | L_EXTENDEDPRICE, 97 | L_DISCOUNT, 98 | L_RETURNFLAG, 99 | L_LINESTATUS, 100 | L_SHIPDATE, 101 | L_COMMITDATE, 102 | L_RECEIPTDATE, 103 | L_SHIPINSTRUCT, 104 | L_SHIPMODE, 105 | L_COMMENT 106 | ] 107 | type_conversion: 108 | activate: false 109 | p6: 110 | portion: 10 111 | column_removing: 112 | activate: true 113 | columns: [ 114 | L_PARTKEY, 115 | L_QUANTITY, 116 | L_EXTENDEDPRICE, 117 | L_DISCOUNT, 118 | L_TAX, 119 | L_LINESTATUS, 120 | L_SHIPDATE, 121 | L_COMMITDATE, 122 | L_RECEIPTDATE, 123 | L_SHIPINSTRUCT, 124 | L_SHIPMODE, 125 | L_COMMENT 126 | ] 127 | type_conversion: 128 | activate: false 129 | p7: 130 | portion: 10 131 | column_removing: 132 | activate: true 133 | columns: [ 134 | L_PARTKEY, 135 | L_QUANTITY, 136 | L_EXTENDEDPRICE, 137 | L_DISCOUNT, 138 | L_TAX, 139 | L_RETURNFLAG, 140 | L_SHIPDATE, 141 | L_COMMITDATE, 142 | L_RECEIPTDATE, 143 | L_SHIPINSTRUCT, 144 | L_SHIPMODE, 145 | L_COMMENT 146 | ] 147 | type_conversion: 148 | activate: false 149 | p8: 150 | portion: 10 151 | column_removing: 152 | activate: true 153 | columns: [ 154 | L_PARTKEY, 155 | L_QUANTITY, 156 | L_EXTENDEDPRICE, 157 | L_DISCOUNT, 158 | L_TAX, 159 | L_RETURNFLAG, 160 | L_LINESTATUS, 161 | L_COMMITDATE, 162 | L_RECEIPTDATE, 163 | L_SHIPINSTRUCT, 164 | L_SHIPMODE, 165 | L_COMMENT 166 | ] 167 | type_conversion: 168 | activate: false 169 | p9: 170 | portion: 10 171 | column_removing: 172 | activate: true 173 | columns: [ 174 | L_PARTKEY, 175 | L_QUANTITY, 176 | L_EXTENDEDPRICE, 177 | L_DISCOUNT, 178 | L_TAX, 179 | L_RETURNFLAG, 180 | L_LINESTATUS, 181 | L_SHIPDATE, 182 | L_RECEIPTDATE, 183 | L_SHIPINSTRUCT, 184 | L_SHIPMODE, 185 | L_COMMENT 186 | ] 187 | type_conversion: 188 | activate: false 189 | p10: 190 | portion: 10 191 | column_removing: 192 | activate: true 193 | columns: [ 194 | L_PARTKEY, 195 | L_QUANTITY, 196 | L_EXTENDEDPRICE, 197 | L_DISCOUNT, 198 | L_TAX, 199 | L_RETURNFLAG, 200 | L_LINESTATUS, 201 | L_SHIPDATE, 202 | L_COMMITDATE, 203 | L_SHIPINSTRUCT, 204 | L_SHIPMODE, 205 | L_COMMENT 206 | ] 207 | type_conversion: 208 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/lineitem_7.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_7 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11] 9 | p1: 10 | portion: 10 11 | column_removing: 12 | activate: true 13 | columns: [ 14 | L_QUANTITY, 15 | L_EXTENDEDPRICE, 16 | L_DISCOUNT, 17 | L_TAX, 18 | L_RETURNFLAG, 19 | L_LINESTATUS, 20 | L_SHIPDATE, 21 | L_COMMITDATE, 22 | L_RECEIPTDATE, 23 | L_SHIPINSTRUCT, 24 | L_SHIPMODE, 25 | L_COMMENT 26 | ] 27 | type_conversion: 28 | activate: false 29 | p2: 30 | portion: 10 31 | column_removing: 32 | activate: true 33 | columns: [ 34 | L_PARTKEY, 35 | L_EXTENDEDPRICE, 36 | L_DISCOUNT, 37 | L_TAX, 38 | L_RETURNFLAG, 39 | L_LINESTATUS, 40 | L_SHIPDATE, 41 | L_COMMITDATE, 42 | L_RECEIPTDATE, 43 | L_SHIPINSTRUCT, 44 | L_SHIPMODE, 45 | L_COMMENT 46 | ] 47 | type_conversion: 48 | activate: false 49 | p3: 50 | portion: 10 51 | column_removing: 52 | activate: true 53 | columns: [ 54 | L_PARTKEY, 55 | L_QUANTITY, 56 | L_DISCOUNT, 57 | L_TAX, 58 | L_RETURNFLAG, 59 | L_LINESTATUS, 60 | L_SHIPDATE, 61 | L_COMMITDATE, 62 | L_RECEIPTDATE, 63 | L_SHIPINSTRUCT, 64 | L_SHIPMODE, 65 | L_COMMENT 66 | ] 67 | type_conversion: 68 | activate: false 69 | p4: 70 | portion: 10 71 | column_removing: 72 | activate: true 73 | columns: [ 74 | L_PARTKEY, 75 | L_QUANTITY, 76 | L_EXTENDEDPRICE, 77 | L_TAX, 78 | L_RETURNFLAG, 79 | L_LINESTATUS, 80 | L_SHIPDATE, 81 | L_COMMITDATE, 82 | L_RECEIPTDATE, 83 | L_SHIPINSTRUCT, 84 | L_SHIPMODE, 85 | L_COMMENT 86 | ] 87 | type_conversion: 88 | activate: false 89 | p5: 90 | portion: 10 91 | column_removing: 92 | activate: true 93 | columns: [ 94 | L_PARTKEY, 95 | L_QUANTITY, 96 | L_EXTENDEDPRICE, 97 | L_DISCOUNT, 98 | L_RETURNFLAG, 99 | L_LINESTATUS, 100 | L_SHIPDATE, 101 | L_COMMITDATE, 102 | L_RECEIPTDATE, 103 | L_SHIPINSTRUCT, 104 | L_SHIPMODE, 105 | L_COMMENT 106 | ] 107 | type_conversion: 108 | activate: false 109 | p6: 110 | portion: 10 111 | column_removing: 112 | activate: true 113 | columns: [ 114 | L_PARTKEY, 115 | L_QUANTITY, 116 | L_EXTENDEDPRICE, 117 | L_DISCOUNT, 118 | L_TAX, 119 | L_LINESTATUS, 120 | L_SHIPDATE, 121 | L_COMMITDATE, 122 | L_RECEIPTDATE, 123 | L_SHIPINSTRUCT, 124 | L_SHIPMODE, 125 | L_COMMENT 126 | ] 127 | type_conversion: 128 | activate: false 129 | p7: 130 | portion: 10 131 | column_removing: 132 | activate: true 133 | columns: [ 134 | L_PARTKEY, 135 | L_QUANTITY, 136 | L_EXTENDEDPRICE, 137 | L_DISCOUNT, 138 | L_TAX, 139 | L_RETURNFLAG, 140 | L_SHIPDATE, 141 | L_COMMITDATE, 142 | L_RECEIPTDATE, 143 | L_SHIPINSTRUCT, 144 | L_SHIPMODE, 145 | L_COMMENT 146 | ] 147 | type_conversion: 148 | activate: false 149 | p8: 150 | portion: 10 151 | column_removing: 152 | activate: true 153 | columns: [ 154 | L_PARTKEY, 155 | L_QUANTITY, 156 | L_EXTENDEDPRICE, 157 | L_DISCOUNT, 158 | L_TAX, 159 | L_RETURNFLAG, 160 | L_LINESTATUS, 161 | L_COMMITDATE, 162 | L_RECEIPTDATE, 163 | L_SHIPINSTRUCT, 164 | L_SHIPMODE, 165 | L_COMMENT 166 | ] 167 | type_conversion: 168 | activate: false 169 | p9: 170 | portion: 10 171 | column_removing: 172 | activate: true 173 | columns: [ 174 | L_PARTKEY, 175 | L_QUANTITY, 176 | L_EXTENDEDPRICE, 177 | L_DISCOUNT, 178 | L_TAX, 179 | L_RETURNFLAG, 180 | L_LINESTATUS, 181 | L_SHIPDATE, 182 | L_RECEIPTDATE, 183 | L_SHIPINSTRUCT, 184 | L_SHIPMODE, 185 | L_COMMENT 186 | ] 187 | type_conversion: 188 | activate: false 189 | p10: 190 | portion: 5 191 | column_removing: 192 | activate: true 193 | columns: [ 194 | L_PARTKEY, 195 | L_QUANTITY, 196 | L_EXTENDEDPRICE, 197 | L_DISCOUNT, 198 | L_TAX, 199 | L_RETURNFLAG, 200 | L_LINESTATUS, 201 | L_SHIPDATE, 202 | L_COMMITDATE, 203 | L_SHIPINSTRUCT, 204 | L_SHIPMODE, 205 | L_COMMENT 206 | ] 207 | type_conversion: 208 | activate: false 209 | p11: 210 | portion: 5 211 | column_removing: 212 | activate: true 213 | columns: [ 214 | L_PARTKEY, 215 | L_QUANTITY, 216 | L_EXTENDEDPRICE, 217 | L_DISCOUNT, 218 | L_TAX, 219 | L_RETURNFLAG, 220 | L_LINESTATUS, 221 | L_SHIPDATE, 222 | L_RECEIPTDATE, 223 | L_SHIPINSTRUCT, 224 | L_SHIPMODE, 225 | L_COMMENT 226 | ] 227 | type_conversion: 228 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/sample_lineitem_conf.yaml: -------------------------------------------------------------------------------- 1 | title: test_lineitem 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 50 17 | column_removing: 18 | activate: true 19 | columns: [L_LINESTATUS] 20 | type_conversion: 21 | activate: true 22 | columns: [L_QUANTITY] 23 | types: [VARCHAR(20)] -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/sample_supplier_conf.yaml: -------------------------------------------------------------------------------- 1 | title: test_supplier 2 | benchmark: tpch 3 | table_name: supplier 4 | schema_file: ./schemas/tpch/supplier.yaml 5 | data_file_path: ./data/tpch/supplier.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 50 17 | column_removing: 18 | activate: true 19 | columns: [S_COMMENT, S_PHONE] 20 | type_conversion: 21 | activate: true 22 | columns: [S_ACCTBAL] 23 | types: [VARCHAR(20)] -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_10.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_10 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: true 13 | columns: [L_SHIPMODE] 14 | type_conversion: 15 | activate: false 16 | p2: 17 | portion: 20 18 | column_removing: 19 | activate: true 20 | columns: [L_COMMITDATE, L_LINESTATUS] 21 | type_conversion: 22 | activate: false 23 | p3: 24 | portion: 20 25 | column_removing: 26 | activate: true 27 | columns: [L_COMMENT] 28 | type_conversion: 29 | activate: false 30 | p4: 31 | portion: 20 32 | column_removing: 33 | activate: true 34 | columns: [L_TAX, L_RETURNFLAG] 35 | type_conversion: 36 | activate: false 37 | p5: 38 | portion: 20 39 | column_removing: 40 | activate: true 41 | columns: [L_SHIPDATE] 42 | type_conversion: 43 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_11.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_11 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 20 17 | column_removing: 18 | activate: true 19 | columns: [L_COMMITDATE] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 20 24 | column_removing: 25 | activate: true 26 | columns: [L_COMMENT] 27 | type_conversion: 28 | activate: false 29 | p4: 30 | portion: 20 31 | column_removing: 32 | activate: true 33 | columns: [L_TAX] 34 | type_conversion: 35 | activate: false 36 | p5: 37 | portion: 10 38 | column_removing: 39 | activate: true 40 | columns: [L_SHIPDATE] 41 | type_conversion: 42 | activate: false 43 | p6: 44 | portion: 10 45 | column_removing: 46 | activate: true 47 | columns: [L_SHIPDATE, L_QUANTITY] 48 | type_conversion: 49 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_12.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_12 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: true 13 | columns: [L_SHIPMODE] 14 | type_conversion: 15 | activate: false 16 | p2: 17 | portion: 20 18 | column_removing: 19 | activate: true 20 | columns: [L_COMMITDATE] 21 | type_conversion: 22 | activate: false 23 | p3: 24 | portion: 20 25 | column_removing: 26 | activate: true 27 | columns: [L_COMMITDATE, L_COMMENT] 28 | type_conversion: 29 | activate: false 30 | p4: 31 | portion: 20 32 | column_removing: 33 | activate: true 34 | columns: [L_TAX] 35 | type_conversion: 36 | activate: false 37 | p5: 38 | portion: 10 39 | column_removing: 40 | activate: true 41 | columns: [L_SHIPDATE] 42 | type_conversion: 43 | activate: false 44 | p6: 45 | portion: 10 46 | column_removing: 47 | activate: true 48 | columns: [L_SHIPDATE, L_QUANTITY] 49 | type_conversion: 50 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_13.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_13 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: true 13 | columns: [L_SHIPMODE] 14 | type_conversion: 15 | activate: false 16 | p2: 17 | portion: 20 18 | column_removing: 19 | activate: true 20 | columns: [L_SHIPMODE, L_COMMITDATE] 21 | type_conversion: 22 | activate: false 23 | p3: 24 | portion: 20 25 | column_removing: 26 | activate: true 27 | columns: [L_SHIPMODE, L_COMMITDATE, L_COMMENT] 28 | type_conversion: 29 | activate: false 30 | p4: 31 | portion: 20 32 | column_removing: 33 | activate: true 34 | columns: [L_TAX] 35 | type_conversion: 36 | activate: false 37 | p5: 38 | portion: 10 39 | column_removing: 40 | activate: true 41 | columns: [L_TAX, L_SHIPDATE] 42 | type_conversion: 43 | activate: false 44 | p6: 45 | portion: 10 46 | column_removing: 47 | activate: true 48 | columns: [L_SHIPDATE, L_DISCOUNT] 49 | type_conversion: 50 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_14.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_14 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5, p6, p7] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 20 17 | column_removing: 18 | activate: true 19 | columns: [L_COMMITDATE] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 20 24 | column_removing: 25 | activate: true 26 | columns: [L_COMMENT] 27 | type_conversion: 28 | activate: false 29 | p4: 30 | portion: 10 31 | column_removing: 32 | activate: true 33 | columns: [L_TAX] 34 | type_conversion: 35 | activate: false 36 | p5: 37 | portion: 10 38 | column_removing: 39 | activate: true 40 | columns: [L_SHIPDATE] 41 | type_conversion: 42 | activate: false 43 | p6: 44 | portion: 10 45 | column_removing: 46 | activate: true 47 | columns: [L_SHIPDATE, L_QUANTITY] 48 | type_conversion: 49 | activate: false 50 | p7: 51 | portion: 10 52 | column_removing: 53 | activate: true 54 | columns: [L_TAX, L_RECEIPTDATE] 55 | type_conversion: 56 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_5.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_5 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 50 17 | column_removing: 18 | activate: true 19 | columns: [L_TAX] 20 | type_conversion: 21 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_6.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_6 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4] 9 | p1: 10 | portion: 25 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 25 17 | column_removing: 18 | activate: true 19 | columns: [L_SHIPMODE, L_COMMENT] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 25 24 | column_removing: 25 | activate: true 26 | columns: [L_SHIPMODE] 27 | type_conversion: 28 | activate: true 29 | columns: [L_EXTENDEDPRICE] 30 | types: [CHAR(20)] 31 | p4: 32 | portion: 25 33 | column_removing: 34 | activate: true 35 | columns: [L_COMMENT] 36 | type_conversion: 37 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_7.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_7 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4] 9 | p1: 10 | portion: 50 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 26 17 | column_removing: 18 | activate: true 19 | columns: [L_COMMITDATE] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 12 24 | column_removing: 25 | activate: true 26 | columns: [L_COMMENT] 27 | type_conversion: 28 | activate: false 29 | p4: 30 | portion: 12 31 | column_removing: 32 | activate: true 33 | columns: [L_TAX] 34 | type_conversion: 35 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_8.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_8 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 20 17 | column_removing: 18 | activate: true 19 | columns: [L_COMMITDATE] 20 | type_conversion: 21 | activate: false 22 | p3: 23 | portion: 20 24 | column_removing: 25 | activate: true 26 | columns: [L_COMMENT] 27 | type_conversion: 28 | activate: false 29 | p4: 30 | portion: 20 31 | column_removing: 32 | activate: true 33 | columns: [L_TAX] 34 | type_conversion: 35 | activate: false 36 | p5: 37 | portion: 20 38 | column_removing: 39 | activate: true 40 | columns: [L_SHIPDATE] 41 | type_conversion: 42 | activate: false -------------------------------------------------------------------------------- /tpch-converter/configurations/tpch/schemaless/lineitem_9.yaml: -------------------------------------------------------------------------------- 1 | title: lineitem_9 2 | benchmark: tpch 3 | table_name: lineitem 4 | schema_file: ./schemas/tpch/lineitem.yaml 5 | data_file_path: ./data/tpch/lineitem.csv 6 | queries_path: ./queries/tpch 7 | output_folders_path: ./results 8 | partitions: [p1, p2, p3, p4, p5] 9 | p1: 10 | portion: 20 11 | column_removing: 12 | activate: false 13 | type_conversion: 14 | activate: false 15 | p2: 16 | portion: 20 17 | column_removing: 18 | activate: true 19 | columns: [L_COMMITDATE] 20 | type_conversion: 21 | activate: true 22 | columns: [L_EXTENDEDPRICE] 23 | types: [CHAR(20)] 24 | p3: 25 | portion: 20 26 | column_removing: 27 | activate: true 28 | columns: [L_COMMENT] 29 | type_conversion: 30 | activate: false 31 | p4: 32 | portion: 20 33 | column_removing: 34 | activate: true 35 | columns: [L_TAX] 36 | type_conversion: 37 | activate: true 38 | columns: [L_RECEIPTDATE] 39 | types: [CHAR(50)] 40 | p5: 41 | portion: 20 42 | column_removing: 43 | activate: true 44 | columns: [L_SHIPDATE] 45 | type_conversion: 46 | activate: false -------------------------------------------------------------------------------- /tpch-converter/execution_time_calculator.py: -------------------------------------------------------------------------------- 1 | 2 | num_queries = 22 3 | execution_time_array = [0] * num_queries 4 | 5 | def parseAndGetExecutionTimeInSecond(result_string): 6 | parsed_string = result_string.split('=') 7 | return float(parsed_string[1]) 8 | 9 | with open('/home/TPCH-Greenplum/results1/results.log', 'r') as results1: 10 | with open('/home/TPCH-Greenplum/results2/results.log', 'r') as results2: 11 | with open('/home/TPCH-Greenplum/results3/results.log', 'r') as results3: 12 | for i in range(0, num_queries): 13 | result1_result = results1.readline() 14 | result2_result = results2.readline() 15 | result3_result = results3.readline() 16 | execution_time_array[i] += parseAndGetExecutionTimeInSecond(result1_result) 17 | execution_time_array[i] += parseAndGetExecutionTimeInSecond(result2_result) 18 | execution_time_array[i] += parseAndGetExecutionTimeInSecond(result3_result) 19 | execution_time_array[i] = round(execution_time_array[i] / 3, 2) 20 | 21 | with open('./execution_average_result.txt', 'w') as result: 22 | execution_time_array = [str(time) for time in execution_time_array] 23 | result.write(','.join(execution_time_array)) -------------------------------------------------------------------------------- /tpch-converter/queryModifier.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from pathlib import Path 5 | import utils 6 | 7 | def modifyExportAllQueries(configuration_yaml, schema_yaml, queries_path, output_queries_path): 8 | # Get queries in the path and iterate over the queries 9 | for filename in os.listdir(queries_path): 10 | filepath = os.path.join(queries_path, filename) 11 | if os.path.isfile(filepath): 12 | # Call modify 13 | with open(filepath, 'r') as query_file: 14 | query = query_file.read() 15 | modified_query = modifyQuery(query, configuration_yaml, schema_yaml) 16 | # Export query 17 | exportQuery(output_queries_path, Path(filepath).stem, modified_query) 18 | return 19 | 20 | def modifyQuery(query, configuration_yaml, schema_yaml): 21 | # Get the table name 22 | target_table_name = configuration_yaml['table_name'] 23 | 24 | # Check if the query contains the table name. If not, return the original query. 25 | if target_table_name not in query: return query 26 | 27 | # Geneate a union query for the table 28 | # Make empty string 29 | union_query = '' 30 | # Append '(' 31 | union_query += '(' 32 | # Iterate over the partitions 33 | partitions = configuration_yaml['partitions'] 34 | for i, partition in enumerate(partitions): 35 | # Get schema columns 36 | schema_columns = schema_yaml['columns'] 37 | # Get columns not removed 38 | columns_wo_removing = utils.getSchemaColumnsWithoutRemovedColumns(schema_columns, configuration_yaml[partition]) 39 | # Get type converted columns 40 | columns_type_converted = utils.getTypeConversionColumns(configuration_yaml[partition]) 41 | # Append '(SELECT' 42 | union_query += '(SELECT ' 43 | # Iterate over the schema column 44 | for j, schema_column in enumerate(schema_columns): 45 | # Check if the column is removed. 46 | if schema_column not in columns_wo_removing: 47 | # If so, append ‘NULL as ’ 48 | union_query += ('NULL as ' + schema_column) 49 | else: 50 | # Check if the column is type converted 51 | if schema_column in columns_type_converted: 52 | # If so, append ‘cast( as ) 53 | column_type = schema_yaml[schema_column] 54 | union_query += ('cast(' + schema_column + ' as ' + column_type + ')') 55 | else: 56 | # Else, append ‘’ 57 | union_query += schema_column 58 | # Append comma is not the last column 59 | if j < len(schema_columns) - 1: 60 | union_query += ', ' 61 | # Append ‘FROM ’ 62 | partition_table_name = utils.getPartitionFileBaseName(configuration_yaml, partition) 63 | union_query += (' FROM ' + partition_table_name) 64 | # If not the last partition, append ‘) \n UNION \n’ 65 | if i < len(partitions) - 1: 66 | union_query += ') UNION ALL ' 67 | # Else, append ‘) \ 68 | else: 69 | union_query += ')' 70 | # Append 'as
' 71 | union_query += (') as ' + target_table_name) 72 | # Replace
in the original query to the generate string 73 | new_query = query.replace(target_table_name, union_query) 74 | # Return the modified query 75 | return new_query 76 | 77 | def exportQuery(output_queries_path, query_file_name, query): 78 | query_file_path = os.path.join(output_queries_path, query_file_name + '.sql') 79 | with open(query_file_path, 'w') as query_file: 80 | query_file.write(query) -------------------------------------------------------------------------------- /tpch-converter/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DBNAME=$1 4 | DATAPATH=$2 5 | MODE=$3 6 | 7 | # Move original datas 8 | echo 'Moving original data to temp folder' 9 | mv /home/TPCH-Greenplum/dss /home/TPCH-Greenplum/temp 10 | 11 | # Make dss folder 12 | echo 'Making dss folder' 13 | mkdir /home/TPCH-Greenplum/dss 14 | mkdir /home/TPCH-Greenplum/dss/data 15 | mkdir /home/TPCH-Greenplum/dss/queries 16 | 17 | # Move generated data 18 | echo 'Moving generated data to dss folder' 19 | cp ${DATAPATH}/*.csv /home/TPCH-Greenplum/dss/data 20 | cp ./data/tpch/*.csv /home/TPCH-Greenplum/dss/data 21 | cp ${DATAPATH}/*.sql /home/TPCH-Greenplum/dss 22 | cp ${DATAPATH}/queries/*.sql /home/TPCH-Greenplum/dss/queries 23 | 24 | # Move to the folder 25 | echo 'Changing directory to TPCH-Greenplum' 26 | cd /home/TPCH-Greenplum 27 | 28 | # Run test 29 | echo 'Running test' 30 | if [ $3 == 'r' ] 31 | then 32 | rm -rf results1 33 | rm -rf results2 34 | rm -rf results3 35 | bash benchmark_test.sh results1 127.0.0.1 $DBNAME gpadmin gpadmin 36 | bash tpch_test_only.sh results2 127.0.0.1 $DBNAME gpadmin gpadmin 37 | bash tpch_test_only.sh results3 127.0.0.1 $DBNAME gpadmin gpadmin 38 | else 39 | rm -rf results 40 | bash benchmark_test.sh results 127.0.0.1 $DBNAME gpadmin gpadmin 41 | fi 42 | 43 | # Restore the data 44 | echo 'Remove dss folder and restore the temp folder' 45 | rm -rf dss 46 | mv temp dss -------------------------------------------------------------------------------- /tpch-converter/runner_test_only.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DBNAME=$1 4 | DATAPATH=$2 5 | 6 | # Move original datas 7 | echo 'Moving original data to temp folder' 8 | mv /home/TPCH-Greenplum/dss /home/TPCH-Greenplum/temp 9 | 10 | # Make dss folder 11 | echo 'Making dss folder' 12 | mkdir /home/TPCH-Greenplum/dss 13 | mkdir /home/TPCH-Greenplum/dss/data 14 | mkdir /home/TPCH-Greenplum/dss/queries 15 | 16 | # Move generated data 17 | echo 'Moving generated data to dss folder' 18 | cp ${DATAPATH}/*.csv /home/TPCH-Greenplum/dss/data 19 | cp ./data/tpch/*.csv /home/TPCH-Greenplum/dss/data 20 | cp ${DATAPATH}/*.sql /home/TPCH-Greenplum/dss 21 | cp ${DATAPATH}/queries/*.sql /home/TPCH-Greenplum/dss/queries 22 | 23 | # Move to the folder 24 | echo 'Changing directory to TPCH-Greenplum' 25 | cd /home/TPCH-Greenplum 26 | 27 | # Run test 28 | echo 'Running test' 29 | rm -rf results 30 | bash tpch_test_only.sh results 127.0.0.1 $DBNAME gpadmin gpadmin 31 | 32 | # Restore the data 33 | echo 'Remove dss folder and restore the temp folder' 34 | rm -rf dss 35 | mv temp dss -------------------------------------------------------------------------------- /tpch-converter/schemas/tpch/lineitem.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | L_ORDERKEY, 3 | L_PARTKEY, 4 | L_SUPPKEY, 5 | L_LINENUMBER, 6 | L_QUANTITY, 7 | L_EXTENDEDPRICE, 8 | L_DISCOUNT, 9 | L_TAX, 10 | L_RETURNFLAG, 11 | L_LINESTATUS, 12 | L_SHIPDATE, 13 | L_COMMITDATE, 14 | L_RECEIPTDATE, 15 | L_SHIPINSTRUCT, 16 | L_SHIPMODE, 17 | L_COMMENT 18 | ] 19 | L_ORDERKEY: INTEGER NOT NULL 20 | L_PARTKEY: INTEGER NOT NULL 21 | L_SUPPKEY: INTEGER NOT NULL 22 | L_LINENUMBER: INTEGER 23 | L_QUANTITY: DECIMAL 24 | L_EXTENDEDPRICE: DECIMAL 25 | L_DISCOUNT: DECIMAL 26 | L_TAX: DECIMAL 27 | L_RETURNFLAG: CHAR(1) 28 | L_LINESTATUS: CHAR(1) 29 | L_SHIPDATE: DATE 30 | L_COMMITDATE: DATE 31 | L_RECEIPTDATE: DATE 32 | L_SHIPINSTRUCT: CHAR(25) 33 | L_SHIPMODE: CHAR(10) 34 | L_COMMENT: VARCHAR(44) 35 | primary_keys: [L_ORDERKEY, L_LINENUMBER] 36 | create_option: with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (l_orderkey, l_linenumber) 37 | -------------------------------------------------------------------------------- /tpch-converter/schemas/tpch/supplier.yaml: -------------------------------------------------------------------------------- 1 | columns: [ 2 | S_SUPPKEY, 3 | S_NAME, 4 | S_ADDRESS, 5 | S_NATIONKEY, 6 | S_PHONE, 7 | S_ACCTBAL, 8 | S_COMMENT 9 | ] 10 | S_SUPPKEY: SERIAL8 11 | S_NAME: CHAR(25) 12 | S_ADDRESS: VARCHAR(40) 13 | S_NATIONKEY: BIGINT NOT NULL 14 | S_PHONE: CHAR(15) 15 | S_ACCTBAL: DECIMAL 16 | S_COMMENT: VARCHAR(101) 17 | primary_keys: [S_SUPPKEY] 18 | create_option: with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (s_suppkey) 19 | -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/customer.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE CUSTOMER ADD FOREIGN KEY (C_NATIONKEY) REFERENCES NATION(N_NATIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/lineitem.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE LINEITEM ADD FOREIGN KEY (L_ORDERKEY) REFERENCES ORDERS(O_ORDERKEY); 2 | ALTER TABLE LINEITEM ADD FOREIGN KEY (L_PARTKEY,L_SUPPKEY) REFERENCES PARTSUPP(PS_PARTKEY,PS_SUPPKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/nation.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE NATION ADD FOREIGN KEY (N_REGIONKEY) REFERENCES REGION(R_REGIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/orders.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE ORDERS ADD FOREIGN KEY (O_CUSTKEY) REFERENCES CUSTOMER(C_CUSTKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/part.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/tpch-converter/templates/tpch/alter/part.sql -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/partsupp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE PARTSUPP ADD FOREIGN KEY (PS_PARTKEY) REFERENCES PART(P_PARTKEY); 2 | ALTER TABLE PARTSUPP ADD FOREIGN KEY (PS_SUPPKEY) REFERENCES SUPPLIER(S_SUPPKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/region.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/tpch-converter/templates/tpch/alter/region.sql -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/alter/supplier.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE SUPPLIER ADD FOREIGN KEY (S_NATIONKEY) REFERENCES NATION(N_NATIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/customer.sql: -------------------------------------------------------------------------------- 1 | analyze CUSTOMER; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/lineitem.sql: -------------------------------------------------------------------------------- 1 | analyze LINEITEM; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/nation.sql: -------------------------------------------------------------------------------- 1 | analyze NATION; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/orders.sql: -------------------------------------------------------------------------------- 1 | analyze ORDERS; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/part.sql: -------------------------------------------------------------------------------- 1 | analyze PART; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/partsupp.sql: -------------------------------------------------------------------------------- 1 | analyze PARTSUPP; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/region.sql: -------------------------------------------------------------------------------- 1 | analyze REGION; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/analyze/supplier.sql: -------------------------------------------------------------------------------- 1 | analyze SUPPLIER; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/customer.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_CUSTOMER_NATIONKEY ON CUSTOMER (C_NATIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/lineitem.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_LINEITEM_ORDERKEY ON LINEITEM (L_ORDERKEY); 2 | CREATE INDEX IDX_LINEITEM_PART_SUPP ON LINEITEM (L_PARTKEY,L_SUPPKEY); 3 | CREATE INDEX IDX_LINEITEM_SHIPDATE ON LINEITEM (L_SHIPDATE, L_DISCOUNT, L_QUANTITY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/nation.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_NATION_REGIONKEY ON NATION (N_REGIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/orders.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_ORDERS_CUSTKEY ON ORDERS (O_CUSTKEY); 2 | CREATE INDEX IDX_ORDERS_ORDERDATE ON ORDERS (O_ORDERDATE); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/part.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/tpch-converter/templates/tpch/index/part.sql -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/partsupp.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_PARTSUPP_PARTKEY ON PARTSUPP (PS_PARTKEY); 2 | CREATE INDEX IDX_PARTSUPP_SUPPKEY ON PARTSUPP (PS_SUPPKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/region.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/postech-dblab-iitp/schemaless-data-converter/da267cc61fb96e2b21de21678771a456b70da729/tpch-converter/templates/tpch/index/region.sql -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/index/supplier.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX IDX_SUPPLIER_NATION_KEY ON SUPPLIER (S_NATIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/customer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE CUSTOMER ( 2 | C_CUSTKEY SERIAL8, 3 | C_NAME VARCHAR(25), 4 | C_ADDRESS VARCHAR(40), 5 | C_NATIONKEY BIGINT NOT NULL, -- references N_NATIONKEY 6 | C_PHONE CHAR(15), 7 | C_ACCTBAL DECIMAL, 8 | C_MKTSEGMENT CHAR(10), 9 | C_COMMENT VARCHAR(117) 10 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (c_custkey); 11 | 12 | \COPY customer FROM '/tmp/dss-data/customer.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/lineitem.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE LINEITEM ( 2 | L_ORDERKEY BIGINT NOT NULL, -- references O_ORDERKEY 3 | L_PARTKEY BIGINT NOT NULL, -- references P_PARTKEY (compound fk to PARTSUPP) 4 | L_SUPPKEY BIGINT NOT NULL, -- references S_SUPPKEY (compound fk to PARTSUPP) 5 | L_LINENUMBER INTEGER, 6 | L_QUANTITY DECIMAL, 7 | L_EXTENDEDPRICE DECIMAL, 8 | L_DISCOUNT DECIMAL, 9 | L_TAX DECIMAL, 10 | L_RETURNFLAG CHAR(1), 11 | L_LINESTATUS CHAR(1), 12 | L_SHIPDATE DATE, 13 | L_COMMITDATE DATE, 14 | L_RECEIPTDATE DATE, 15 | L_SHIPINSTRUCT CHAR(25), 16 | L_SHIPMODE CHAR(10), 17 | L_COMMENT VARCHAR(44) 18 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (l_orderkey, l_linenumber); 19 | 20 | \COPY lineitem FROM '/tmp/dss-data/lineitem.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/nation.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE NATION ( 2 | N_NATIONKEY SERIAL8, 3 | N_NAME CHAR(25), 4 | N_REGIONKEY BIGINT NOT NULL, -- references R_REGIONKEY 5 | N_COMMENT VARCHAR(152) 6 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (n_nationkey); 7 | 8 | \COPY nation FROM '/tmp/dss-data/nation.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/orders.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE ORDERS ( 2 | O_ORDERKEY SERIAL8, 3 | O_CUSTKEY BIGINT NOT NULL, -- references C_CUSTKEY 4 | O_ORDERSTATUS CHAR(1), 5 | O_TOTALPRICE DECIMAL, 6 | O_ORDERDATE DATE, 7 | O_ORDERPRIORITY CHAR(15), 8 | O_CLERK CHAR(15), 9 | O_SHIPPRIORITY INTEGER, 10 | O_COMMENT VARCHAR(79) 11 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (o_orderkey); 12 | 13 | \COPY orders FROM '/tmp/dss-data/orders.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/part.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE PART ( 2 | P_PARTKEY SERIAL8, 3 | P_NAME VARCHAR(55), 4 | P_MFGR CHAR(25), 5 | P_BRAND CHAR(10), 6 | P_TYPE VARCHAR(25), 7 | P_SIZE INTEGER, 8 | P_CONTAINER CHAR(10), 9 | P_RETAILPRICE DECIMAL, 10 | P_COMMENT VARCHAR(23) 11 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (p_partkey); 12 | 13 | \COPY part FROM '/tmp/dss-data/part.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/partsupp.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE PARTSUPP ( 2 | PS_PARTKEY BIGINT NOT NULL, -- references P_PARTKEY 3 | PS_SUPPKEY BIGINT NOT NULL, -- references S_SUPPKEY 4 | PS_AVAILQTY INTEGER, 5 | PS_SUPPLYCOST DECIMAL, 6 | PS_COMMENT VARCHAR(199) 7 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (ps_partkey,ps_suppkey); 8 | 9 | \COPY partsupp FROM '/tmp/dss-data/partsupp.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/region.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE REGION ( 2 | R_REGIONKEY SERIAL8, 3 | R_NAME CHAR(25), 4 | R_COMMENT VARCHAR(152) 5 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (r_regionkey); 6 | 7 | \COPY region FROM '/tmp/dss-data/region.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/load/supplier.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE SUPPLIER ( 2 | S_SUPPKEY SERIAL8, 3 | S_NAME CHAR(25), 4 | S_ADDRESS VARCHAR(40), 5 | S_NATIONKEY BIGINT NOT NULL, -- references N_NATIONKEY 6 | S_PHONE CHAR(15), 7 | S_ACCTBAL DECIMAL, 8 | S_COMMENT VARCHAR(101) 9 | ) with (APPENDONLY=true,BLOCKSIZE=2097152,ORIENTATION=COLUMN,CHECKSUM=true,OIDS=false) DISTRIBUTED BY (s_suppkey); 10 | 11 | \COPY supplier FROM '/tmp/dss-data/supplier.csv' WITH csv DELIMITER '|'; -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/customer.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE CUSTOMER ADD PRIMARY KEY (C_CUSTKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/lineitem.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE LINEITEM ADD PRIMARY KEY (L_ORDERKEY, L_LINENUMBER); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/nation.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE NATION ADD PRIMARY KEY (N_NATIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/orders.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE ORDERS ADD PRIMARY KEY (O_ORDERKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/part.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE PART ADD PRIMARY KEY (P_PARTKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/partsupp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE PARTSUPP ADD PRIMARY KEY (PS_PARTKEY, PS_SUPPKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/region.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE REGION ADD PRIMARY KEY (R_REGIONKEY); -------------------------------------------------------------------------------- /tpch-converter/templates/tpch/pkeys/supplier.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE SUPPLIER ADD PRIMARY KEY (S_SUPPKEY); -------------------------------------------------------------------------------- /tpch-converter/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python generator.py configurations/sample_tpch_conf.yaml -------------------------------------------------------------------------------- /tpch-converter/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | def getOutputFolderPath(configuration_yaml): 5 | title = configuration_yaml['title'] 6 | output_folders_path = configuration_yaml['output_folders_path'] 7 | output_folder_path = os.path.join(output_folders_path, title) 8 | return output_folder_path 9 | 10 | def getPartitionOutputFilePath(configuration_yaml, partition): 11 | file_name = getPartitionFileBaseName(configuration_yaml, partition) + '.csv' 12 | output_folder_path = getOutputFolderPath(configuration_yaml) 13 | output_file_path = os.path.join(output_folder_path, file_name) 14 | return output_file_path 15 | 16 | def getPartitionFileBaseName(configuration_yaml, partition): 17 | data_file_path = configuration_yaml['data_file_path'] 18 | base_file_name = Path(data_file_path).stem 19 | return base_file_name + '_' + partition 20 | 21 | def getSchemaColumnsWithoutRemovedColumns(schema_columns, partition_configuration): 22 | # If column_removing is active 23 | column_removing_configuration = partition_configuration['column_removing'] 24 | if column_removing_configuration['activate']: 25 | columns_to_remove = column_removing_configuration['columns'] 26 | column_indicies = [schema_columns.index(column_to_remove) for column_to_remove in columns_to_remove] 27 | new_schema_columns = [column for i, column in enumerate(schema_columns) if i not in column_indicies] 28 | return new_schema_columns 29 | else: # If no to-be-removed columns, just return the original one 30 | return schema_columns 31 | 32 | def getRemovedColumns(partition_configuration): 33 | column_removing_configuration = partition_configuration['column_removing'] 34 | if column_removing_configuration['activate']: 35 | return column_removing_configuration['columns'] 36 | else: # If no to-be-removed columns, just return the original one 37 | return [] 38 | 39 | def getTypeConversionColumns(partition_configuration): 40 | type_conversion_configuration = partition_configuration['type_conversion'] 41 | if type_conversion_configuration['activate']: 42 | return type_conversion_configuration['columns'] 43 | else: 44 | return [] 45 | 46 | def getConversionType(partition_configuration, column_name): 47 | type_conversion_configuration = partition_configuration['type_conversion'] 48 | type_conversion_columns = type_conversion_configuration['columns'] 49 | type_columns = type_conversion_configuration['types'] 50 | column_index = type_conversion_columns.index(column_name) 51 | return type_columns[column_index] --------------------------------------------------------------------------------