├── samples └── quickstart │ ├── src │ ├── test │ │ └── resources │ │ │ └── input │ │ │ ├── lax │ │ │ └── 2015-09-01 │ │ │ │ ├── 0000 │ │ │ │ ├── _READY │ │ │ │ ├── file1.txt │ │ │ │ └── file2.txt │ │ │ │ ├── 0100 │ │ │ │ ├── _READY │ │ │ │ ├── file3.txt │ │ │ │ └── file4.txt │ │ │ │ └── 0200 │ │ │ │ ├── _READY │ │ │ │ └── file5.txt │ │ │ └── nyc │ │ │ └── 2015-09-01 │ │ │ ├── 0000 │ │ │ ├── _READY │ │ │ ├── file1.txt │ │ │ └── file2.txt │ │ │ ├── 0100 │ │ │ ├── _READY │ │ │ ├── file3.txt │ │ │ └── file4.txt │ │ │ └── 0200 │ │ │ ├── _READY │ │ │ └── file5.txt │ └── main │ │ ├── oozie │ │ └── workflow.xml │ │ └── celos │ │ ├── defaults.js │ │ ├── example-settings.js │ │ └── workflow.js │ ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties │ ├── ci │ ├── undeploy.sh │ ├── target.json │ └── deploy.sh │ └── build.gradle ├── scripts ├── test │ ├── test_files │ │ ├── empty_workflow_list.json │ │ ├── file-copy │ │ │ ├── src │ │ │ │ ├── test │ │ │ │ │ └── celos-ci │ │ │ │ │ │ ├── test-1 │ │ │ │ │ │ ├── input │ │ │ │ │ │ │ ├── 2013-12-20T1600.txt │ │ │ │ │ │ │ ├── 2013-12-20T1700.txt │ │ │ │ │ │ │ └── 2013-12-20T1800.txt │ │ │ │ │ │ └── output │ │ │ │ │ │ │ ├── 2013-12-20T1600.txt │ │ │ │ │ │ │ ├── 2013-12-20T1700.txt │ │ │ │ │ │ │ └── 2013-12-20T1800.txt │ │ │ │ │ │ └── test.js │ │ │ │ └── main │ │ │ │ │ ├── oozie │ │ │ │ │ └── workflow.xml │ │ │ │ │ └── celos │ │ │ │ │ └── workflow.js │ │ │ ├── scripts │ │ │ │ ├── ci-undeploy │ │ │ │ ├── ci-deploy │ │ │ │ └── ci-test │ │ │ ├── README.md │ │ │ └── celos_deploy │ │ │ │ ├── hdfs │ │ │ │ └── workflow.xml │ │ │ │ └── workflow.js │ │ ├── workflow_list.json │ │ ├── workflow_list_register.json │ │ ├── slot_state_workflow_1_1800.json │ │ ├── slot_state_workflow_1_1900.json │ │ └── trigger-status.json │ ├── check-upload │ │ ├── gradle │ │ │ └── wrapper │ │ │ │ ├── gradle-wrapper.jar │ │ │ │ └── gradle-wrapper.properties │ │ └── build.gradle │ ├── conf │ │ ├── integration-params.json │ │ ├── inventory-ui │ │ └── inventory-server │ ├── target_int_test.json │ ├── ui-test.yaml │ ├── stop-start-test.sh │ └── integration-test.sh ├── templates │ ├── run.tpl │ ├── log-run.tpl │ ├── check.tpl │ ├── service-start.tpl │ └── license.tpl ├── build.sh ├── params │ ├── common-ui.json │ └── common-server.json ├── inventory │ ├── production-ui │ ├── integration-ui │ ├── jdbc-production-ui │ ├── production-server │ ├── integration-server │ └── jdbc-production-server ├── playbooks │ ├── celos-stop.yaml │ ├── celos-check.yaml │ ├── celos-start.yaml │ ├── kinit.yaml │ └── celos-purge.yaml ├── add-license.sh ├── prod │ ├── stop.sh │ ├── start.sh │ └── deploy.sh ├── README.asciidoc ├── celos-purge.yaml ├── runit-restart-service.sh ├── server-and-ui-action.sh └── push-javadoc.sh ├── celos-ci ├── src │ ├── test │ │ ├── resources │ │ │ └── com │ │ │ │ └── collective │ │ │ │ └── celos │ │ │ │ ├── ci │ │ │ │ ├── testing │ │ │ │ │ ├── create │ │ │ │ │ │ └── dir1 │ │ │ │ │ │ │ ├── file1 │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ ├── file2 │ │ │ │ │ │ │ └── file3 │ │ │ │ │ ├── deploy │ │ │ │ │ │ ├── hdfs │ │ │ │ │ │ │ ├── file1 │ │ │ │ │ │ │ └── file2 │ │ │ │ │ │ └── workflow.js │ │ │ │ │ ├── testrun │ │ │ │ │ │ ├── testcase-1 │ │ │ │ │ │ │ └── file │ │ │ │ │ │ └── testcase-2 │ │ │ │ │ │ │ └── file │ │ │ │ │ └── config │ │ │ │ │ │ ├── core-site.xml │ │ │ │ │ │ ├── hdfs-site.xml │ │ │ │ │ │ ├── target-no-deffile.json │ │ │ │ │ │ ├── target-no-wf-uri.json │ │ │ │ │ │ ├── target-no-sec.json │ │ │ │ │ │ ├── target.json │ │ │ │ │ │ └── test.js │ │ │ │ ├── fixtures │ │ │ │ │ ├── jsoncompare │ │ │ │ │ │ └── 1 │ │ │ │ │ │ │ ├── _SUCCESS │ │ │ │ │ │ │ └── avro3.json │ │ │ │ │ ├── input │ │ │ │ │ │ ├── Avro │ │ │ │ │ │ │ ├── dir1 │ │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ │ │ └── file1 │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ │ └── dir3 │ │ │ │ │ │ │ │ └── dir4 │ │ │ │ │ │ │ │ └── file1 │ │ │ │ │ │ ├── PLAIN │ │ │ │ │ │ │ ├── dir1 │ │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ │ │ └── file1 │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ │ └── dir3 │ │ │ │ │ │ │ │ └── dir4 │ │ │ │ │ │ │ │ └── file1 │ │ │ │ │ │ └── plain │ │ │ │ │ │ │ ├── dir1 │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ │ └── file1 │ │ │ │ │ │ │ └── dir2 │ │ │ │ │ │ │ └── dir3 │ │ │ │ │ │ │ └── dir4 │ │ │ │ │ │ │ └── file1 │ │ │ │ │ ├── compare │ │ │ │ │ │ ├── equal1 │ │ │ │ │ │ ├── equal2 │ │ │ │ │ │ └── not-equal │ │ │ │ │ └── avro │ │ │ │ │ │ ├── avro.avro │ │ │ │ │ │ └── avro.schema │ │ │ │ └── client │ │ │ │ │ └── cron-task-test │ │ │ │ │ ├── workflow-1.js │ │ │ │ │ └── workflow-2.js │ │ │ │ └── defaults │ │ │ │ └── test.js │ │ └── java │ │ │ └── com │ │ │ └── collective │ │ │ └── celos │ │ │ └── ci │ │ │ └── testing │ │ │ ├── fixtures │ │ │ ├── create │ │ │ │ └── StringFixObjectCreatorTest.java │ │ │ └── convert │ │ │ │ └── UpperCaseStringFixFileConverter.java │ │ │ └── structure │ │ │ └── fixobject │ │ │ └── FixTableTest.java │ └── main │ │ └── java │ │ └── com │ │ └── collective │ │ └── celos │ │ └── ci │ │ ├── testing │ │ ├── structure │ │ │ ├── fixobject │ │ │ │ ├── FixObject.java │ │ │ │ ├── AbstractFixObjectConverter.java │ │ │ │ ├── FixFsObject.java │ │ │ │ ├── FixDir.java │ │ │ │ ├── FixFile.java │ │ │ │ └── ConversionCreator.java │ │ │ └── tree │ │ │ │ ├── TreeObject.java │ │ │ │ └── TreeObjectProcessor.java │ │ └── fixtures │ │ │ ├── deploy │ │ │ ├── CelosCiDirtyStateException.java │ │ │ └── FixtureDeployer.java │ │ │ ├── compare │ │ │ ├── FixtureComparer.java │ │ │ └── PlainFileComparer.java │ │ │ ├── create │ │ │ ├── FixObjectCreator.java │ │ │ ├── FixFileFromStringCreator.java │ │ │ ├── FixDirHierarchyCreator.java │ │ │ └── FixFileFromResourceCreator.java │ │ │ └── convert │ │ │ ├── FixTableToTSVFileConverter.java │ │ │ └── FixTableToJsonFileConverter.java │ │ ├── mode │ │ └── test │ │ │ ├── TestRunFailedException.java │ │ │ └── TestRunCelosServerMode.java │ │ └── config │ │ └── deploy │ │ └── CelosCiTarget.java └── build.gradle ├── etc ├── images │ ├── arch.png │ ├── oozie.png │ ├── slots.png │ ├── celos-ui.png │ ├── states.png │ ├── hue-index.png │ ├── hue-details.png │ └── quickstart-ui.png └── omnigraffle │ ├── arch.graffle │ ├── slots.graffle │ ├── states.graffle │ └── oozie.graffle │ ├── data.plist │ ├── image1.tiff │ ├── image2.tiff │ ├── image3.tiff │ ├── image4.tiff │ ├── image5.tiff │ ├── image6.tiff │ └── image7.tiff ├── settings.gradle ├── celos-server └── src │ ├── test │ ├── resources │ │ ├── com │ │ │ └── collective │ │ │ │ └── celos │ │ │ │ ├── workflow-configuration-test │ │ │ │ ├── empty │ │ │ │ │ └── this-file-intentionally-left-empty │ │ │ │ ├── single-error │ │ │ │ │ ├── workflow-2-broken.js │ │ │ │ │ ├── workflow-1.js │ │ │ │ │ └── workflow-3.js │ │ │ │ ├── ill-formatted │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── separate-scopes │ │ │ │ │ ├── workflow-1.js │ │ │ │ │ └── workflow-2.js │ │ │ │ ├── cron-task-test │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── json-workflow-list-servlet-test │ │ │ │ │ ├── workflow-1.js │ │ │ │ │ └── workflow-2.js │ │ │ │ ├── id-missing │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── id-not-a-string │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── maxretrycount-missing │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── uses-defaults │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── register-test │ │ │ │ │ └── workflow.js │ │ │ │ ├── duplicate-ids │ │ │ │ │ ├── workflow-1.js │ │ │ │ │ └── workflow-2.js │ │ │ │ ├── maxretrycount-not-a-number │ │ │ │ │ └── workflow-1.js │ │ │ │ └── properties-test │ │ │ │ │ └── workflow-1.js │ │ │ │ ├── server │ │ │ │ └── slot-db-1 │ │ │ │ │ └── state │ │ │ │ │ ├── workflow-1 │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ │ └── workflow-2 │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ ├── state-database-test │ │ │ │ ├── db-1-old │ │ │ │ │ └── state │ │ │ │ │ │ ├── workflow-1 │ │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ │ │ └── workflow-2 │ │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ ├── rerun-db │ │ │ │ │ └── state │ │ │ │ │ │ ├── wf0 │ │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ │ │ └── wf1 │ │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ ├── 15:00:00.000Z │ │ │ │ │ │ └── 16:00:00.000Z │ │ │ │ └── db-1 │ │ │ │ │ └── state │ │ │ │ │ ├── workflow-1 │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ │ └── workflow-2 │ │ │ │ │ └── 2013-12-02 │ │ │ │ │ ├── 17:00:00.000Z │ │ │ │ │ ├── 18:00:00.000Z │ │ │ │ │ └── 19:00:00.000Z │ │ │ │ ├── read-js │ │ │ │ ├── correct-wf-js │ │ │ │ └── incorrect-wf-js │ │ │ │ ├── ui │ │ │ │ └── config.json │ │ │ │ ├── defaults │ │ │ │ └── test.js │ │ │ │ ├── client │ │ │ │ └── wf-list │ │ │ │ │ └── workflow-1.js │ │ │ │ └── defaults-oozie-props │ │ │ │ └── test.js │ │ └── js-tests │ │ │ └── test-registers.js │ └── java │ │ └── com │ │ └── collective │ │ └── celos │ │ ├── JSConfigParserTest.java │ │ ├── MemoryStateDatabaseTest.java │ │ ├── NeverTrigger.java │ │ ├── OnDateTrigger.java │ │ ├── NotTriggerTest.java │ │ ├── servlet │ │ ├── JSONWorkflowListServletTest.java │ │ └── SchedulerServletTest.java │ │ └── RerunStateTest.java │ └── main │ └── java │ └── com │ └── collective │ └── celos │ ├── database │ └── StateDatabase.java │ ├── ExternalStatus.java │ ├── HourlySchedule.java │ ├── MinutelySchedule.java │ ├── ExternalServiceException.java │ ├── servlet │ ├── AbstractJSONServlet.java │ └── ClearCacheServlet.java │ ├── PropertiesGenerator.java │ ├── trigger │ ├── AlwaysTrigger.java │ ├── TriggerStatus.java │ └── NotTrigger.java │ ├── TrivialSchedulingStrategy.java │ ├── Schedule.java │ ├── WorkflowConfiguration.java │ ├── SlotStateStatusPredicate.java │ ├── OozieExternalStatus.java │ ├── ExternalService.java │ ├── DependentSchedule.java │ ├── SchedulingStrategy.java │ └── RerunState.java ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── .gitignore ├── celos-ui ├── src │ └── main │ │ ├── webapp │ │ ├── static │ │ │ ├── style.css │ │ │ └── script.js │ │ └── WEB-INF │ │ │ └── web.xml │ │ └── java │ │ └── com │ │ └── collective │ │ └── celos │ │ └── ui │ │ ├── WorkflowGroup.java │ │ ├── UICommandLine.java │ │ └── Main.java └── build.gradle └── celos-common ├── src └── main │ └── java │ └── com │ └── collective │ └── celos │ ├── BucketID.java │ ├── RegisterKey.java │ ├── DatabaseName.java │ ├── ValueObject.java │ ├── WorkflowID.java │ ├── WorkflowStatus.java │ ├── Constants.java │ ├── servlet │ └── VersionServlet.java │ ├── SlotID.java │ └── WorkflowInfo.java └── build.gradle /samples/quickstart/src/test/resources/input/lax/2015-09-01/0000/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0100/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0200/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0000/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0100/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0200/_READY: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/test/test_files/empty_workflow_list.json: -------------------------------------------------------------------------------- 1 | { 2 | "ids" : [ ] 3 | } -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/create/dir1/file1: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/deploy/hdfs/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/deploy/hdfs/file2: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/jsoncompare/1/_SUCCESS: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/create/dir1/dir2/file2: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/create/dir1/dir2/file3: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/testrun/testcase-1/file: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/testrun/testcase-2/file: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/Avro/dir1/dir2/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/PLAIN/dir1/dir2/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/plain/dir1/dir2/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/compare/equal1: -------------------------------------------------------------------------------- 1 | files are equal -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/compare/equal2: -------------------------------------------------------------------------------- 1 | files are equal -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/Avro/dir2/dir3/dir4/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/PLAIN/dir2/dir3/dir4/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/input/plain/dir2/dir3/dir4/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/core-site.xml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/compare/not-equal: -------------------------------------------------------------------------------- 1 | files are not equal -------------------------------------------------------------------------------- /etc/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/arch.png -------------------------------------------------------------------------------- /etc/images/oozie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/oozie.png -------------------------------------------------------------------------------- /etc/images/slots.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/slots.png -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/input/2013-12-20T1600.txt: -------------------------------------------------------------------------------- 1 | Hello! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/input/2013-12-20T1700.txt: -------------------------------------------------------------------------------- 1 | World! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/input/2013-12-20T1800.txt: -------------------------------------------------------------------------------- 1 | Celos! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/output/2013-12-20T1600.txt: -------------------------------------------------------------------------------- 1 | Hello! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/output/2013-12-20T1700.txt: -------------------------------------------------------------------------------- 1 | World! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test-1/output/2013-12-20T1800.txt: -------------------------------------------------------------------------------- 1 | Celos! 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/workflow_list.json: -------------------------------------------------------------------------------- 1 | { 2 | "ids" : [ "workflow-1", "workflow-2" ] 3 | } -------------------------------------------------------------------------------- /etc/images/celos-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/celos-ui.png -------------------------------------------------------------------------------- /etc/images/states.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/states.png -------------------------------------------------------------------------------- /etc/images/hue-index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/hue-index.png -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'celos' 2 | include 'celos-ci', 'celos-ui', 'celos-server', 'celos-common' -------------------------------------------------------------------------------- /etc/images/hue-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/hue-details.png -------------------------------------------------------------------------------- /etc/images/quickstart-ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/images/quickstart-ui.png -------------------------------------------------------------------------------- /etc/omnigraffle/arch.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/arch.graffle -------------------------------------------------------------------------------- /scripts/templates/run.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec chpst -u {{ service_user }} {{ start_script_path }} 3 | -------------------------------------------------------------------------------- /scripts/test/test_files/workflow_list_register.json: -------------------------------------------------------------------------------- 1 | { 2 | "ids" : [ "workflow-reg-1", "workflow-reg-2" ] 3 | } -------------------------------------------------------------------------------- /etc/omnigraffle/slots.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/slots.graffle -------------------------------------------------------------------------------- /etc/omnigraffle/states.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/states.graffle -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/empty/this-file-intentionally-left-empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /scripts/templates/log-run.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec chpst -u {{ service_user }} svlogd -tt {{ stdout_log_path }} 3 | -------------------------------------------------------------------------------- /scripts/templates/check.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec curl --fail "http://localhost:{{ service_port }}/version" &> /dev/null 3 | -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/data.plist: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/data.plist -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image1.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image1.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image2.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image2.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image3.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image3.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image4.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image4.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image5.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image5.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image6.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image6.tiff -------------------------------------------------------------------------------- /etc/omnigraffle/oozie.graffle/image7.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/etc/omnigraffle/oozie.graffle/image7.tiff -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | CELOS_VERSION=test ./gradlew clean test celos-server:jar celos-ui:jar celos-ci:jar 5 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-1/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"READY","externalID":null,"retryCount":14} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-2/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"READY","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /samples/quickstart/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/samples/quickstart/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-1/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-2/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-2/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"RUNNING","externalID":"quux","retryCount":2} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/server/slot-db-1/state/workflow-1/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"RUNNING","externalID":"foo-bar","retryCount":0} -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.daemon=false 2 | org.gradle.configureondemand=true 3 | org.gradle.parallel=true 4 | org.gradle.jvmopts=-Xmx4096M -XX:MaxPermSize=1024m 5 | 6 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-1/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-1/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"READY","externalID":null,"retryCount":14} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-2/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-2/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"READY","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-2/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"RUNNING","externalID":"quux","retryCount":2} -------------------------------------------------------------------------------- /scripts/test/check-upload/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/scripts/test/check-upload/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1-old/state/workflow-1/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"status":"RUNNING","externalID":"foo-bar","retryCount":0} -------------------------------------------------------------------------------- /scripts/test/test_files/slot_state_workflow_1_1800.json: -------------------------------------------------------------------------------- 1 | { 2 | "time" : "2013-12-02T18:00:00.000Z", 3 | "status" : "READY", 4 | "externalID" : null, 5 | "retryCount" : 14 6 | } -------------------------------------------------------------------------------- /scripts/test/test_files/slot_state_workflow_1_1900.json: -------------------------------------------------------------------------------- 1 | { 2 | "time" : "2013-12-02T19:00:00.000Z", 3 | "status" : "RUNNING", 4 | "externalID" : "foo-bar", 5 | "retryCount" : 0 6 | } -------------------------------------------------------------------------------- /scripts/test/test_files/trigger-status.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "HDFSCheckTrigger", 3 | "ready" : false, 4 | "description" : "HDFS path foo is not ready", 5 | "subStatuses" : [ ] 6 | } -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/rerun-db/state/wf0/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T17:00:00.000Z","status":"SUCCESS","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/rerun-db/state/wf1/2013-12-02/15:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T16:00:00.000Z","status":"SUCCESS","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/rerun-db/state/wf1/2013-12-02/16:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T17:00:00.000Z","status":"SUCCESS","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/avro/avro.avro: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/collectivemedia/celos/HEAD/celos-ci/src/test/resources/com/collective/celos/ci/fixtures/avro/avro.avro -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-1/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T17:00:00.000Z","status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-1/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T18:00:00.000Z","status":"READY","externalID":null,"retryCount":14} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-2/2013-12-02/17:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T17:00:00.000Z","status":"WAITING","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-2/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T18:00:00.000Z","status":"READY","externalID":null,"retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-2/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T19:00:00.000Z","status":"RUNNING","externalID":"quux","retryCount":2} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/rerun-db/state/wf0/2013-12-02/18:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T18:00:00.000Z","status":"SUCCESS","externalID":null,"retryCount":14} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/rerun-db/state/wf0/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T19:00:00.000Z","status":"SUCCESS","externalID":"foo-bar","retryCount":0} -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/state-database-test/db-1/state/workflow-1/2013-12-02/19:00:00.000Z: -------------------------------------------------------------------------------- 1 | {"time":"2013-12-02T19:00:00.000Z","status":"RUNNING","externalID":"foo-bar","retryCount":0} -------------------------------------------------------------------------------- /scripts/templates/service-start.tpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | CELOS_VERSION={{ service_version }} exec java 2>&1 -cp "{{ dest_jar_path }}:/etc/hadoop/conf" {{ main_class }} --port {{ service_port }} {{ service_args }} 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | reports/ 2 | target/ 3 | provisioner/tmp/ 4 | **/build/* 5 | **/bin/* 6 | .gradle/ 7 | .idea/ 8 | *.iml 9 | javadoc-tmp/ 10 | /bin/ 11 | *~ 12 | **/.classpath 13 | **/.project 14 | **/.settings 15 | **/.DS_Store 16 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/target-no-deffile.json: -------------------------------------------------------------------------------- 1 | { 2 | "workflows.dir.uri": "celoswfdir", 3 | "hadoop.hdfs-site.xml": "hadoophdfsurl", 4 | "hadoop.core-site.xml": "hadoopcoreurl" 5 | } 6 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/target-no-wf-uri.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaults.dir.uri": "celoswfdir", 3 | "hadoop.hdfs-site.xml": "hadoophdfsurl", 4 | "hadoop.core-site.xml": "hadoopcoreurl" 5 | } 6 | -------------------------------------------------------------------------------- /samples/quickstart/ci/undeploy.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | set -e 3 | export HDFS_ROOT=$1 4 | 5 | # Run Celos CI 6 | java -jar ../../celos-ci/build/libs/celos-ci-fat.jar --workflowName wordcount --mode undeploy --target $TARGET_FILE --hdfsRoot $HDFS_ROOT 7 | -------------------------------------------------------------------------------- /scripts/test/conf/integration-params.json: -------------------------------------------------------------------------------- 1 | { 2 | "celos_wrk_dir": "/home/celos-ci/integration", 3 | "server_service_name": "int-celos-server", 4 | "server_service_port": 19091, 5 | "ui_service_name": "int-celos-ui", 6 | "ui_service_port": 19092 7 | } -------------------------------------------------------------------------------- /scripts/params/common-ui.json: -------------------------------------------------------------------------------- 1 | { 2 | "jar_path": "{{ root }}/celos-ui/build/libs/celos-ui.jar", 3 | "main_class": "com.collective.celos.ui.Main", 4 | "service_args": "--celos {{ service_discovery }} --hue {{ ui_hue_url }} --config {{ ui_config_file }}" 5 | } 6 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/target-no-sec.json: -------------------------------------------------------------------------------- 1 | { 2 | "workflows.dir.uri": "celoswfdir", 3 | "hadoop.hdfs-site.xml": "hadoophdfsurl", 4 | "hadoop.core-site.xml": "hadoopcoreurl", 5 | "defaults.dir.uri": "defdir" 6 | } 7 | -------------------------------------------------------------------------------- /scripts/inventory/production-ui: -------------------------------------------------------------------------------- 1 | [host] 2 | celos002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name="celos-ui" 6 | service_port=9092 7 | ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie" 8 | ui_config_file="/home/celos/prod/ui/config.json" 9 | -------------------------------------------------------------------------------- /scripts/playbooks/celos-stop.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | 9 | tasks: 10 | 11 | - shell: "/sbin/sv stop {{service_name}}" 12 | -------------------------------------------------------------------------------- /scripts/playbooks/celos-check.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | 9 | tasks: 10 | 11 | - shell: "/sbin/sv check {{service_name}}" 12 | -------------------------------------------------------------------------------- /scripts/playbooks/celos-start.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | 9 | tasks: 10 | 11 | - shell: "/sbin/sv start {{service_name}}" 12 | -------------------------------------------------------------------------------- /scripts/inventory/integration-ui: -------------------------------------------------------------------------------- 1 | [host] 2 | oj002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name=celos-integration-ui 6 | service_port=19096 7 | ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie" 8 | ui_config_file="/home/celos/prod/ui/config.json" 9 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Aug 27 15:20:36 EDT 2014 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip 7 | -------------------------------------------------------------------------------- /scripts/inventory/jdbc-production-ui: -------------------------------------------------------------------------------- 1 | [host] 2 | celos002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name="jdbc-celos-ui" 6 | service_port=9094 7 | ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie" 8 | ui_config_file="/home/celos/prod/ui/config-jdbc.json" 9 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/target.json: -------------------------------------------------------------------------------- 1 | { 2 | "security.settings": "secsettings", 3 | "workflows.dir.uri": "celoswfdir", 4 | "hadoop.hdfs-site.xml": "hadoophdfsurl", 5 | "hadoop.core-site.xml": "hadoopcoreurl", 6 | "defaults.dir.uri": "defdir" 7 | } 8 | -------------------------------------------------------------------------------- /samples/quickstart/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Mar 18 16:43:54 EDT 2014 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-1.11-bin.zip 7 | -------------------------------------------------------------------------------- /scripts/params/common-server.json: -------------------------------------------------------------------------------- 1 | { 2 | "service_args": "--workflows {{ celos_workflows_dir }} --defaults {{ celos_defaults_dir }} --logs {{ celos_log_dir }} {{ extra_args }}", 3 | "jar_path": "../../celos-server/build/libs/celos-server.jar", 4 | "main_class": "com.collective.celos.server.Main" 5 | } 6 | 7 | -------------------------------------------------------------------------------- /scripts/test/check-upload/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Mar 18 16:43:54 EDT 2014 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-1.11-bin.zip 7 | -------------------------------------------------------------------------------- /scripts/inventory/production-server: -------------------------------------------------------------------------------- 1 | [host] 2 | celos002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name=celos-server 6 | service_port=9091 7 | celos_wrk_dir=/home/celos/prod 8 | celos_workflows_dir=/etc/celos/workflows 9 | celos_defaults_dir=/etc/celos/defaults 10 | celos_log_dir=/var/log/celos 11 | extra_args="--db /var/lib/celos/db" 12 | -------------------------------------------------------------------------------- /scripts/test/conf/inventory-ui: -------------------------------------------------------------------------------- 1 | [host] 2 | oj002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | celos_server_host="oj002.ewr004.collective-media.net" 6 | service_name={{ui_service_name}} 7 | service_port={{ui_service_port}} 8 | ui_hue_url="http://cldmgr001.ewr004.collective-media.net:8888/oozie" 9 | ui_config_file="/home/celos/prod/ui/config.json" 10 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0100/file3.txt: -------------------------------------------------------------------------------- 1 | The standard chunk of Lorem Ipsum used since the 1500s is reproduced below for those interested. Sections 1.10.32 and 1.10.33 from "de Finibus Bonorum et Malorum" by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham. 2 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0100/file3.txt: -------------------------------------------------------------------------------- 1 | The standard chunk of Lorem Ipsum used since the 1500s is reproduced below for those interested. Sections 1.10.32 and 1.10.33 from "de Finibus Bonorum et Malorum" by Cicero are also reproduced in their exact original form, accompanied by English versions from the 1914 translation by H. Rackham. 2 | -------------------------------------------------------------------------------- /samples/quickstart/ci/target.json: -------------------------------------------------------------------------------- 1 | { 2 | "hadoop.hdfs-site.xml": "file:///etc/hadoop/conf/hdfs-site.xml", 3 | "hadoop.core-site.xml": "file:///etc/hadoop/conf/core-site.xml", 4 | "defaults.dir.uri": "file:///home/manuel/celos/samples/quickstart/celos.d/defaults", 5 | "workflows.dir.uri": "file:///home/manuel/celos/samples/quickstart/celos.d/workflows" 6 | } 7 | -------------------------------------------------------------------------------- /scripts/playbooks/kinit.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | vars: 9 | service_user: "{{ ansible_ssh_user }}" 10 | 11 | tasks: 12 | 13 | - shell: kinit -k -t /home/{{service_user}}/{{service_user}}.keytab {{service_user}} 14 | -------------------------------------------------------------------------------- /scripts/test/conf/inventory-server: -------------------------------------------------------------------------------- 1 | [host] 2 | oj002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | celos_workflows_dir={{ celos_wrk_dir }}/workflows 6 | celos_defaults_dir={{ celos_wrk_dir }}/defaults 7 | celos_db_dir={{ celos_wrk_dir }}/db 8 | celos_log_dir={{ celos_wrk_dir }}/logs 9 | service_name={{server_service_name}} 10 | service_port={{server_service_port}} 11 | -------------------------------------------------------------------------------- /scripts/add-license.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set x 3 | 4 | for x in $(find . -name *.js -o -name *.java) 5 | do 6 | if [ "$(grep '* Copyright 2015 Collective, Inc.' ${x})" ] 7 | then 8 | echo skipping ${x} 9 | else 10 | echo processing ${x} 11 | cp ./scripts/templates/license.tpl tmp.txt 12 | cat ${x} >> tmp.txt 13 | mv tmp.txt ${x} 14 | fi 15 | done 16 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/scripts/ci-undeploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -x 4 | 5 | cd $1 6 | 7 | ROOT=$(pwd) 8 | 9 | cd $ROOT/scripts/test/test_files/file-copy 10 | 11 | java -jar $ROOT/celos-ci/build/libs/celos-ci.jar --workflowName file-copy --mode undeploy --target file://$ROOT/scripts/test/target_int_test.json --hdfsRoot /user/celos-ci/celos-int-test 12 | 13 | echo You win! All tests OK. 14 | -------------------------------------------------------------------------------- /scripts/prod/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | export CELOS_USER=celos 5 | export INVENTORY_SERVER=scripts/inventory/production-server 6 | export INVENTORY_UI=scripts/inventory/production-ui 7 | ./scripts/server-and-ui-action.sh stop 8 | 9 | export INVENTORY_SERVER=scripts/inventory/jdbc-production-server 10 | export INVENTORY_UI=scripts/inventory/jdbc-production-ui 11 | ./scripts/server-and-ui-action.sh stop 12 | -------------------------------------------------------------------------------- /scripts/prod/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | export CELOS_USER=celos 5 | export INVENTORY_SERVER=scripts/inventory/production-server 6 | export INVENTORY_UI=scripts/inventory/production-ui 7 | ./scripts/server-and-ui-action.sh start 8 | 9 | export INVENTORY_SERVER=scripts/inventory/jdbc-production-server 10 | export INVENTORY_UI=scripts/inventory/jdbc-production-ui 11 | ./scripts/server-and-ui-action.sh start 12 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/read-js/correct-wf-js: -------------------------------------------------------------------------------- 1 | importDefault("collective.js"); 2 | 3 | celos.defineWorkflow({ 4 | "id": "workflow-1", 5 | "schedule": celos.hourlySchedule(), 6 | "schedulingStrategy": celos.serialSchedulingStrategy(), 7 | "trigger": celos.hdfsCheckTrigger("/", "file:///"), 8 | "externalService": celos.oozdieExternalService({}, "oj01/oozie"), 9 | "maxRetryCount": 0 10 | }); 11 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/read-js/incorrect-wf-js: -------------------------------------------------------------------------------- 1 | importDefault("collective.js"); 2 | 3 | celos.defineWorkflow({ 4 | "id": "workflow-1", 5 | "schedule": celos.hourlySchedule(), 6 | "schedulingStrategy": celos.serialSchedulingStrategy(), 7 | "trigger": celos.hdfsCheckTrigger("/", "file:///") 8 | "externalService": celos.oozdieExternalService({}, "oj01/oozie"), 9 | "maxRetryCount": 0 10 | }); 11 | -------------------------------------------------------------------------------- /scripts/inventory/integration-server: -------------------------------------------------------------------------------- 1 | [host] 2 | oj002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name=celos-integration-server 6 | service_port=19095 7 | celos_wrk_dir="/home/{{ ansible_ssh_user }}/integration" 8 | celos_workflows_dir="{{ celos_wrk_dir }}/workflows" 9 | celos_defaults_dir="{{ celos_wrk_dir }}/defaults" 10 | celos_log_dir="{{ celos_wrk_dir }}/logs" 11 | celos_db_dir="{{ celos_wrk_dir }}/db" 12 | extra_args="--db {{ celos_db_dir }}" 13 | -------------------------------------------------------------------------------- /samples/quickstart/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'application' 2 | apply plugin: 'java' 3 | apply plugin: 'idea' 4 | 5 | sourceCompatibility = 1.7 6 | version = '1.0' 7 | jar.baseName = 'wordcount' 8 | 9 | repositories { 10 | mavenLocal() 11 | mavenCentral() 12 | maven { url "http://repo1.maven.org/maven2" } 13 | maven { url "https://repository.cloudera.com/artifactory/cloudera-repos" } 14 | } 15 | 16 | dependencies { 17 | compile("org.apache.hadoop:hadoop-client:2.5.0") 18 | } 19 | 20 | -------------------------------------------------------------------------------- /scripts/inventory/jdbc-production-server: -------------------------------------------------------------------------------- 1 | [host] 2 | celos002.ewr004.collective-media.net 3 | 4 | [host:vars] 5 | service_name=jdbc-celos-server 6 | service_port=9093 7 | celos_wrk_dir=/home/celos/jdbc-prod 8 | celos_workflows_dir="{{ celos_wrk_dir }}/workflows" 9 | celos_defaults_dir="{{ celos_wrk_dir }}/defaults" 10 | celos_log_dir="{{ celos_wrk_dir }}/logs" 11 | extra_args="--dbType JDBC --jdbcUrl jdbc:postgresql://dev-workflow002.ewr004.collective-media.net:5432/celos_test --jdbcName celos --jdbcPassword celos" -------------------------------------------------------------------------------- /scripts/README.asciidoc: -------------------------------------------------------------------------------- 1 | This directory contains our Ansible + Runit based production setup, as 2 | well as the integration test suite. 3 | 4 | With a little modification of the Ansible inventory files, the 5 | production setup can be used to deploy Celos and Celos UI to any 6 | host(s) with Runit installed. 7 | 8 | link:https://github.com/collectivemedia/celos#get-in-touch[Contact us] 9 | if you're interested in using it at your site. It could probably be 10 | made a bit more easily reusable if there's interest. 11 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/scripts/ci-deploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -x 4 | 5 | cd $1 6 | 7 | ROOT=$(pwd) 8 | 9 | cd $ROOT/scripts/test/test_files/file-copy 10 | 11 | rm -fr celos_deploy 12 | mkdir -p celos_deploy/hdfs 13 | 14 | cp src/main/celos/* celos_deploy 15 | cp src/main/oozie/* celos_deploy/hdfs 16 | 17 | java -jar $ROOT/celos-ci/build/libs/celos-ci.jar --deployDir celos_deploy --workflowName file-copy --mode deploy --target file://$ROOT/scripts/test/target_int_test.json --hdfsRoot /user/celos-ci/celos-int-test 18 | 19 | echo You win! All tests OK. 20 | -------------------------------------------------------------------------------- /scripts/test/target_int_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "hadoop.hdfs-site.xml": "sftp://oj002.ewr004.collective-media.net/etc/hadoop/conf/hdfs-site.xml", 3 | "hadoop.core-site.xml": "sftp://oj002.ewr004.collective-media.net/etc/hadoop/conf/core-site.xml", 4 | "defaults.dir.uri": "sftp://oj002.ewr004.collective-media.net/home/celos-ci/integration/defaults", 5 | "workflows.dir.uri": "sftp://oj002.ewr004.collective-media.net/home/celos-ci/integration/workflows", 6 | "hive.jdbc.url" : "jdbc:hive2://hive002.ny7.collective-media.net:10000/default;principal=hive/hive002.ny7.collective-media.net@COLLECTIVE-MEDIA.NET" 7 | } 8 | -------------------------------------------------------------------------------- /samples/quickstart/ci/deploy.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | set -e 3 | export HDFS_ROOT=$1 4 | 5 | # Build JAR 6 | ./gradlew clean jar 7 | 8 | # Create deploy dir at build/celos-deploy 9 | export DEPLOY_DIR=build/celos-deploy 10 | rm -rf $DEPLOY_DIR 11 | mkdir -p $DEPLOY_DIR/hdfs/lib 12 | 13 | # Copy files into deploy dir 14 | cp src/main/celos/* $DEPLOY_DIR/ 15 | cp src/main/oozie/* $DEPLOY_DIR/hdfs/ 16 | cp build/libs/* $DEPLOY_DIR/hdfs/lib/ 17 | 18 | # Run Celos CI 19 | java -jar ../../celos-ci/build/libs/celos-ci-fat.jar --workflowName wordcount --mode deploy --deployDir $DEPLOY_DIR --target $TARGET_FILE --hdfsRoot $HDFS_ROOT 20 | -------------------------------------------------------------------------------- /scripts/test/ui-test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | vars: 9 | service_url: "http://{{inventory_hostname}}:{{service_port}}" 10 | 11 | tasks: 12 | 13 | - name: Check static content is served 14 | local_action: 15 | module: uri 16 | status_code: 200 17 | url: "{{service_url}}/static/style.css" 18 | 19 | - name: Check dynamic content is served 20 | local_action: 21 | module: uri 22 | status_code: 200 23 | url: "{{service_url}}/ui" 24 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0000/file1.txt: -------------------------------------------------------------------------------- 1 | Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. 2 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0000/file1.txt: -------------------------------------------------------------------------------- 1 | Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/main/oozie/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | ${wf:errorCode("failed")} 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /samples/quickstart/src/main/oozie/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${jobTracker} 7 | ${nameNode} 8 | com.example.WordCount 9 | ${inputPath} 10 | ${outputPath} 11 | 12 | 13 | 14 | 15 | 16 | ${wf:errorCode("failed")} 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/README.md: -------------------------------------------------------------------------------- 1 | # Celos Sample Workflow 2 | 3 | An Oozie workflow that simply copies (moves, actually) its hourly 4 | input files to an output directory. 5 | 6 | The [input](input) dir contains hourly text files. 7 | 8 | The [output](output) dir contains the expected outputs (the same 9 | files, in this case, since the workflow is basically an identity 10 | function). 11 | 12 | The [workflow](workflow) dir contains the Oozie [`workflow.xml`](workflow/workflow.xml). 13 | 14 | The [`file-copy.json`](file-copy.json) file is the Celos workflow definition. 15 | 16 | The [`test.yaml`](test.yaml) file is the Ansible based testing script. 17 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/celos_deploy/hdfs/workflow.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | ${wf:errorCode("failed")} 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/ui/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "hueURL": "http://cldmgr001.ewr004.collective-media.net:8888/oozie/list_oozie_workflow/${EXTERNAL_ID}/", 3 | "workflowToSlotMap": [ 4 | { 5 | "name": "Flume", 6 | "workflows": [ 7 | "flume-ready-dal", 8 | "flume-ready-dc", 9 | "flume-ready-dc3", 10 | "flume-ready-lax1", 11 | "flume-ready-nym1", 12 | "flume-ready-sea", 13 | "flume-ready-sv4", 14 | "flume-tmp-file-closer" 15 | ] 16 | }, 17 | { 18 | "name": "FTAS", 19 | "workflows": [ 20 | "ftas-gc-aof" 21 | ] 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0100/file4.txt: -------------------------------------------------------------------------------- 1 | It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like). 2 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0100/file4.txt: -------------------------------------------------------------------------------- 1 | It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like). 2 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/scripts/ci-test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -x 4 | 5 | CELOS_SERVER=$1 6 | cd $2 7 | 8 | ROOT=$(pwd) 9 | 10 | cd $ROOT/scripts/test/test_files/file-copy 11 | # Run integration tests against currently checkout local source on test cluster. 12 | rm -fr celos_deploy 13 | mkdir -p celos_deploy/hdfs 14 | 15 | cp src/main/celos/* celos_deploy 16 | cp src/main/oozie/* celos_deploy/hdfs 17 | 18 | java -jar $ROOT/celos-ci/build/libs/celos-ci.jar --testDir src/test/celos-ci --deployDir celos_deploy --workflowName file-copy --mode test --target file://$ROOT/scripts/test/target_int_test.json --celos $CELOS_SERVER 19 | 20 | echo You win! All tests OK. 21 | -------------------------------------------------------------------------------- /scripts/templates/license.tpl: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/deploy/workflow.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/single-error/workflow-2-broken.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | blah -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/ill-formatted/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | THIS IS NOT A JS FILE LOL!!!1!! 17 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0200/file5.txt: -------------------------------------------------------------------------------- 1 | There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration in some form, by injected humour, or randomised words which don't look even slightly believable. If you are going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in the middle of text. All the Lorem Ipsum generators on the Internet tend to repeat predefined chunks as necessary, making this the first true generator on the Internet. It uses a dictionary of over 200 Latin words, combined with a handful of model sentence structures, to generate Lorem Ipsum which looks reasonable. The generated Lorem Ipsum is therefore always free from repetition, injected humour, or non-characteristic words etc. 2 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0200/file5.txt: -------------------------------------------------------------------------------- 1 | There are many variations of passages of Lorem Ipsum available, but the majority have suffered alteration in some form, by injected humour, or randomised words which don't look even slightly believable. If you are going to use a passage of Lorem Ipsum, you need to be sure there isn't anything embarrassing hidden in the middle of text. All the Lorem Ipsum generators on the Internet tend to repeat predefined chunks as necessary, making this the first true generator on the Internet. It uses a dictionary of over 200 Latin words, combined with a handful of model sentence structures, to generate Lorem Ipsum which looks reasonable. The generated Lorem Ipsum is therefore always free from repetition, injected humour, or non-characteristic words etc. 2 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/separate-scopes/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | // Set variable in first workflow 17 | FOO = 1; 18 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/defaults/test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | var FOO = 1; 17 | var BAR = 2; 18 | 19 | function myFunction() { 20 | return BAR; 21 | } 22 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/defaults/test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | var FOO = 1; 17 | var BAR = 2; 18 | 19 | function myFunction() { 20 | return BAR; 21 | } 22 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/lax/2015-09-01/0000/file2.txt: -------------------------------------------------------------------------------- 1 | Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32. 2 | -------------------------------------------------------------------------------- /samples/quickstart/src/test/resources/input/nyc/2015-09-01/0000/file2.txt: -------------------------------------------------------------------------------- 1 | Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor sit amet..", comes from a line in section 1.10.32. 2 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/separate-scopes/workflow-2.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | // Make sure variable from workflow-1 isn't visible 17 | if (typeof FOO !== "undefined") throw "FOO leaked from workflow-1"; -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/database/StateDatabase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.database; 17 | 18 | public interface StateDatabase { 19 | 20 | public StateDatabaseConnection openConnection() throws Exception; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/FixObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | /** 19 | * Created by akonopko on 22.01.15. 20 | */ 21 | public abstract class FixObject { 22 | } 23 | -------------------------------------------------------------------------------- /scripts/prod/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | [[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1 5 | 6 | export CELOS_USER=celos 7 | export ANSIBLE_SSH_ARGS="" 8 | 9 | scripts/build.sh 10 | 11 | export INVENTORY_SERVER=scripts/inventory/production-server 12 | export INVENTORY_UI=scripts/inventory/production-ui 13 | scripts/server-and-ui-action.sh deploy 14 | 15 | export INVENTORY_SERVER=scripts/inventory/jdbc-production-server 16 | export INVENTORY_UI=scripts/inventory/jdbc-production-ui 17 | scripts/server-and-ui-action.sh deploy 18 | 19 | MINUTES_JAN01_2015=23667720 20 | MINUTES_SINCE_JAN01_2015=$(($(date +%s) / 60 - MINUTES_JAN01_2015)) 21 | export CELOS_BUILD_NUMBER=$MINUTES_SINCE_JAN01_2015 22 | ./gradlew clean celos-server:uploadArchives celos-ci:uploadArchives celos-common:uploadArchives 23 | (cd scripts/test/check-upload && ./gradlew checkVersion --refresh-dependencies) 24 | -------------------------------------------------------------------------------- /celos-ui/src/main/webapp/static/style.css: -------------------------------------------------------------------------------- 1 | html { font-family: sans; background-color: #fff; font-size: 50%; } 2 | a { text-decoration: none; } 3 | .mainTable {} 4 | .workflowGroup { text-align: right; font-size: large; font-weight: bold; padding-top: 1em; padding-right: 20px; white-space: nowrap; } 5 | .workflow { text-align: right; padding-right: 20px; font-weight: normal; white-space: nowrap; } 6 | .missing { color: red; } 7 | .hour, .day, .noDay, .dayHeader { font-family: monospace; text-align: center; } 8 | .day { background-color: black; color: white; font-weight: bold; } 9 | .currentDate { font-family: monospace; text-align: right; padding-right: 20px; font-weight: bold; } 10 | .slot { font-family: monospace; font-size: small; } 11 | .RUNNING, .READY { background-color: #ffc; } 12 | .SUCCESS { background-color: #cfc; } 13 | .WAITING { background-color: #ccf; } 14 | .FAILURE, .WAIT_TIMEOUT { background-color: #fcc; } 15 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/ExternalStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Status of a workflow in an external service. 20 | */ 21 | public interface ExternalStatus { 22 | 23 | boolean isRunning(); 24 | 25 | boolean isSuccess(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/mode/test/TestRunFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.mode.test; 17 | 18 | public class TestRunFailedException extends Exception { 19 | 20 | public TestRunFailedException(String message) { 21 | super(message); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/HourlySchedule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Simple schedule for hourly execution. 20 | */ 21 | public class HourlySchedule extends CronSchedule { 22 | 23 | public HourlySchedule() { 24 | super("0 0 * * * ?"); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/MinutelySchedule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Simple schedule for - ahem - minutely execution. 20 | */ 21 | public class MinutelySchedule extends CronSchedule { 22 | 23 | public MinutelySchedule() { 24 | super("0 * * * * ?"); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/tree/TreeObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.tree; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * Created by akonopko on 10/9/14. 22 | */ 23 | public interface TreeObject { 24 | 25 | public Map getChildren(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/ExternalServiceException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Exception from external service. 20 | */ 21 | @SuppressWarnings("serial") 22 | public class ExternalServiceException extends Exception { 23 | 24 | public ExternalServiceException(Throwable t) { 25 | super(t); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/deploy/CelosCiDirtyStateException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.deploy; 17 | 18 | /** 19 | * Created by akonopko on 04.02.15. 20 | */ 21 | public class CelosCiDirtyStateException extends Exception { 22 | 23 | public CelosCiDirtyStateException(String message) { 24 | super(message); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /scripts/test/stop-start-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | SERVER_PARAMS="-c ssh -u celos-ci -i scripts/test/conf/inventory-server -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-server.json" 3 | UI_PARAMS=" -c ssh -u celos-ci -i scripts/test/conf/inventory-ui -e @scripts/test/conf/integration-params.json -e @scripts/conf/common-params-ui.json" 4 | set -e 5 | set -x 6 | ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS} 7 | ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS} 8 | ansible-playbook scripts/prod/celos-stop.yaml ${SERVER_PARAMS} 9 | ansible-playbook scripts/prod/celos-stop.yaml ${UI_PARAMS} 10 | ! ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS} 11 | ! ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS} 12 | ansible-playbook scripts/prod/celos-start.yaml ${SERVER_PARAMS} 13 | ansible-playbook scripts/prod/celos-start.yaml ${UI_PARAMS} 14 | ansible-playbook scripts/prod/celos-check.yaml ${SERVER_PARAMS} 15 | ansible-playbook scripts/prod/celos-check.yaml ${UI_PARAMS} 16 | -------------------------------------------------------------------------------- /scripts/test/check-upload/build.gradle: -------------------------------------------------------------------------------- 1 | repositories { 2 | maven { url "$System.env.NEXUS_MAVEN_REPO" } 3 | } 4 | 5 | configurations { 6 | compile { 7 | transitive = false 8 | } 9 | } 10 | 11 | dependencies { 12 | compile "com.collective:celos-ci:2.+" 13 | compile "com.collective:celos-server:2.+" 14 | } 15 | 16 | task checkVersion << { 17 | checkLib('celos-ci') 18 | checkLib('celos-server') 19 | checkLib('celos-common') 20 | } 21 | 22 | def checkLib(libName) { 23 | configurations.compile.files { dep -> dep.name == libName }.each { file -> 24 | if (!System.getenv("CELOS_BUILD_NUMBER")) throw new GradleException("CELOS_BUILD_NUMBER env variable should be present") 25 | ext.expFilename = libName + "-2.0." + System.getenv("CELOS_BUILD_NUMBER") + ".jar" 26 | if (file.name != ext.expFilename) { 27 | throw new GradleException("Failed to get most recently uploaded lib version: last available is " + file.name + " but expected " + ext.expFilename) 28 | } 29 | 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/compare/FixtureComparer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.compare; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | 20 | /** 21 | * Created by akonopko on 10/10/14. 22 | */ 23 | public interface FixtureComparer { 24 | 25 | public FixObjectCompareResult check(TestRun testRun) throws Exception; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/client/wf-list/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": celos.hourlySchedule(), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("/", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/client/cron-task-test/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": celos.cronSchedule("0 12 * * * ?"), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/client/cron-task-test/workflow-2.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": celos.cronSchedule("0 12 * * * ?"), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/JSConfigParserTest.java: -------------------------------------------------------------------------------- 1 | 2 | package com.collective.celos; 3 | 4 | import org.junit.Test; 5 | import org.mozilla.javascript.EvaluatorException; 6 | 7 | import java.io.File; 8 | import java.io.FileReader; 9 | 10 | public class JSConfigParserTest { 11 | 12 | @Test(expected = EvaluatorException.class) 13 | public void testCompileFails() throws Exception { 14 | JSConfigParser parser = new JSConfigParser(); 15 | File file = new File(Thread.currentThread().getContextClassLoader().getResource("com/collective/celos/read-js/incorrect-wf-js").toURI()); 16 | 17 | parser.validateJsSyntax(new FileReader(file), file.getName()); 18 | } 19 | 20 | @Test 21 | public void testCompilePasses() throws Exception { 22 | JSConfigParser parser = new JSConfigParser(); 23 | File file = new File(Thread.currentThread().getContextClassLoader().getResource("com/collective/celos/read-js/correct-wf-js").toURI()); 24 | 25 | parser.validateJsSyntax(new FileReader(file), file.getName()); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/test/celos-ci/test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | ci.addTestCase({ 17 | name: "filecopy test case 1", 18 | sampleTimeStart: "2013-12-20T16:00Z", 19 | sampleTimeEnd: "2013-12-20T18:00Z", 20 | inputs: [ 21 | ci.hdfsInput(ci.fixDirFromResource("test-1/input"), "/input/file-copy") 22 | ], 23 | outputs: [ 24 | ci.plainCompare(ci.fixDirFromResource("test-1/output"), "/output/file-copy") 25 | ] 26 | }); -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/cron-task-test/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": celos.cronSchedule("0 12 * * * ?"), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/json-workflow-list-servlet-test/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": celos.hourlySchedule(), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/json-workflow-list-servlet-test/workflow-2.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-2", 18 | "schedule": celos.hourlySchedule(), 19 | "schedulingStrategy": celos.serialSchedulingStrategy(), 20 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 21 | "externalService": celos.oozieExternalService({}, "oj01/oozie"), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/AbstractFixObjectConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | 20 | /** 21 | * Created by akonopko on 10/7/14. 22 | */ 23 | public abstract class AbstractFixObjectConverter { 24 | 25 | public abstract T convert(TestRun tr, S ff) throws Exception; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/id-missing/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 18 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 19 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 20 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 21 | "maxRetryCount": 0 22 | }); 23 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/id-not-a-string/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": [ "foo" ], 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService() 22 | }); 23 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/MemoryStateDatabaseTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.collective.celos.database.StateDatabaseConnection; 19 | 20 | import java.io.IOException; 21 | 22 | public class MemoryStateDatabaseTest extends AbstractStateDatabaseTest { 23 | 24 | @Override 25 | public StateDatabaseConnection getStateDatabaseConnection() throws IOException { 26 | return new MemoryStateDatabase().openConnection(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/maxretrycount-missing/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService() 22 | }); 23 | -------------------------------------------------------------------------------- /scripts/test/integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | [[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1 5 | 6 | export CELOS_USER=celos-ci 7 | export INVENTORY_SERVER=scripts/inventory/integration-server 8 | export INVENTORY_UI=scripts/inventory/integration-ui 9 | export ANSIBLE_SSH_ARGS="" 10 | 11 | scripts/build.sh 12 | ansible-playbook scripts/playbooks/kinit.yaml -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER} 13 | 14 | ./scripts/server-and-ui-action.sh deploy 15 | # begin stop-strat 16 | ./scripts/server-and-ui-action.sh check 17 | ./scripts/server-and-ui-action.sh stop 18 | ! ./scripts/server-and-ui-action.sh check 19 | ./scripts/server-and-ui-action.sh start 20 | ./scripts/server-and-ui-action.sh check 21 | # end stop-strat 22 | ansible-playbook scripts/test/celos_test.yaml -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER} -e @scripts/params/common-server.json -e service_version=${GIT_COMMIT} 23 | ansible-playbook scripts/test/ui-test.yaml -c ssh -u ${CELOS_USER} -i ${INVENTORY_UI} -e @scripts/params/common-ui.json -e service_version=${GIT_COMMIT} 24 | # cleanup 25 | ./scripts/server-and-ui-action.sh purge 26 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/mode/test/TestRunCelosServerMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.mode.test; 17 | 18 | import java.net.URI; 19 | 20 | /** 21 | * Created by akonopko on 12.08.15. 22 | */ 23 | interface TestRunCelosServerMode { 24 | 25 | URI startServer(TestRun testRun) throws Exception; 26 | 27 | void stopServer(TestRun testRun); 28 | 29 | URI getCelosWorkflowDir(); 30 | 31 | URI getCelosDefaultsDir(); 32 | 33 | String getHdfsPrefix(); 34 | } 35 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/deploy/FixtureDeployer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.deploy; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | 20 | /** 21 | * Created by akonopko on 22.11.14. 22 | */ 23 | public interface FixtureDeployer { 24 | 25 | void deploy(TestRun testRun) throws Exception; 26 | 27 | void undeploy(TestRun testRun) throws Exception; 28 | 29 | void validate(TestRun testRun) throws Exception; 30 | } 31 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/uses-defaults/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | if (typeof FOO !== "undefined") throw "FOO already set"; 17 | if (typeof BAR !== "undefined") throw "BAR already set"; 18 | if (typeof myFunction !== "undefined") throw "myFunction already set"; 19 | 20 | importDefaults("test"); 21 | 22 | if (FOO !== 1) throw "FOO was not imported"; 23 | if (BAR !== 2) throw "BAR was not imported"; 24 | if (myFunction() !== 2) throw "myFunction was not imported"; 25 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/single-error/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/single-error/workflow-3.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-3", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/register-test/workflow.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.forEachRegister("celos-register", function(key, value) { 17 | celos.defineWorkflow({ 18 | "id": "workflow-reg-" + value.data, 19 | "schedule": celos.hourlySchedule(), 20 | "schedulingStrategy": celos.serialSchedulingStrategy(), 21 | "trigger": celos.hdfsCheckTrigger("foo", "file:///"), 22 | "externalService": celos.oozieExternalService({}, "oj01/oozie") 23 | }); 24 | }); -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/duplicate-ids/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "this-is-a-duplicate", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/duplicate-ids/workflow-2.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "this-is-a-duplicate", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 22 | "maxRetryCount": 0 23 | }); 24 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/maxretrycount-not-a-number/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.defineWorkflow({ 17 | "id": "workflow-1", 18 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 19 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 20 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 21 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 22 | "maxRetryCount": "foo" 23 | }); -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/servlet/AbstractJSONServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.servlet; 17 | 18 | import com.collective.celos.Util; 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import com.fasterxml.jackson.databind.ObjectWriter; 21 | 22 | /** 23 | * Utilities for JSON API servlets. 24 | */ 25 | @SuppressWarnings("serial") 26 | public class AbstractJSONServlet extends AbstractServlet { 27 | 28 | protected final ObjectWriter writer = Util.JSON_WRITER.withDefaultPrettyPrinter(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /celos-ui/build.gradle: -------------------------------------------------------------------------------- 1 | mainClassName = 'com.collective.celos.ui.Main' 2 | version = '0.1' 3 | 4 | jar { 5 | baseName "celos-ui" 6 | manifest { 7 | attributes("Main-Class": mainClassName) 8 | } 9 | } 10 | 11 | sourceSets { 12 | main { 13 | java { 14 | srcDir 'src/main/java' 15 | } 16 | resources { 17 | srcDir 'src/main/resources' 18 | srcDir 'src/main/webapp' 19 | } 20 | } 21 | test { 22 | java { 23 | srcDir 'src/test/java' 24 | } 25 | } 26 | } 27 | 28 | dependencies { 29 | compile project(':celos-common') 30 | compile("com.j2html:j2html:0.5.0") 31 | testCompile("junit:junit:4.8.2") 32 | testCompile("net.sourceforge.htmlunit:htmlunit:2.4") 33 | testCompile("org.powermock:powermock-module-junit4:1.5.5") { 34 | exclude module: 'junit' 35 | } 36 | testCompile("org.powermock:powermock-api-mockito:1.5.5") { 37 | exclude module: 'junit' 38 | } 39 | testCompile("org.mockito:mockito-all:1.9.5") 40 | } 41 | 42 | jar { 43 | from configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } 44 | archiveName "celos-ui.jar" 45 | } 46 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/BucketID.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | public class BucketID extends ValueObject implements Comparable { 19 | 20 | private final String id; 21 | 22 | public BucketID(String id) { 23 | this.id = Util.requireProperBucketIDorRegisterKey(id); 24 | } 25 | 26 | public String toString() { 27 | return id; 28 | } 29 | 30 | @Override 31 | public int compareTo(BucketID o) { 32 | Util.requireNonNull(o); 33 | return id.compareTo(o.id); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/defaults-oozie-props/test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | var CELOS_DEFAULT_OOZIE = "http://oozie:11000/oozie"; 17 | var CELOS_DEFAULT_HDFS = "file://"; 18 | var NAME_NODE = CELOS_DEFAULT_HDFS; 19 | var JOB_TRACKER = "jobtracker:8032"; 20 | var HIVE_METASTORE = "thrift://hive:9083"; 21 | 22 | var CELOS_DEFAULT_OOZIE_PROPERTIES = { 23 | "user.name": "default", 24 | "jobTracker" : "JOB_TRACKER", 25 | "nameNode" : "CELOS_DEFAULT_HDFS", 26 | "hiveMetastore": "HIVE_METASTORE", 27 | "hiveDefaults": "/hive/hive-site.xml", 28 | "oozie.use.system.libpath": "true" 29 | }; 30 | -------------------------------------------------------------------------------- /celos-ui/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | UIServlet 8 | com.collective.celos.old.UIServlet 9 | 10 | 11 | 12 | UIRerunServlet 13 | com.collective.celos.old.UIRerunServlet 14 | 15 | 16 | 17 | VersionServlet 18 | com.collective.celos.servlet.VersionServlet 19 | 20 | 21 | 22 | UIServlet 23 | /ui 24 | 25 | 26 | 27 | UIRerunServlet 28 | /rerun 29 | 30 | 31 | 32 | VersionServlet 33 | /version 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/RegisterKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | public class RegisterKey extends ValueObject implements Comparable { 19 | 20 | private final String keyString; 21 | 22 | public RegisterKey(String keyString) { 23 | this.keyString = Util.requireProperBucketIDorRegisterKey(keyString); 24 | } 25 | 26 | public String toString() { 27 | return keyString; 28 | } 29 | 30 | @Override 31 | public int compareTo(RegisterKey o) { 32 | Util.requireNonNull(o); 33 | return keyString.compareTo(o.keyString); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/PropertiesGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import com.fasterxml.jackson.databind.node.ObjectNode; 20 | 21 | /** 22 | * This one is tricky to explain. 23 | */ 24 | public interface PropertiesGenerator { 25 | 26 | public ObjectNode getProperties(SlotID id); 27 | 28 | public static final PropertiesGenerator EMPTY = new PropertiesGenerator() { 29 | @Override 30 | public ObjectNode getProperties(SlotID id) { 31 | return Util.MAPPER.createObjectNode(); 32 | } 33 | }; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/create/FixObjectCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.create; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.FixFsObject; 20 | import com.collective.celos.ci.testing.structure.fixobject.FixObject; 21 | 22 | import java.net.URISyntaxException; 23 | 24 | /** 25 | * Created by akonopko on 10/7/14. 26 | */ 27 | public interface FixObjectCreator { 28 | 29 | T create(TestRun testRun) throws Exception; 30 | 31 | String getDescription(TestRun testRun) throws Exception; 32 | } 33 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/NeverTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.collective.celos.database.StateDatabaseConnection; 19 | import com.collective.celos.trigger.Trigger; 20 | import com.collective.celos.trigger.TriggerStatus; 21 | 22 | /** 23 | * Utility trigger for testing that never signals data availability. 24 | */ 25 | public class NeverTrigger extends Trigger { 26 | 27 | @Override 28 | public TriggerStatus getTriggerStatus(StateDatabaseConnection connection, ScheduledTime now, ScheduledTime scheduledTime) throws Exception { 29 | return makeTriggerStatus(false, "Never ready"); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/servlet/ClearCacheServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.servlet; 17 | 18 | import javax.servlet.ServletException; 19 | import javax.servlet.http.HttpServletRequest; 20 | import javax.servlet.http.HttpServletResponse; 21 | 22 | /** 23 | * Posting to this servlet clears the scheduler cache. 24 | * 25 | * This is mostly for internal use during testing. 26 | */ 27 | @SuppressWarnings("serial") 28 | public class ClearCacheServlet extends AbstractServlet { 29 | 30 | protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException { 31 | clearSchedulerCache(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /samples/quickstart/src/main/celos/defaults.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | // Set this to your Hadoop username 17 | var CELOS_USER = "manuel"; 18 | // Set this to your Oozie API endpoint 19 | var CELOS_DEFAULT_OOZIE = "http://oozie003.ewr004.collective-media.net:11000/oozie"; 20 | // Set this to your HDFS name node 21 | var CELOS_DEFAULT_HDFS = "hdfs://nameservice1"; 22 | // Set this to your Hadoop job tracker 23 | var JOB_TRACKER = "cldmgr001.ewr004.collective-media.net:8032"; 24 | 25 | // No changes beyond this point should be required. 26 | 27 | var CELOS_DEFAULT_OOZIE_PROPERTIES = { 28 | "user.name": CELOS_USER, 29 | "jobTracker" : JOB_TRACKER, 30 | "nameNode" : CELOS_DEFAULT_HDFS 31 | }; 32 | -------------------------------------------------------------------------------- /samples/quickstart/src/main/celos/example-settings.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | // Set this to your Hadoop username 17 | var CELOS_USER = "manuel"; 18 | // Set this to your Oozie API endpoint 19 | var CELOS_DEFAULT_OOZIE = "http://oozie002.ny7.collective-media.net:11000/oozie"; 20 | // Set this to your HDFS name node 21 | var CELOS_DEFAULT_HDFS = "hdfs://nameservice1"; 22 | // Set this to your Hadoop job tracker 23 | var JOB_TRACKER = "cldmgr001.ewr004.collective-media.net:8032"; 24 | 25 | // No changes beyond this point should be required. 26 | 27 | var CELOS_DEFAULT_OOZIE_PROPERTIES = { 28 | "user.name": CELOS_USER, 29 | "jobTracker" : JOB_TRACKER, 30 | "nameNode" : CELOS_DEFAULT_HDFS 31 | }; 32 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/FixFsObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import com.collective.celos.ci.testing.structure.tree.TreeObject; 19 | 20 | /** 21 | * Created by akonopko on 10/7/14. 22 | */ 23 | public abstract class FixFsObject extends FixObject implements TreeObject { 24 | 25 | public abstract boolean isFile(); 26 | 27 | public boolean isDir() { 28 | return !isFile(); 29 | } 30 | 31 | public FixFile asFile() { 32 | return (FixFile) this; 33 | } 34 | 35 | public FixDir asDir() { 36 | return (FixDir) this; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/DatabaseName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.UUID; 19 | 20 | /** 21 | * Utility class for Celos CI that wraps a database name. 22 | */ 23 | public class DatabaseName extends ValueObject { 24 | 25 | private final static String AUGMENTED_DB_NAME = "celosci_%s_%s"; 26 | 27 | private final String databaseName; 28 | 29 | public DatabaseName(String databaseName) { 30 | this.databaseName = databaseName; 31 | } 32 | 33 | public String getMockedName(UUID testUuid) { 34 | return String.format(AUGMENTED_DB_NAME, databaseName, testUuid.toString().replace("-", "_")); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/FixDir.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import java.util.Collections; 19 | import java.util.Map; 20 | 21 | /** 22 | * Created by akonopko on 10/7/14. 23 | */ 24 | public class FixDir extends FixFsObject { 25 | 26 | private final Map content; 27 | 28 | public FixDir(Map content) { 29 | this.content = content; 30 | } 31 | 32 | public Map getChildren() { 33 | return Collections.unmodifiableMap(content); 34 | } 35 | 36 | @Override 37 | public boolean isFile() { 38 | return false; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/js-tests/test-registers.js: -------------------------------------------------------------------------------- 1 | importPackage(Packages.org.junit); 2 | 3 | var b1 = "b1-Iñtërnâtiônàlizætiøn"; 4 | var b2 = "b2-Iñtërnâtiônàlizætiøn"; 5 | var b3 = "b3-Iñtërnâtiônàlizætiøn"; 6 | var k1 = "k1-Iñtërnâtiônàlizætiøn"; 7 | var k2 = "k2-Iñtërnâtiônàlizætiøn"; 8 | var k3 = "k3-Iñtërnâtiônàlizætiøn"; 9 | var v1 = { foo: "bar-Iñtërnâtiônàlizætiøn" }; 10 | var v2 = { quux: "meh-Iñtërnâtiônàlizætiøn" }; 11 | var v3 = { bla: "baz-Iñtërnâtiônàlizætiøn" }; 12 | 13 | Assert.assertNull(celos.getRegister(b1, k3)); 14 | Assert.assertNull(celos.getRegister(b2, k1)); 15 | Assert.assertEquals(JSON.stringify({}), JSON.stringify(getRegistersAsMap(b3))); 16 | 17 | Assert.assertEquals(JSON.stringify(v1), JSON.stringify(celos.getRegister(b1, k1))); 18 | Assert.assertEquals(JSON.stringify(v2), JSON.stringify(celos.getRegister(b1, k2))); 19 | Assert.assertEquals(JSON.stringify(v3), JSON.stringify(celos.getRegister(b2, k3))); 20 | 21 | Assert.assertEquals(JSON.stringify({ "k1-Iñtërnâtiônàlizætiøn": v1, "k2-Iñtërnâtiônàlizætiøn": v2 }), JSON.stringify(getRegistersAsMap(b1))); 22 | Assert.assertEquals(JSON.stringify({ "k3-Iñtërnâtiônàlizætiøn": v3 }), JSON.stringify(getRegistersAsMap(b2))); 23 | 24 | function getRegistersAsMap(bucket) { 25 | var map = {}; 26 | celos.forEachRegister(bucket, function(k, v) { 27 | map[k] = v; 28 | }); 29 | return map; 30 | } -------------------------------------------------------------------------------- /scripts/playbooks/celos-purge.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | vars: 9 | root: ".." 10 | service_user: "{{ ansible_ssh_user }}" 11 | prefix: "/home/{{ service_user }}/local" 12 | service_dir: "{{ prefix }}/service/{{ service_name }}" 13 | start_script_path: "{{ prefix }}/bin/{{ service_name}}" 14 | dest_jar_path: "{{ prefix }}/lib/{{ service_name}}.jar" 15 | stdout_log_path: "{{ prefix }}/log/{{ service_name }}" 16 | 17 | tasks: 18 | 19 | - ping: 20 | 21 | - shell: /sbin/sv stop {{service_name}} 22 | 23 | - file: state=absent path="/etc/service/{{ service_name }}" 24 | 25 | - shell: "! curl -f http://{{inventory_hostname}}:{{service_port}}/version" 26 | 27 | - when: celos_defaults_dir is defined 28 | file: state=absent path={{ celos_defaults_dir }} 29 | - when: celos_workflows_dir is defined 30 | file: state=absent path={{ celos_workflows_dir }} 31 | - when: celos_db_dir is defined 32 | file: state=absent path={{ celos_db_dir }} 33 | 34 | - when: celos_wrk_dir is defined 35 | file: state=absent path={{ celos_wrk_dir }} 36 | 37 | - file: state=absent path={{ start_script_path }} 38 | - file: state=absent path={{ dest_jar_path }} 39 | - file: state=absent path={{ stdout_log_path }} 40 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/celos_deploy/workflow.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | importDefaults("collective"); 17 | 18 | celos.defineWorkflow({ 19 | "id": "file-copy", 20 | "maxRetryCount": 0, 21 | "schedule": celos.hourlySchedule(), 22 | "schedulingStrategy": celos.serialSchedulingStrategy(), 23 | "trigger": celos.hdfsCheckTrigger("/user/celos-ci/celos.selftest/input/file-copy/${year}-${month}-${day}T${hour}00.txt"), 24 | "externalService": celos.oozieExternalService({ 25 | "oozie.wf.application.path": "/user/celos-ci/celos.selftest/user/celos/app/file-copy/workflow.xml", 26 | "inputDir": "hdfs:/user/celos-ci/celos.selftest/input/file-copy", 27 | "outputDir": "hdfs:/user/celos-ci/celos.selftest/output/file-copy" 28 | }) 29 | 30 | }); 31 | -------------------------------------------------------------------------------- /scripts/test/test_files/file-copy/src/main/celos/workflow.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | importDefaults("collective"); 17 | 18 | celos.defineWorkflow({ 19 | "id": "file-copy", 20 | "maxRetryCount": 0, 21 | "schedule": celos.hourlySchedule(), 22 | "schedulingStrategy": celos.serialSchedulingStrategy(), 23 | "trigger": celos.hdfsCheckTrigger("/user/celos-ci/celos.selftest/input/file-copy/${year}-${month}-${day}T${hour}00.txt"), 24 | "externalService": celos.oozieExternalService({ 25 | "oozie.wf.application.path": "/user/celos-ci/celos.selftest/user/celos/app/file-copy/workflow.xml", 26 | "inputDir": "hdfs:/user/celos-ci/celos.selftest/input/file-copy", 27 | "outputDir": "hdfs:/user/celos-ci/celos.selftest/output/file-copy" 28 | }) 29 | 30 | }); 31 | -------------------------------------------------------------------------------- /scripts/celos-purge.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Playbook to perform integration testing on Celos 3 | # 4 | # Performs various tests against the servlets. 5 | 6 | - hosts: host 7 | gather_facts: no 8 | vars: 9 | root: ".." 10 | service_user: "{{ ansible_ssh_user }}" 11 | prefix: "/home/{{ service_user }}/local" 12 | service_dir: "{{ prefix }}/service/{{ service_name }}" 13 | runit_restart: "{{ root }}/scripts/runit-restart-service.sh" 14 | start_script_path: "{{ prefix }}/bin/{{ service_name}}" 15 | dest_jar_path: "{{ prefix }}/lib/{{ service_name}}.jar" 16 | stdout_log_path: "{{ prefix }}/log/{{ service_name }}" 17 | 18 | tasks: 19 | 20 | - ping: 21 | 22 | - shell: /sbin/sv stop {{service_name}} 23 | 24 | - file: state=absent path="/etc/service/{{ service_name }}" 25 | 26 | - shell: "! curl -f http://{{inventory_hostname}}:{{service_port}}/version" 27 | 28 | - when: celos_defaults_dir is defined 29 | file: state=absent path={{ celos_defaults_dir }} 30 | - when: celos_workflows_dir is defined 31 | file: state=absent path={{ celos_workflows_dir }} 32 | - when: celos_db_dir is defined 33 | file: state=absent path={{ celos_db_dir }} 34 | 35 | - when: celos_wrk_dir is defined 36 | file: state=absent path={{ celos_wrk_dir }} 37 | 38 | - file: state=absent path={{ start_script_path }} 39 | - file: state=absent path={{ dest_jar_path }} 40 | - file: state=absent path={{ stdout_log_path }} 41 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/trigger/AlwaysTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.trigger; 17 | 18 | import com.collective.celos.ScheduledTime; 19 | import com.collective.celos.database.StateDatabaseConnection; 20 | 21 | /** 22 | * Trivial trigger that always signals data availability, 23 | * for use when a workflow doesn't have any data dependencies 24 | * and simply needs to run at every scheduled time. 25 | */ 26 | public class AlwaysTrigger extends Trigger { 27 | 28 | public AlwaysTrigger() { 29 | } 30 | 31 | @Override 32 | public TriggerStatus getTriggerStatus(StateDatabaseConnection connection, ScheduledTime now, ScheduledTime scheduledTime) throws Exception { 33 | return makeTriggerStatus(true, "Always ready"); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/TrivialSchedulingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | /** 22 | * Trivial scheduling strategy that submits as many ready jobs as possible in parallel. 23 | */ 24 | public class TrivialSchedulingStrategy implements SchedulingStrategy { 25 | 26 | public TrivialSchedulingStrategy() { 27 | } 28 | 29 | public List getSchedulingCandidates(List states) { 30 | List candidates = new LinkedList(); 31 | for (SlotState slotState : states) { 32 | if (slotState.getStatus() == SlotState.Status.READY) { 33 | candidates.add(slotState); 34 | } 35 | } 36 | return candidates; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/FixFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import java.io.InputStream; 19 | import java.util.Collections; 20 | import java.util.Map; 21 | 22 | /** 23 | * Created by akonopko on 10/7/14. 24 | */ 25 | public class FixFile extends FixFsObject { 26 | 27 | private final InputStream inputStream; 28 | 29 | public FixFile(InputStream inputStream) { 30 | this.inputStream = inputStream; 31 | } 32 | 33 | public InputStream getContent() { 34 | return inputStream; 35 | } 36 | 37 | @Override 38 | public Map getChildren() { 39 | return Collections.EMPTY_MAP; 40 | } 41 | 42 | @Override 43 | public boolean isFile() { 44 | return true; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /celos-ci/src/test/java/com/collective/celos/ci/testing/fixtures/create/StringFixObjectCreatorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.create; 17 | 18 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 19 | import junit.framework.Assert; 20 | import org.apache.commons.io.IOUtils; 21 | import org.junit.Test; 22 | 23 | /** 24 | * Created by akonopko on 10/7/14. 25 | */ 26 | public class StringFixObjectCreatorTest { 27 | 28 | public static final String SOME_TEXT = "some test"; 29 | 30 | @Test 31 | public void testStringFixObjectCreator() throws Exception { 32 | FixFileFromStringCreator creator = new FixFileFromStringCreator(SOME_TEXT); 33 | FixFile fixFile = creator.create(null); 34 | Assert.assertEquals(SOME_TEXT, IOUtils.toString(fixFile.getContent())); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /celos-ci/src/test/java/com/collective/celos/ci/testing/fixtures/convert/UpperCaseStringFixFileConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.convert; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.AbstractFixObjectConverter; 20 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 21 | import org.apache.commons.io.IOUtils; 22 | 23 | import java.io.IOException; 24 | 25 | public class UpperCaseStringFixFileConverter extends AbstractFixObjectConverter { 26 | 27 | @Override 28 | public FixFile convert(TestRun tr, FixFile ff) throws IOException { 29 | String newContent = IOUtils.toString(ff.getContent()).toUpperCase(); 30 | return new FixFile(IOUtils.toInputStream(newContent)); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/ValueObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import org.apache.commons.lang.builder.EqualsBuilder; 19 | import org.apache.commons.lang.builder.HashCodeBuilder; 20 | import org.apache.commons.lang.builder.ToStringBuilder; 21 | import org.apache.commons.lang.builder.ToStringStyle; 22 | 23 | /** 24 | * Provides automatic equals, hashCode, and toString for simple objects. 25 | */ 26 | public abstract class ValueObject { 27 | 28 | public boolean equals(Object other) { 29 | return EqualsBuilder.reflectionEquals(this, other); 30 | } 31 | 32 | public int hashCode() { 33 | return HashCodeBuilder.reflectionHashCode(this); 34 | } 35 | 36 | public String toString() { 37 | return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /celos-ui/src/main/webapp/static/script.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | var celos = { 17 | load: function() { 18 | $("a.slotLink").click(function(event) { 19 | if (event.shiftKey) { 20 | event.preventDefault(); 21 | var slotID = $(event.target).data("slot-id"); 22 | $.ajax({ 23 | type: "POST", 24 | url: "rerun?id=" + encodeURIComponent(slotID), 25 | success: function() { $(event.target).html("rrun"); }, 26 | error: function(jqXHR, textStatus, errorThrown) { 27 | alert("Error when rerunning slot " + slotID + ": " + textStatus + " (check console)"); 28 | console.log(errorThrown); 29 | } 30 | }); 31 | } 32 | }); 33 | } 34 | } 35 | $(document).ready(celos.load); 36 | -------------------------------------------------------------------------------- /scripts/runit-restart-service.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | SV_TIMEOUT=60 5 | 6 | while [[ $# > 1 ]] 7 | do 8 | case $1 in 9 | --SERVICE_NAME) 10 | shift # past argument 11 | SERVICE_NAME="$1" 12 | ;; 13 | --SERVICE_DIR) 14 | shift # past argument 15 | SERVICE_DIR="$1" 16 | ;; 17 | *) 18 | echo $@ 19 | echo usage: SERVICE_NAME ... SERVICE_DIR ... 20 | exit 1 # unknown option 21 | ;; 22 | esac 23 | shift # past argument 24 | done 25 | 26 | [[ -z ${SERVICE_NAME} ]] && echo pls specify SERVICE_NAME && exit 1 27 | [[ -z ${SERVICE_DIR} ]] && echo pls specify SERVICE_DIR && exit 1 28 | 29 | set -x 30 | 31 | if [ -e /sbin/sv ] 32 | then 33 | SV=/sbin/sv 34 | else 35 | SV=sv 36 | fi 37 | 38 | ln -sf ${SERVICE_DIR} /etc/service/ 39 | chmod a+w "/etc/service/${SERVICE_NAME}" 40 | # check runsv is running 41 | if ${SV} status ${SERVICE_NAME} &> /dev/null 42 | then 43 | ${SV} -w ${SV_TIMEOUT} restart ${SERVICE_NAME}/log 44 | ${SV} -w ${SV_TIMEOUT} restart ${SERVICE_NAME} 45 | else 46 | # runsv starts new service with delay, 47 | # so this needs to fix 'fail: $SERVICE_NAME: runsv not running' 48 | i=0 49 | while (( i <= ${SV_TIMEOUT} )) && ! ${SV} status ${SERVICE_NAME} &> /dev/null 50 | do 51 | : runsv not running, wait one more second 52 | (( i += 1 )) 53 | sleep 1 54 | done 55 | sleep 1 56 | ${SV} -w ${SV_TIMEOUT} start ${SERVICE_NAME}/log 57 | ${SV} -w ${SV_TIMEOUT} start ${SERVICE_NAME} 58 | fi 59 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/WorkflowID.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Uniquely identifies a workflow. 20 | */ 21 | public class WorkflowID extends ValueObject implements Comparable { 22 | 23 | protected final String id; 24 | 25 | public WorkflowID(String id) { 26 | this.id = Util.requireNonNull(id); 27 | if (id.trim().equals("")) { 28 | throw new IllegalArgumentException("Workflow ID can't be only whitespace."); 29 | } 30 | if (id.contains("/")) { 31 | throw new IllegalArgumentException("Workflow ID can't contain slash."); 32 | } 33 | } 34 | 35 | public String toString() { 36 | return id; 37 | } 38 | 39 | @Override 40 | public int compareTo(WorkflowID o) { 41 | return this.id.compareTo(o.id); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /celos-ui/src/main/java/com/collective/celos/ui/WorkflowGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ui; 17 | 18 | import java.util.Collections; 19 | import java.util.List; 20 | 21 | import com.collective.celos.Util; 22 | import com.collective.celos.WorkflowID; 23 | 24 | /** 25 | * A named group of workflows, for rendering the workflows list. 26 | */ 27 | public class WorkflowGroup { 28 | 29 | private final String name; 30 | private final List workflows; 31 | 32 | public WorkflowGroup(String name, List workflows) { 33 | this.name = Util.requireNonNull(name); 34 | this.workflows = Collections.unmodifiableList(Util.requireNonNull(workflows)); 35 | } 36 | 37 | public String getName() { 38 | return name; 39 | } 40 | 41 | public List getWorkflows() { 42 | return workflows; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/create/FixFileFromStringCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.create; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 20 | import org.apache.commons.io.IOUtils; 21 | 22 | /** 23 | * Created by akonopko on 10/7/14. 24 | */ 25 | public class FixFileFromStringCreator implements FixObjectCreator { 26 | 27 | private final String content; 28 | 29 | public FixFileFromStringCreator(String content) { 30 | this.content = content; 31 | } 32 | 33 | public FixFile create(TestRun testRun) throws Exception { 34 | return new FixFile(IOUtils.toInputStream(content)); 35 | } 36 | 37 | @Override 38 | public String getDescription(TestRun testRun) { 39 | return "String fixture"; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/WorkflowStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * Bean returned by HTTP API that contains information about a workflow and a subset of its slots. 22 | */ 23 | public class WorkflowStatus { 24 | 25 | private final WorkflowInfo info; 26 | private final List slotStates; 27 | private final boolean paused; 28 | 29 | public WorkflowStatus(WorkflowInfo info, List slotStates, boolean paused) { 30 | this.slotStates = slotStates; 31 | this.info = info; 32 | this.paused = paused; 33 | } 34 | 35 | public WorkflowInfo getInfo() { 36 | return info; 37 | } 38 | 39 | public List getSlotStates() { 40 | return slotStates; 41 | } 42 | 43 | public boolean isPaused() { 44 | return paused; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/Schedule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.SortedSet; 19 | 20 | /** 21 | * Determines the times at which a workflow is supposed to run. 22 | */ 23 | public interface Schedule { 24 | 25 | /** 26 | * Returns the set of scheduled times of this schedule within the 27 | * window defined by start (inclusive) and end (exclusive). 28 | * 29 | * The scheduler parameter is currently only used by DependentSchedule, so tests 30 | * of other schedules pass null as scheduler, which is a bit of a hack, but works. 31 | */ 32 | public SortedSet getScheduledTimes(Scheduler scheduler, ScheduledTime start, ScheduledTime end); 33 | 34 | default boolean isTimeInSchedule(ScheduledTime time, Scheduler scheduler) { 35 | return getScheduledTimes(scheduler, time, time.plusSeconds(1)).contains(time); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/avro/avro.schema: -------------------------------------------------------------------------------- 1 | {"type":"record","name":"ConsolidatedEvent","namespace":"com.collective.pythia.avro","fields":[{"name":"visitor","type":["null",{"type":"record","name":"Visitor","fields":[{"name":"cookie_id","type":"string"},{"name":"segments","type":{"type":"array","items":{"type":"record","name":"Segment","fields":[{"name":"id","type":"int"},{"name":"expiration","type":"long","default":0}]}},"default":[]},{"name":"edges","type":{"type":"map","values":"long"},"default":{}},{"name":"behaviors","type":{"type":"map","values":{"type":"map","values":"int"},"default":{}},"doc":"Map of net.context to map of YYYYMMDD->number_of_hits","default":{}},{"name":"birthdate","type":"long"},{"name":"association_ids","type":{"type":"map","values":"string"},"default":{}}]}],"default":null},{"name":"events","type":{"type":"array","items":{"type":"record","name":"ProfileEvent","fields":[{"name":"cookie_id","type":"string"},{"name":"tstamp","type":"long"},{"name":"edge","type":"string"},{"name":"changes","type":[{"type":"record","name":"Hit","fields":[{"name":"daystamp","type":"string"},{"name":"context","type":"string"},{"name":"type","type":{"type":"enum","name":"HitType","symbols":["ADX","RETARGET"]}},{"name":"count","type":"int","default":1}]},{"type":"record","name":"Command","fields":[{"name":"operation","type":{"type":"enum","name":"OperationType","symbols":["ADD","REPLACE","UPDATE","REMOVE"]}},{"name":"association_id","type":["null","string"],"default":null},{"name":"network","type":"string"},{"name":"segments","type":{"type":"array","items":"int"},"default":[]}]}]}]}},"default":[]}]} -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Miscellaneous constants. 20 | */ 21 | public class Constants { 22 | 23 | public static final String DEFAULT_WORKFLOWS_DIR = "/etc/celos/workflows"; 24 | public static final String DEFAULT_DEFAULTS_DIR = "/etc/celos/defaults"; 25 | public static final String DEFAULT_DB_DIR = "/var/lib/celos/db"; 26 | public static final String DEFAULT_LOG_DIR = "/var/log/celos"; 27 | public static final String DEFAULT_HDFS_ROOT = "/user/celos/app"; 28 | 29 | public static final String WORKFLOW_CONFIGURATION_PATH_ATTR = "workflow.configuration.path"; 30 | public static final String DEFAULTS_CONFIGURATION_PATH_ATTR = "defaults.configuration.path"; 31 | public static final String ADDITIONAL_JS_VARIABLES = "additional.js.variables"; 32 | public static final String DATABASE = "database"; 33 | 34 | public static final Integer SECOND_MS = 1000; 35 | } 36 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/OnDateTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.collective.celos.database.StateDatabaseConnection; 19 | import com.collective.celos.trigger.Trigger; 20 | import com.collective.celos.trigger.TriggerStatus; 21 | 22 | import java.util.Collections; 23 | 24 | public class OnDateTrigger extends Trigger { 25 | 26 | private final ScheduledTime scheduledTime; 27 | 28 | public OnDateTrigger(String scheduledTime) { 29 | this(new ScheduledTime(scheduledTime)); 30 | } 31 | 32 | public OnDateTrigger(ScheduledTime scheduledTime) { 33 | this.scheduledTime = scheduledTime; 34 | } 35 | 36 | 37 | @Override 38 | public TriggerStatus getTriggerStatus(StateDatabaseConnection connection, ScheduledTime now, ScheduledTime time) throws Exception { 39 | return makeTriggerStatus(time.equals(scheduledTime), "unused", Collections.emptyList()); 40 | } 41 | 42 | 43 | } 44 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/WorkflowConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.Collection; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * The set of all workflows loaded into the scheduler. 24 | */ 25 | public class WorkflowConfiguration { 26 | 27 | private final Map workflows = new HashMap<>(); 28 | 29 | public Collection getWorkflows() { 30 | return workflows.values(); 31 | } 32 | 33 | public Workflow findWorkflow(WorkflowID id) { 34 | return workflows.get(Util.requireNonNull(id)); 35 | } 36 | 37 | public void addWorkflow(Workflow wf) { 38 | Util.requireNonNull(wf); 39 | WorkflowID id = wf.getID(); 40 | if (findWorkflow(id) != null) { 41 | throw new IllegalArgumentException("Workflow with this ID already exists: " + id); 42 | } 43 | workflows.put(id, wf); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /scripts/server-and-ui-action.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -x 3 | set -e 4 | 5 | [[ -z ${CELOS_USER} ]] && echo pls specify CELOS_USER && exit 1 6 | [[ -z ${GIT_COMMIT} ]] && echo pls specify GIT_COMMIT && exit 1 7 | if [[ -f ${INVENTORY_SERVER} ]] 8 | then true 9 | else echo "pls specify INVENTORY_SERVER and make sure INVENTORY_SERVER exists" && exit 1 10 | fi 11 | if [[ -f ${INVENTORY_UI} ]] 12 | then true 13 | else echo "pls specify INVENTORY_UI and make sure INVENTORY_UI exists" && exit 1 14 | fi 15 | 16 | CELOS_ACTION=$1 17 | 18 | [[ -z ${CELOS_ACTION} ]] && echo 'usage ./scripts/server-and-ui-action.sh CELOS_ACTION (deploy, check, purge, start, stop)' && exit 1 19 | if [ $CELOS_ACTION = "deploy" ] || 20 | [ $CELOS_ACTION = "check" ] || 21 | [ $CELOS_ACTION = "purge" ] || 22 | [ $CELOS_ACTION = "start" ] || 23 | [ $CELOS_ACTION = "stop" ] 24 | then true 25 | else echo 'usage ./scripts/server-and-ui-action.sh CELOS_ACTION (deploy, check, purge, start, stop)' && exit 1 26 | fi 27 | 28 | export ANSIBLE_SSH_ARGS="" 29 | ansible-playbook "scripts/playbooks/celos-${CELOS_ACTION}.yaml" -c ssh -u ${CELOS_USER} -i ${INVENTORY_SERVER} -e "@scripts/params/common-server.json" -e service_version=${GIT_COMMIT} 30 | # parse server host and port from inventory file 31 | TMP1="$(head -2 ${INVENTORY_SERVER} | tail -1)" 32 | TMP2="$(grep service_port ${INVENTORY_SERVER} | cut -d '=' -f 2)" 33 | SERVER_URL="http://${TMP1}:${TMP2}" 34 | 35 | ansible-playbook "scripts/playbooks/celos-${CELOS_ACTION}.yaml" -c ssh -u ${CELOS_USER} -i ${INVENTORY_UI} -e "@scripts/params/common-ui.json" -e service_discovery=${SERVER_URL} -e service_version=${GIT_COMMIT} 36 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/SlotStateStatusPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.Arrays; 19 | import java.util.HashSet; 20 | import java.util.Set; 21 | 22 | import org.apache.commons.collections.Predicate; 23 | 24 | import com.collective.celos.SlotState.Status; 25 | 26 | /** 27 | * Ivor who's a Scala wonk did this. 28 | */ 29 | public class SlotStateStatusPredicate implements Predicate { 30 | 31 | private Set targetValues; 32 | 33 | public SlotStateStatusPredicate(Status... targetValues) { 34 | this.targetValues = new HashSet(Arrays.asList(targetValues)); 35 | if (this.targetValues.isEmpty()) { 36 | throw new IllegalArgumentException("please specify some status values"); 37 | } 38 | } 39 | 40 | @Override 41 | public boolean evaluate(Object object) { 42 | SlotState state = (SlotState) object; 43 | return targetValues.contains(state.status); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /celos-ui/src/main/java/com/collective/celos/ui/UICommandLine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ui; 17 | 18 | import com.collective.celos.Util; 19 | import java.io.File; 20 | import java.net.URI; 21 | import java.net.URL; 22 | 23 | /** 24 | * UI command-line options. 25 | */ 26 | public class UICommandLine { 27 | 28 | private final URL celosUrl; 29 | private final URL hueUrl; 30 | private final int port; 31 | private final File configFile; 32 | 33 | public UICommandLine(URL celosUrl, URL hueURL, int port, File configFile) { 34 | this.celosUrl = Util.requireNonNull(celosUrl); 35 | this.configFile = configFile; 36 | this.hueUrl = hueURL; 37 | this.port = port; 38 | } 39 | 40 | public URL getCelosUrl() { 41 | return celosUrl; 42 | } 43 | 44 | public int getPort() { 45 | return port; 46 | } 47 | 48 | public URL getHueUrl() { 49 | return hueUrl; 50 | } 51 | 52 | public File getConfigFile() { 53 | return configFile; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/compare/PlainFileComparer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.compare; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 20 | import org.apache.commons.io.IOUtils; 21 | 22 | import java.io.InputStream; 23 | 24 | /** 25 | * Created by akonopko on 10/7/14. 26 | */ 27 | public class PlainFileComparer implements FixtureComparer { 28 | 29 | private final InputStream content; 30 | private final FixFile file; 31 | 32 | public PlainFileComparer(InputStream content, FixFile file) { 33 | this.content = content; 34 | this.file = file; 35 | } 36 | 37 | public FixObjectCompareResult check(TestRun testRun) throws Exception { 38 | if (!IOUtils.contentEquals(content, file.getContent())) { 39 | return FixObjectCompareResult.failed("File contents differed"); 40 | } 41 | return FixObjectCompareResult.SUCCESS; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /celos-common/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven' 2 | 3 | version = '0.1' 4 | 5 | dependencies { 6 | compile("org.eclipse.jetty.aggregate:jetty-all:9.2.2.v20140723") 7 | compile("javax.servlet:javax.servlet-api:3.1.0") 8 | 9 | compile("com.google.guava:guava:11.0.2") 10 | compile("commons-lang:commons-lang:2.6") 11 | compile("commons-io:commons-io:2.4") 12 | compile("commons-cli:commons-cli:1.2") 13 | compile("joda-time:joda-time:2.3") 14 | compile('org.apache.httpcomponents:httpclient:4.2.5') 15 | 16 | compile("com.fasterxml.jackson.core:jackson-core:2.3.0") 17 | compile("com.fasterxml.jackson.core:jackson-databind:2.3.0") 18 | compile("com.fasterxml.jackson.core:jackson-annotations:2.3.0") 19 | compile("org.mozilla:rhino:1.7R4") 20 | compile("log4j:apache-log4j-extras:1.2.17") 21 | compile("log4j:log4j:1.2.17") 22 | } 23 | 24 | jar { 25 | from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }} 26 | baseName "celos-common" 27 | archiveName "celos-common.jar" 28 | } 29 | 30 | def repoUser = System.getenv("CELOS_NEXUS_USER"); 31 | def repoPw = System.getenv("CELOS_NEXUS_PASS"); 32 | 33 | uploadArchives.doFirst { 34 | repositories { 35 | mavenDeployer { 36 | repository(url: System.getenv("NEXUS_MAVEN_REPO")) { 37 | authentication(userName: repoUser, password: repoPw) 38 | } 39 | if (!System.getenv("CELOS_BUILD_NUMBER")) throw new GradleException("CELOS_BUILD_NUMBER env variable should be present") 40 | pom.version = "2.0." + System.getenv("CELOS_BUILD_NUMBER") 41 | pom.groupId = "com.collective" 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/NotTriggerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.collective.celos.database.StateDatabaseConnection; 19 | import com.collective.celos.trigger.NotTrigger; 20 | import junit.framework.Assert; 21 | 22 | import org.junit.Test; 23 | 24 | import static org.mockito.Mockito.mock; 25 | 26 | public class NotTriggerTest { 27 | 28 | @Test 29 | public void invertsTrue() throws Exception { 30 | StateDatabaseConnection connection = mock(StateDatabaseConnection.class); 31 | Assert.assertFalse(new NotTrigger(AndTriggerTest.createAlwaysTrigger()).isDataAvailable(connection, ScheduledTime.now(), ScheduledTime.now())); 32 | } 33 | 34 | @Test 35 | public void invertsFalse() throws Exception { 36 | StateDatabaseConnection connection = mock(StateDatabaseConnection.class); 37 | Assert.assertTrue(new NotTrigger(AndTriggerTest.createNeverTrigger()).isDataAvailable(connection, ScheduledTime.now(), ScheduledTime.now())); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/OozieExternalStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import static org.apache.oozie.client.WorkflowJob.Status.PREP; 19 | import static org.apache.oozie.client.WorkflowJob.Status.RUNNING; 20 | import static org.apache.oozie.client.WorkflowJob.Status.SUCCEEDED; 21 | 22 | import org.apache.oozie.client.WorkflowJob.Status; 23 | 24 | /** 25 | * Status of Oozie execution of a workflow. 26 | */ 27 | public class OozieExternalStatus implements ExternalStatus { 28 | 29 | private Status status; 30 | 31 | public OozieExternalStatus(String statusString) { 32 | this.status = Status.valueOf(statusString); 33 | if (status == null) { 34 | throw new IllegalArgumentException("Invalid status string: '" 35 | + statusString + "'"); 36 | } 37 | } 38 | 39 | public boolean isRunning() { 40 | return status == RUNNING || status == PREP; 41 | } 42 | 43 | public boolean isSuccess() { 44 | return status == SUCCEEDED; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /celos-server/src/test/resources/com/collective/celos/workflow-configuration-test/properties-test/workflow-1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | importPackage(Packages.com.collective.celos); 17 | 18 | celos.defineWorkflow({ 19 | "id": "workflow-1", 20 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 21 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 22 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 23 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 24 | "maxRetryCount": 55 25 | }); 26 | 27 | celos.defineWorkflow({ 28 | "id": "workflow-2", 29 | "schedule": new WorkflowConfigurationParserTest$TestSchedule(), 30 | "schedulingStrategy": new WorkflowConfigurationParserTest$TestSchedulingStrategy(), 31 | "trigger": new WorkflowConfigurationParserTest$TestTrigger(), 32 | "externalService": new WorkflowConfigurationParserTest$TestExternalService(), 33 | "maxRetryCount": 66, 34 | "startTime": "2014-03-10T12:34:56.789Z", 35 | "waitTimeoutSeconds": 23 36 | }); 37 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/fixtures/jsoncompare/1/avro3.json: -------------------------------------------------------------------------------- 1 | {"visitor":{"com.collective.pythia.avro.Visitor":{"cookie_id":"133263e9e100000","segments":[],"edges":{},"behaviors":{},"birthdate":0,"association_ids":{}}},"events":[{"cookie_id":"133263e9e100000","tstamp":1403721385042,"edge":"batchimport","changes":{"com.collective.pythia.avro.Command":{"operation":"REMOVE","association_id":null,"network":"et","segments":[49118]}}},{"cookie_id":"133263e9e100000","tstamp":1403721385042,"edge":"batchimport","changes":{"com.collective.pythia.avro.Command":{"operation":"ADD","association_id":null,"network":"et","segments":[49117]}}}]} 2 | {"visitor":{"com.collective.pythia.avro.Visitor":{"cookie_id":"134adb391b00000","segments":[],"edges":{},"behaviors":{},"birthdate":0,"association_ids":{}}},"events":[{"cookie_id":"134adb391b00000","tstamp":1403721376988,"edge":"batchimport","changes":{"com.collective.pythia.avro.Command":{"operation":"ADD","association_id":null,"network":"et","segments":[49118]}}}]} 3 | {"visitor":{"com.collective.pythia.avro.Visitor":{"cookie_id":"1317beb84b00000","segments":[],"edges":{},"behaviors":{},"birthdate":0,"association_ids":{}}},"events":[{"cookie_id":"1317beb84b00000","tstamp":1403721380452,"edge":"batchimport","changes":{"com.collective.pythia.avro.Command":{"operation":"ADD","association_id":null,"network":"et","segments":[49118]}}}]} 4 | {"visitor":{"com.collective.pythia.avro.Visitor":{"cookie_id":"12b811f59080000","segments":[],"edges":{},"behaviors":{},"birthdate":0,"association_ids":{}}},"events":[{"cookie_id":"12b811f59080000","tstamp":1403721375367,"edge":"batchimport","changes":{"com.collective.pythia.avro.Command":{"operation":"ADD","association_id":null,"network":"et","segments":[49118]}}}]} -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/servlet/VersionServlet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.servlet; 17 | 18 | import javax.servlet.ServletException; 19 | import javax.servlet.http.HttpServlet; 20 | import javax.servlet.http.HttpServletRequest; 21 | import javax.servlet.http.HttpServletResponse; 22 | 23 | /** 24 | * This servlet is used in production to verify that the running instance 25 | * was started from the latest Git commit. 26 | */ 27 | @SuppressWarnings("serial") 28 | public class VersionServlet extends HttpServlet { 29 | 30 | protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException { 31 | try { 32 | final String serviceVersion = System.getenv("CELOS_VERSION"); 33 | if (serviceVersion != null) { 34 | res.getOutputStream().write(serviceVersion.getBytes()); 35 | } else { 36 | throw new Exception("CELOS_VERSION is undefined"); 37 | } 38 | } catch (Exception e) { 39 | throw new ServletException(e); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/tree/TreeObjectProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.tree; 17 | 18 | import java.io.IOException; 19 | import java.nio.file.Path; 20 | import java.nio.file.Paths; 21 | import java.util.Map; 22 | 23 | /** 24 | * Created by akonopko on 10/7/14. 25 | */ 26 | public abstract class TreeObjectProcessor { 27 | 28 | public static void process(TreeObject object, TreeObjectProcessor processor) throws IOException { 29 | process(Paths.get(""), object, processor); 30 | } 31 | 32 | private static void process(Path path, TreeObject object, TreeObjectProcessor processor) throws IOException { 33 | processor.process(path, object); 34 | Map map = object.getChildren(); 35 | for(Map.Entry entry : map.entrySet()) { 36 | process(Paths.get(path.toString(), entry.getKey()), entry.getValue(), processor); 37 | } 38 | } 39 | 40 | 41 | public abstract void process(Path path, T ff) throws IOException; 42 | 43 | } 44 | -------------------------------------------------------------------------------- /celos-ci/src/test/resources/com/collective/celos/ci/testing/config/test.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | ci.addTestCase({ 17 | name: "wordcount test case 1", 18 | sampleTimeStart: "2013-11-20T11:00Z", 19 | sampleTimeEnd: "2013-11-20T18:00Z", 20 | inputs: [ 21 | ci.hdfsInput(ci.fixDirFromResource("src/test/celos-ci/test-1/input/plain/input/wordcount1"), "input/wordcount1"), 22 | ci.hdfsInput(ci.fixDirFromResource("src/test/celos-ci/test-1/input/plain/input/wordcount11"), "input/wordcount11") 23 | ], 24 | outputs: [ 25 | ci.plainCompare(ci.fixDirFromResource("src/test/celos-ci/test-1/output/plain/output/wordcount1"), "output/wordcount1") 26 | ] 27 | }) 28 | 29 | ci.addTestCase({ 30 | name: "wordcount test case 2", 31 | sampleTimeStart: "2013-12-20T16:00Z", 32 | sampleTimeEnd: "2013-12-20T18:00Z", 33 | inputs: [ 34 | ci.hdfsInput(ci.fixDirFromResource("src/test/celos-ci/test-1/input/plain/input/wordcount2"), "input/wordcount2") 35 | ], 36 | outputs: [ 37 | ci.plainCompare(ci.fixDirFromResource("src/test/celos-ci/test-1/output/plain/output/wordcount2"), "output/wordcount2") 38 | ] 39 | }); -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/convert/FixTableToTSVFileConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.convert; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | import org.apache.commons.io.IOUtils; 22 | import org.apache.commons.lang.StringUtils; 23 | 24 | import com.collective.celos.ci.mode.test.TestRun; 25 | import com.collective.celos.ci.testing.structure.fixobject.AbstractFixObjectConverter; 26 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 27 | import com.collective.celos.ci.testing.structure.fixobject.FixTable; 28 | 29 | public class FixTableToTSVFileConverter extends AbstractFixObjectConverter { 30 | 31 | @Override 32 | public FixFile convert(TestRun tr, FixTable table) throws Exception { 33 | List lines = new ArrayList<>(); 34 | for (FixTable.FixRow row : table.getRows()) { 35 | lines.add(StringUtils.join(row.getOrderedColumns(table.getColumnNames()), "\t")); 36 | } 37 | return new FixFile(IOUtils.toInputStream(StringUtils.join(lines, "\n") + "\n")); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/ExternalService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * Execution engine for workflows. 20 | */ 21 | public interface ExternalService { 22 | 23 | /** 24 | * Submits slot to external service. 25 | * 26 | * Returns external ID of submitted workflow, or throws an exception. 27 | */ 28 | public String submit(SlotID id) throws ExternalServiceException; 29 | 30 | /** 31 | * Starts workflow with the given external ID. 32 | * 33 | * The slot ID is the same that previously passed to submit. 34 | */ 35 | public void start(SlotID id, String externalID) throws ExternalServiceException; 36 | 37 | /** 38 | * Gets the status of the externally running workflow with the given ID. 39 | */ 40 | public ExternalStatus getStatus(SlotID id, String externalID) throws ExternalServiceException; 41 | 42 | /** 43 | * Kills workflow with the given external ID. 44 | * 45 | * The slot ID is the same that previously passed to start. 46 | */ 47 | public void kill(SlotID id, String externalID) throws ExternalServiceException; 48 | } -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/servlet/JSONWorkflowListServletTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.servlet; 17 | 18 | import com.collective.celos.Util; 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | import com.collective.celos.WorkflowConfiguration; 23 | import com.collective.celos.WorkflowConfigurationParserTest; 24 | import com.fasterxml.jackson.databind.ObjectMapper; 25 | import com.fasterxml.jackson.databind.node.ArrayNode; 26 | import com.fasterxml.jackson.databind.node.ObjectNode; 27 | 28 | public class JSONWorkflowListServletTest { 29 | 30 | @Test 31 | public void jsonCorrectlyProduced() throws Exception { 32 | WorkflowConfiguration cfg = 33 | WorkflowConfigurationParserTest.parseDir("json-workflow-list-servlet-test"); 34 | ArrayNode list = Util.MAPPER.createArrayNode(); 35 | list.add(new String("workflow-1")); 36 | list.add(new String("workflow-2")); 37 | ObjectNode obj = Util.MAPPER.createObjectNode(); 38 | obj.put("ids", list); 39 | Assert.assertEquals(obj, new JSONWorkflowListServlet().createJSONObject(cfg)); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/DependentSchedule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.SortedSet; 19 | 20 | /** 21 | * Makes a workflow use the same schedule as another workflow. 22 | */ 23 | public class DependentSchedule implements Schedule { 24 | 25 | private final WorkflowID otherWorkflowID; 26 | 27 | public DependentSchedule(WorkflowID otherWorkflowID) { 28 | this.otherWorkflowID = Util.requireNonNull(otherWorkflowID); 29 | } 30 | 31 | @Override 32 | public SortedSet getScheduledTimes(Scheduler scheduler, ScheduledTime start, ScheduledTime end) { 33 | Workflow otherWorkflow = scheduler.getWorkflowConfiguration().findWorkflow(getOtherWorkflowID()); 34 | if (otherWorkflow == null) { 35 | throw new RuntimeException("Workflow " + getOtherWorkflowID() + " doesn't exist."); 36 | } 37 | Schedule otherSchedule = otherWorkflow.getSchedule(); 38 | return otherSchedule.getScheduledTimes(scheduler, start, end); 39 | } 40 | 41 | public WorkflowID getOtherWorkflowID() { 42 | return otherWorkflowID; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/trigger/TriggerStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.trigger; 17 | 18 | import com.collective.celos.Util; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * Human-readable information about the status of a trigger. 24 | */ 25 | public final class TriggerStatus { 26 | 27 | private final String type; 28 | private final boolean ready; 29 | private final String description; 30 | private final List subStatuses; 31 | 32 | public TriggerStatus(String triggerClassName, boolean ready, String description, List subStatuses) { 33 | this.type = Util.requireNonNull(triggerClassName); 34 | this.ready = ready; 35 | this.description = Util.requireNonNull(description); 36 | this.subStatuses = Util.requireNonNull(subStatuses); 37 | } 38 | 39 | public String getType() { 40 | return type; 41 | } 42 | 43 | public boolean isReady() { 44 | return ready; 45 | } 46 | 47 | public String getDescription() { 48 | return description; 49 | } 50 | 51 | public List getSubStatuses() { 52 | return subStatuses; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /scripts/push-javadoc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # get sources path 4 | DIR="." 5 | INDEXES="com.collective.celos.ci.config.deploy com.collective.celos.ci.testing.structure.fixobject com.collective.celos.ui com.collective.celos.ci.testing.fixtures.deploy com.collective.celos.ci.testing.structure.tree com.collective.celos.ci.mode.test com.collective.celos.ci.config com.collective.celos.trigger com.collective.celos com.collective.celos.ci.mode com.collective.celos.ci.testing.fixtures.create com.collective.celos.ci.deploy com.collective.celos.ci com.collective.celos.ci.testing.fixtures.compare.json com.collective.celos.ci.testing.tree com.collective.celos.ci.testing.fixtures.deploy.hive com.collective.celos.server com.collective.celos.ci.testing.fixtures.compare com.collective.celos.ci.testing.fixtures.convert com.collective.celos.servlet com.collective.celos.trigger" 6 | SOURCES="${DIR}/celos-common/src/main/java:${DIR}/celos-ui/src/main/java:${DIR}/celos-ui/src/main/webapp:${DIR}/celos-server/src/main/java:${DIR}/celos-server/src/main/resources:${DIR}/celos-server/src/main/webapp:${DIR}/celos-ci/src/main/java:${DIR}/celos-ci/src/main/resources" 7 | JAVADOC=${JAVA_HOME:-"/usr/java/latest"}/bin/javadoc 8 | 9 | set -e 10 | set -x 11 | 12 | [ $(git rev-parse --abbrev-ref HEAD) == "gh-pages" ] && exit 1 13 | git clean -fd 14 | rm -rf ./javadoc 15 | rm -rf ./javadoc-tmp 16 | ${JAVADOC} -notimestamp -encoding UTF8 -protected -splitindex ${INDEXES} -sourcepath ${SOURCES} -d ${DIR}/javadoc 17 | mv ./javadoc ./javadoc-tmp 18 | git branch -D gh-pages || : 19 | git checkout gh-pages 20 | git rm -rf --quiet ${DIR}/javadoc 21 | mv ./javadoc-tmp ./javadoc 22 | git add ./javadoc 23 | # check current branch name 24 | [ $(git rev-parse --abbrev-ref HEAD) == "gh-pages" ] || exit 1 25 | git commit -m "Published documentation to [gh-pages]." || : 26 | git push origin gh-pages 27 | git checkout - 28 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/RerunStateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.io.IOException; 19 | 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | import com.fasterxml.jackson.databind.node.ObjectNode; 25 | 26 | public class RerunStateTest { 27 | 28 | @Test 29 | public void testExpiration() { 30 | RerunState st = new RerunState(new ScheduledTime("2015-09-07T00:00Z")); 31 | Assert.assertTrue(st.isExpired(new ScheduledTime("2015-10-07T00:00Z"))); 32 | Assert.assertFalse(st.isExpired(new ScheduledTime("2015-09-14T00:00Z"))); 33 | Assert.assertFalse(st.isExpired(new ScheduledTime("2014-09-14T00:00Z"))); 34 | } 35 | 36 | @Test 37 | public void testJSON() throws IOException { 38 | RerunState st = new RerunState(new ScheduledTime("2015-09-07T00:00Z")); 39 | String json = Util.JSON_WRITER.writeValueAsString(st.toJSONNode()); 40 | Assert.assertEquals("{\"rerunTime\":\"2015-09-07T00:00:00.000Z\"}", json); 41 | Assert.assertEquals(st, RerunState.fromJSONNode((ObjectNode) Util.JSON_READER.readTree(json))); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/SchedulingStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * A scheduling strategy determines the order in which slots of a workflow are executed. 22 | * 23 | * Scheduling strategies are configured on a per-workflow basis. 24 | * 25 | * On each scheduler cycle, a workflow's scheduling strategy is asked 26 | * which workflow instances (slots) should be executed. 27 | * 28 | * As input for making this decision, the scheduling strategy receives 29 | * all slot states of the given workflow within the sliding window. 30 | * 31 | * There's no guarantee that all slots returned by a strategy will be executed. 32 | */ 33 | public interface SchedulingStrategy { 34 | 35 | /** 36 | * Returns the slots that this strategy wishes to execute. 37 | * 38 | * Only slots in READY state must be returned. 39 | * 40 | * @param states The states of all slots within sliding window for this workflow, with oldest ones first. 41 | * @return List of slots this strategy wishes to execute - the first in the list will be executed first. 42 | */ 43 | public List getSchedulingCandidates(List states); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/SlotID.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | /** 19 | * A single execution of a workflow at a given time. 20 | */ 21 | public class SlotID extends ValueObject { 22 | 23 | protected final WorkflowID workflowID; 24 | protected final ScheduledTime scheduledTime; 25 | 26 | public SlotID(WorkflowID workflowID, ScheduledTime scheduledTime) { 27 | this.workflowID = Util.requireNonNull(workflowID); 28 | this.scheduledTime = Util.requireNonNull(scheduledTime); 29 | } 30 | 31 | public WorkflowID getWorkflowID() { 32 | return workflowID; 33 | } 34 | 35 | public ScheduledTime getScheduledTime() { 36 | return scheduledTime; 37 | } 38 | 39 | public String toString() { 40 | return workflowID + "@" + scheduledTime; 41 | } 42 | 43 | public static SlotID fromString(String idStr) { 44 | Util.requireNonNull(idStr); 45 | String[] parts = idStr.split("@"); 46 | if (!(parts.length == 2)) { 47 | throw new IllegalArgumentException("Malformed slot ID: " + idStr); 48 | } 49 | return new SlotID(new WorkflowID(parts[0]), new ScheduledTime(parts[1])); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/convert/FixTableToJsonFileConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.convert; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.AbstractFixObjectConverter; 20 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 21 | import com.collective.celos.ci.testing.structure.fixobject.FixTable; 22 | import com.google.common.collect.Lists; 23 | import com.google.gson.Gson; 24 | import com.google.gson.GsonBuilder; 25 | import org.apache.commons.io.IOUtils; 26 | import org.apache.commons.lang.StringUtils; 27 | 28 | import java.util.List; 29 | 30 | /** 31 | * Created by akonopko on 22.01.15. 32 | */ 33 | public class FixTableToJsonFileConverter extends AbstractFixObjectConverter { 34 | 35 | private final Gson gson = new GsonBuilder().create(); 36 | 37 | @Override 38 | public FixFile convert(TestRun tr, FixTable ff) throws Exception { 39 | List jsonStrs = Lists.newArrayList(); 40 | for (FixTable.FixRow fr : ff.getRows()) { 41 | jsonStrs.add(gson.toJson(fr.getCells())); 42 | } 43 | return new FixFile(IOUtils.toInputStream(StringUtils.join(jsonStrs, "\n"))); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /celos-ui/src/main/java/com/collective/celos/ui/Main.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ui; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | import com.collective.celos.JettyServer; 22 | 23 | /** 24 | * Main class that launches the Celos UI. 25 | */ 26 | public class Main { 27 | 28 | public static String CELOS_URL_ATTR = "CELOS_URL"; 29 | public static String HUE_URL_ATTR = "HUE_URL"; 30 | public static String CONFIG_FILE_ATTR = "CONFIG_FILE"; 31 | 32 | public static void main(String... args) throws Exception { 33 | UICommandLineParser UICommandLineParser = new UICommandLineParser(); 34 | UICommandLine commandLine = UICommandLineParser.parse(args); 35 | JettyServer jettyServer = new JettyServer(); 36 | jettyServer.start(commandLine.getPort()); 37 | jettyServer.setupContext(getAttributes(commandLine), new HashMap()); 38 | } 39 | 40 | private static Map getAttributes(UICommandLine commandLine) { 41 | Map attrs = new HashMap<>(); 42 | attrs.put(CELOS_URL_ATTR, commandLine.getCelosUrl()); 43 | attrs.put(HUE_URL_ATTR, commandLine.getHueUrl()); 44 | attrs.put(CONFIG_FILE_ATTR, commandLine.getConfigFile()); 45 | return attrs; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/create/FixDirHierarchyCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.create; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.structure.fixobject.FixDir; 20 | import com.collective.celos.ci.testing.structure.fixobject.FixFsObject; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.Map; 24 | 25 | /** 26 | * Created by akonopko on 10/7/14. 27 | */ 28 | public class FixDirHierarchyCreator implements FixObjectCreator { 29 | 30 | private final Map> hierarchy; 31 | 32 | public FixDirHierarchyCreator(Map> hierarchy) { 33 | this.hierarchy = hierarchy; 34 | } 35 | 36 | public FixDir create(TestRun testRun) throws Exception { 37 | Map content = Maps.newHashMap(); 38 | for (Map.Entry> entry : hierarchy.entrySet()) { 39 | content.put(entry.getKey(), entry.getValue().create(testRun)); 40 | } 41 | return new FixDir(content); 42 | } 43 | 44 | @Override 45 | public String getDescription(TestRun testRun) { 46 | return hierarchy.keySet().toString(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/RerunState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import com.fasterxml.jackson.databind.ObjectMapper; 19 | import com.fasterxml.jackson.databind.node.ObjectNode; 20 | 21 | public class RerunState extends ValueObject { 22 | 23 | public static final int EXPIRATION_DAYS = 14; 24 | 25 | private static final String RERUN_TIME_PROP = "rerunTime"; 26 | 27 | // The wallclock time at which the slot was marked for rerun 28 | private final ScheduledTime rerunTime; 29 | 30 | public RerunState(ScheduledTime rerunTime) { 31 | this.rerunTime = Util.requireNonNull(rerunTime); 32 | } 33 | 34 | public ScheduledTime getRerunTime() { 35 | return rerunTime; 36 | } 37 | 38 | public boolean isExpired(ScheduledTime now) { 39 | return rerunTime.plusDays(EXPIRATION_DAYS).getDateTime().isBefore(now.getDateTime()); 40 | } 41 | 42 | public ObjectNode toJSONNode() { 43 | ObjectNode node = Util.MAPPER.createObjectNode(); 44 | node.put(RERUN_TIME_PROP, rerunTime.toString()); 45 | return node; 46 | } 47 | 48 | public static RerunState fromJSONNode(ObjectNode node) { 49 | String timeStr = node.get(RERUN_TIME_PROP).textValue(); 50 | return new RerunState(new ScheduledTime(timeStr)); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /celos-server/src/test/java/com/collective/celos/servlet/SchedulerServletTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.servlet; 17 | 18 | import com.collective.celos.CelosClient; 19 | import com.collective.celos.WorkflowID; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import javax.servlet.http.HttpServletRequest; 24 | import java.util.Collections; 25 | import java.util.HashSet; 26 | import java.util.Set; 27 | 28 | import static org.mockito.Mockito.mock; 29 | import static org.mockito.Mockito.when; 30 | 31 | public class SchedulerServletTest { 32 | 33 | @Test 34 | public void parsesIDsParameterMissing() throws Exception { 35 | HttpServletRequest req = mock(HttpServletRequest.class); 36 | when(req.getParameter(CelosClient.IDS_PARAM)).thenReturn(null); 37 | Assert.assertEquals(Collections.emptySet(), new SchedulerServlet().getWorkflowIDs(req)); 38 | } 39 | 40 | @Test 41 | public void parsesIDsParameter() throws Exception { 42 | HttpServletRequest req = mock(HttpServletRequest.class); 43 | when(req.getParameter(CelosClient.IDS_PARAM)).thenReturn("foo,bar"); 44 | Set ids = new HashSet<>(); 45 | ids.add(new WorkflowID("foo")); 46 | ids.add(new WorkflowID("bar")); 47 | Assert.assertEquals(ids, new SchedulerServlet().getWorkflowIDs(req)); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/config/deploy/CelosCiTarget.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.config.deploy; 17 | 18 | import com.collective.celos.Util; 19 | 20 | import java.net.URI; 21 | 22 | public class CelosCiTarget { 23 | 24 | private final URI pathToHdfsSite; 25 | private final URI pathToCoreSite; 26 | private final URI workflowsDirUri; 27 | private final URI defaultsDirUri; 28 | private final URI hiveJdbc; 29 | 30 | public CelosCiTarget(URI pathToHdfsSite, URI pathToCoreSite, URI workflowsDirUri, URI defaultsDirUri, URI hiveJdbc) { 31 | this.pathToHdfsSite = Util.requireNonNull(pathToHdfsSite); 32 | this.pathToCoreSite = Util.requireNonNull(pathToCoreSite); 33 | this.workflowsDirUri = Util.requireNonNull(workflowsDirUri); 34 | this.defaultsDirUri = Util.requireNonNull(defaultsDirUri); 35 | this.hiveJdbc = hiveJdbc; 36 | } 37 | 38 | public URI getPathToHdfsSite() { 39 | return pathToHdfsSite; 40 | } 41 | 42 | public URI getPathToCoreSite() { 43 | return pathToCoreSite; 44 | } 45 | 46 | public URI getWorkflowsDirUri() { 47 | return workflowsDirUri; 48 | } 49 | 50 | public URI getDefaultsDirUri() { 51 | return defaultsDirUri; 52 | } 53 | 54 | public URI getHiveJdbc() { 55 | return hiveJdbc; 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/structure/fixobject/ConversionCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import com.collective.celos.ci.mode.test.TestRun; 19 | import com.collective.celos.ci.testing.fixtures.create.FixObjectCreator; 20 | 21 | /** 22 | * Created by akonopko on 10/7/14. 23 | */ 24 | public class ConversionCreator implements FixObjectCreator { 25 | 26 | protected final FixObjectCreator creator; 27 | protected final AbstractFixObjectConverter fixObjectConverter; 28 | 29 | public ConversionCreator(FixObjectCreator creator, AbstractFixObjectConverter fixObjectConverter) { 30 | this.creator = creator; 31 | this.fixObjectConverter = fixObjectConverter; 32 | } 33 | 34 | @Override 35 | public T create(TestRun testRun) throws Exception { 36 | return fixObjectConverter.convert(testRun, creator.create(testRun)); 37 | } 38 | 39 | @Override 40 | public String getDescription(TestRun testRun) throws Exception { 41 | return creator.getDescription(testRun); 42 | } 43 | 44 | public FixObjectCreator getCreator() { 45 | return creator; 46 | } 47 | 48 | public AbstractFixObjectConverter getFixObjectConverter() { 49 | return fixObjectConverter; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /celos-ci/src/test/java/com/collective/celos/ci/testing/structure/fixobject/FixTableTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.structure.fixobject; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | import junit.framework.Assert; 22 | 23 | import org.junit.Test; 24 | 25 | import com.google.common.collect.Lists; 26 | import com.google.common.collect.Maps; 27 | 28 | public class FixTableTest { 29 | 30 | @Test 31 | public void testGetOrderedColumns() { 32 | List fixRows = Lists.newArrayList(); 33 | 34 | Map map = Maps.newHashMap(); 35 | map.put("col1", "val1"); 36 | map.put("col2", "val2"); 37 | 38 | Map map2 = Maps.newHashMap(); 39 | map2.put("col1", "val11"); 40 | map2.put("col2", "val22"); 41 | 42 | FixTable.FixRow row1 = new FixTable.FixRow(map); 43 | FixTable.FixRow row2 = new FixTable.FixRow(map2); 44 | fixRows.add(row1); 45 | fixRows.add(row2); 46 | FixTable fixTable = new FixTable(Lists.newArrayList("col1", "col2"), fixRows); 47 | 48 | Assert.assertEquals(Lists.newArrayList("val1", "val2"), row1.getOrderedColumns(fixTable.getColumnNames())); 49 | Assert.assertEquals(Lists.newArrayList("val11", "val22"), row2.getOrderedColumns(fixTable.getColumnNames())); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /samples/quickstart/src/main/celos/workflow.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | celos.importDefaults("wordcount"); 17 | 18 | var ROOT = "/user/" + CELOS_USER + "/celos/quickstart"; 19 | 20 | function inputPathForDataCenter(dc) { 21 | return ROOT + "/input/" + dc + "/${year}-${month}-${day}/${hour}00"; 22 | } 23 | 24 | function outputPathForDataCenter(dc) { 25 | return ROOT + "/output/" + dc + "/${year}-${month}-${day}/${hour}00"; 26 | } 27 | 28 | /* 29 | * This function defines a wordcount workflow for the given data 30 | * center. 31 | */ 32 | function defineWordCountWorkflow(dc) { 33 | var inputPath = inputPathForDataCenter(dc); 34 | var outputPath = outputPathForDataCenter(dc); 35 | celos.defineWorkflow({ 36 | "id": "wordcount-" + dc, 37 | "schedule": celos.hourlySchedule(), 38 | "schedulingStrategy": celos.serialSchedulingStrategy(), 39 | "trigger": celos.hdfsCheckTrigger(inputPath + "/_READY"), 40 | "externalService": celos.oozieExternalService({ 41 | // These properties are passed to the Oozie job 42 | "oozie.wf.application.path": ROOT + "/wordcount/workflow.xml", 43 | "inputPath": inputPath, 44 | "outputPath": outputPath 45 | }) 46 | }); 47 | } 48 | 49 | /* 50 | * Set up two workflows, one for each data center. 51 | */ 52 | defineWordCountWorkflow("lax"); 53 | defineWordCountWorkflow("nyc"); 54 | 55 | -------------------------------------------------------------------------------- /celos-server/src/main/java/com/collective/celos/trigger/NotTrigger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.trigger; 17 | 18 | 19 | import com.collective.celos.ScheduledTime; 20 | import com.collective.celos.database.StateDatabaseConnection; 21 | import com.collective.celos.Util; 22 | 23 | import java.util.Collections; 24 | 25 | /** 26 | * Trigger that takes a nested trigger and does a logical NOT. 27 | */ 28 | public class NotTrigger extends Trigger { 29 | 30 | private final Trigger trigger; 31 | 32 | public NotTrigger(Trigger trigger) throws Exception { 33 | this.trigger = Util.requireNonNull(trigger); 34 | } 35 | 36 | @Override 37 | public TriggerStatus getTriggerStatus(StateDatabaseConnection connection, ScheduledTime now, ScheduledTime scheduledTime) throws Exception { 38 | TriggerStatus status = trigger.getTriggerStatus(connection, now, scheduledTime); 39 | boolean ready = !status.isReady(); 40 | return makeTriggerStatus(ready, humanReadableDescription(ready), Collections.singletonList(status)); 41 | } 42 | 43 | private String humanReadableDescription(boolean ready) { 44 | if (ready) { 45 | return "Ready, nested trigger isn't ready"; 46 | } else { 47 | return "Not ready, nested trigger is ready"; 48 | } 49 | } 50 | 51 | public Trigger getTrigger() { 52 | return trigger; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /celos-common/src/main/java/com/collective/celos/WorkflowInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos; 17 | 18 | import java.net.URI; 19 | import java.net.URL; 20 | import java.util.List; 21 | 22 | /** 23 | * Metadata such as URL and authors of a workflow. 24 | */ 25 | public class WorkflowInfo { 26 | 27 | public static class ContactsInfo { 28 | 29 | private final URI email; 30 | 31 | private final String name; 32 | 33 | private ContactsInfo() { 34 | this.email = null; 35 | this.name = null; 36 | } 37 | 38 | public ContactsInfo(String name, String email) { 39 | this.email = email == null ? null : URI.create(email); 40 | this.name = name; 41 | } 42 | 43 | public URI getEmail() { 44 | return email; 45 | } 46 | 47 | public String getName() { 48 | return name; 49 | } 50 | } 51 | 52 | private final URL url; 53 | private final List contacts; 54 | 55 | private WorkflowInfo() { 56 | this.url = null; 57 | this.contacts = null; 58 | } 59 | 60 | public WorkflowInfo(URL url, List contacts) { 61 | this.url = url; 62 | this.contacts = contacts; 63 | } 64 | 65 | public URL getUrl() { 66 | return url; 67 | } 68 | 69 | public List getContacts() { 70 | return contacts; 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /celos-ci/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven' 2 | 3 | mainClassName = 'com.collective.celos.ci.CelosCi' 4 | version = '0.1' 5 | 6 | configurations { 7 | compileIncluded 8 | compile.extendsFrom compileIncluded 9 | } 10 | 11 | dependencies { 12 | compileIncluded project(':celos-server') 13 | compile("com.jcraft:jsch:0.1.51") 14 | 15 | compile("org.apache.commons:commons-vfs2:2.0") 16 | compile("com.google.code.gson:gson:2.2.4") 17 | compile("org.apache.hive:hive-jdbc:0.13.1-cdh5.3.1") 18 | // this leads fatJar to auth fail, i am not sure why did i put it here, so disabling it 19 | // compile("org.apache.hadoop:hadoop-mapreduce-client-core:2.5.0-cdh5.3.1") 20 | compile("net.sf.opencsv:opencsv:2.3") 21 | 22 | testCompile("junit:junit:4.8.2") 23 | testCompile("org.powermock:powermock-module-junit4:1.5.5") { 24 | exclude module: 'junit' 25 | } 26 | testCompile("org.powermock:powermock-api-mockito:1.5.5") { 27 | exclude module: 'junit' 28 | } 29 | testCompile("org.mockito:mockito-all:1.9.5") 30 | } 31 | 32 | jar { 33 | from(sourceSets.main.output) { 34 | include "**/**" 35 | } 36 | archiveName "celos-ci.jar" 37 | baseName "celos-ci" 38 | from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }} 39 | manifest { 40 | attributes("Main-Class": mainClassName) 41 | } 42 | zip64 = true 43 | } 44 | 45 | def repoUser = System.getenv("CELOS_NEXUS_USER"); 46 | def repoPw = System.getenv("CELOS_NEXUS_PASS"); 47 | 48 | uploadArchives.doFirst { 49 | repositories { 50 | mavenDeployer { 51 | repository(url: System.getenv("NEXUS_MAVEN_REPO")) { 52 | authentication(userName: repoUser, password: repoPw) 53 | } 54 | if (!System.getenv("CELOS_BUILD_NUMBER")) throw new GradleException("CELOS_BUILD_NUMBER env variable should be present") 55 | pom.version = "2.0." + System.getenv("CELOS_BUILD_NUMBER") 56 | pom.groupId = "com.collective" 57 | pom.dependencies.removeAll { it.scope == "compileIncluded" } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /celos-ci/src/main/java/com/collective/celos/ci/testing/fixtures/create/FixFileFromResourceCreator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Collective, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | * implied. See the License for the specific language governing 14 | * permissions and limitations under the License. 15 | */ 16 | package com.collective.celos.ci.testing.fixtures.create; 17 | 18 | import com.collective.celos.Util; 19 | import com.collective.celos.ci.mode.test.TestRun; 20 | import com.collective.celos.ci.testing.structure.fixobject.FixFile; 21 | 22 | import java.io.File; 23 | import java.io.FileInputStream; 24 | import java.nio.file.Path; 25 | import java.nio.file.Paths; 26 | 27 | /** 28 | * Created by akonopko on 10/7/14. 29 | */ 30 | public class FixFileFromResourceCreator implements FixObjectCreator { 31 | 32 | private final Path relativePath; 33 | 34 | public FixFileFromResourceCreator(Path path) { 35 | this.relativePath = Util.requireNonNull(path); 36 | } 37 | 38 | public FixFileFromResourceCreator(String path) { 39 | this(Paths.get(path)); 40 | } 41 | 42 | public FixFile create(TestRun testRun) throws Exception { 43 | File path = getPath(testRun); 44 | if (!path.isFile()) { 45 | throw new IllegalStateException("Cannot find file: " + path); 46 | } 47 | return new FixFile(new FileInputStream(path)); 48 | } 49 | 50 | public File getPath(TestRun testRun) { 51 | return new File(testRun.getTestCasesDir(), relativePath.toString()); 52 | } 53 | 54 | @Override 55 | public String getDescription(TestRun testRun) { 56 | return getPath(testRun).getAbsolutePath(); 57 | } 58 | 59 | } 60 | --------------------------------------------------------------------------------