├── .asf.yaml ├── .github ├── PULL_REQUEST_TEMPLATE └── workflows │ ├── build-ci-image.yaml │ ├── integration-tests.yaml │ └── unit-tests.yaml ├── .gitignore ├── .rat-excludes ├── .travis.yml ├── DISCLAIMER ├── LICENSE ├── NOTICE ├── README.md ├── api ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── livy │ │ ├── Job.java │ │ ├── JobContext.java │ │ ├── JobHandle.java │ │ ├── LivyClient.java │ │ ├── LivyClientBuilder.java │ │ ├── LivyClientFactory.java │ │ ├── annotations │ │ └── Private.java │ │ └── package-info.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── livy │ │ ├── TestClientFactory.java │ │ └── TestLivyClientBuilder.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.apache.livy.LivyClientFactory │ ├── livy-client.conf │ └── spark-defaults.conf ├── assembly ├── assembly.xml └── pom.xml ├── bin └── livy-server ├── checkstyle-suppressions.xml ├── checkstyle.xml ├── client-common ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── livy │ │ └── client │ │ └── common │ │ ├── AbstractJobHandle.java │ │ ├── BufferUtils.java │ │ ├── ClientConf.java │ │ ├── HttpMessages.java │ │ ├── Serializer.java │ │ └── TestUtils.java │ └── test │ └── java │ └── org │ └── apache │ └── livy │ └── client │ └── common │ ├── TestAbstractJobHandle.java │ ├── TestBufferUtils.java │ ├── TestClientConf.java │ ├── TestHttpMessages.java │ ├── TestSerializer.java │ └── TestTestUtils.java ├── client-http ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── client │ │ │ └── http │ │ │ ├── HttpClient.java │ │ │ ├── HttpClientFactory.java │ │ │ ├── HttpConf.java │ │ │ ├── JobHandleImpl.java │ │ │ └── LivyConnection.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.apache.livy.LivyClientFactory │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── org │ └── apache │ └── livy │ └── client │ └── http │ ├── HttpClientSpec.scala │ └── LivyConnectionSpec.scala ├── conf ├── livy-client.conf.template ├── livy-env.sh.template ├── livy.conf.template ├── log4j.properties.template └── spark-blacklist.conf.template ├── core ├── pom.xml ├── scala-2.11 │ └── pom.xml ├── scala-2.12 │ └── pom.xml └── src │ ├── main │ ├── resources │ │ └── build.marker │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ ├── EOLUtils.scala │ │ ├── Logging.scala │ │ ├── Utils.scala │ │ ├── msgs.scala │ │ └── sessions │ │ ├── Kind.scala │ │ └── SessionState.scala │ └── test │ └── scala │ └── org │ └── apache │ └── livy │ ├── EOLUtilsSuite.scala │ └── LivyBaseUnitTestSuite.scala ├── coverage └── pom.xml ├── dev ├── beeline ├── docker │ ├── README.md │ ├── build-images.sh │ ├── livy-dev-base │ │ └── Dockerfile │ ├── livy-dev-cluster │ │ ├── conf │ │ │ ├── livy │ │ │ │ ├── livy-env.sh │ │ │ │ ├── livy.conf │ │ │ │ └── log4j.properties │ │ │ ├── master │ │ │ │ ├── log4j.properties │ │ │ │ ├── spark-default.conf │ │ │ │ └── spark-env.sh │ │ │ └── worker │ │ │ │ ├── log4j.properties │ │ │ │ ├── spark-default.conf │ │ │ │ └── spark-env.sh │ │ └── docker-compose.yml │ ├── livy-dev-server │ │ └── Dockerfile │ └── livy-dev-spark │ │ └── Dockerfile ├── livy-build-info.sh ├── livy-shell ├── merge_livy_pr.py ├── release-build.sh ├── spark │ └── bin │ │ └── spark-submit ├── third-party-file.ftl └── third-party-missing-license.properties ├── docs ├── Gemfile ├── Gemfile.lock ├── README.md ├── _config.yml ├── _data │ ├── navigation.yml │ └── project.yml ├── _includes │ ├── JB │ │ └── setup │ └── themes │ │ └── apache │ │ ├── _navigation.html │ │ ├── default.html │ │ ├── footer.html │ │ ├── page.html │ │ └── settings.yml ├── _layouts │ ├── default.html │ ├── page.html │ └── post.html ├── _plugins │ └── build-apis.rb ├── assets │ └── themes │ │ └── apache │ │ ├── bootstrap │ │ ├── css │ │ │ ├── bootstrap-theme.css │ │ │ ├── bootstrap-theme.min.css │ │ │ └── bootstrap.css │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ └── glyphicons-halflings-regular.woff │ │ ├── img │ │ │ ├── glyphicons-halflings-white.png │ │ │ └── glyphicons-halflings.png │ │ └── js │ │ │ ├── bootstrap.js │ │ │ └── bootstrap.min.js │ │ ├── css │ │ ├── style.css │ │ └── syntax.css │ │ ├── img │ │ ├── egg-logo.png │ │ └── logo.png │ │ └── jquery │ │ └── jquery-2.1.1.min.js ├── index.md ├── programmatic-api.md └── rest-api.md ├── examples ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── livy │ │ └── examples │ │ └── PiApp.java │ ├── python │ └── pi_app.py │ └── scala │ └── org │ └── apache │ └── livy │ └── examples │ └── WordCountApp.scala ├── integration-test ├── pom.xml └── src │ ├── main │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── test │ │ └── framework │ │ ├── BaseIntegrationTestSuite.scala │ │ ├── Cluster.scala │ │ ├── ExternalCluster.scala │ │ ├── LivyRestClient.scala │ │ ├── MiniCluster.scala │ │ └── MiniClusterUtils.scala │ ├── test │ ├── resources │ │ ├── batch.py │ │ ├── cluster.spec.template │ │ ├── log4j.properties │ │ ├── rtest.R │ │ └── test_python_api.py │ ├── scala │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── test │ │ │ ├── BatchIT.scala │ │ │ ├── InteractiveIT.scala │ │ │ └── JobApiIT.scala │ └── spark2 │ │ └── scala │ │ └── Spark2JobApiIT.scala │ └── thriftserver │ ├── main │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── test │ │ └── framework │ │ └── BaseThriftIntegrationTestSuite.scala │ └── test │ └── scala │ └── org │ └── apache │ └── livy │ └── test │ └── JdbcIT.scala ├── pom.xml ├── python-api ├── .gitattributes ├── MANIFEST.in ├── pom.xml ├── setup.cfg ├── setup.py └── src │ ├── main │ └── python │ │ └── livy │ │ ├── __init__.py │ │ ├── client.py │ │ ├── job_context.py │ │ └── job_handle.py │ └── test │ └── python │ └── livy-tests │ ├── client_test.py │ └── resources │ ├── livy-client.conf │ ├── spark-defaults.conf │ └── text_file.txt ├── repl ├── pom.xml ├── scala-2.11 │ ├── pom.xml │ └── src │ │ ├── main │ │ └── scala │ │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── repl │ │ │ └── SparkInterpreter.scala │ │ └── test │ │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── repl │ │ └── SparkInterpreterSpec.scala ├── scala-2.12 │ ├── pom.xml │ └── src │ │ ├── main │ │ └── scala │ │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── repl │ │ │ └── SparkInterpreter.scala │ │ └── test │ │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── repl │ │ └── SparkInterpreterSpec.scala └── src │ ├── main │ ├── resources │ │ ├── build.marker │ │ └── fake_shell.py │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── repl │ │ ├── AbstractSparkInterpreter.scala │ │ ├── BypassPySparkJob.scala │ │ ├── Interpreter.scala │ │ ├── ProcessInterpreter.scala │ │ ├── PySparkJobProcessor.scala │ │ ├── PythonInterpreter.scala │ │ ├── ReplDriver.scala │ │ ├── SQLInterpreter.scala │ │ ├── Session.scala │ │ ├── SparkRInterpreter.scala │ │ └── package.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── org │ └── apache │ └── livy │ └── repl │ ├── BaseInterpreterSpec.scala │ ├── BaseSessionSpec.scala │ ├── PythonInterpreterSpec.scala │ ├── PythonSessionSpec.scala │ ├── ReplDriverSuite.scala │ ├── SQLInterpreterSpec.scala │ ├── ScalaInterpreterSpec.scala │ ├── SessionSpec.scala │ ├── SharedSessionSpec.scala │ ├── SparkRInterpreterSpec.scala │ ├── SparkRSessionSpec.scala │ └── SparkSessionSpec.scala ├── rsc ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── rsc │ │ │ ├── BaseProtocol.java │ │ │ ├── BypassJobStatus.java │ │ │ ├── ContextInfo.java │ │ │ ├── ContextLauncher.java │ │ │ ├── DriverProcessInfo.java │ │ │ ├── FutureListener.java │ │ │ ├── JobHandleImpl.java │ │ │ ├── PingJob.java │ │ │ ├── RSCClient.java │ │ │ ├── RSCClientFactory.java │ │ │ ├── RSCConf.java │ │ │ ├── ReplJobResults.java │ │ │ ├── Utils.java │ │ │ ├── driver │ │ │ ├── AddFileJob.java │ │ │ ├── AddJarJob.java │ │ │ ├── BypassJob.java │ │ │ ├── BypassJobWrapper.java │ │ │ ├── JobContextImpl.java │ │ │ ├── JobWrapper.java │ │ │ ├── MutableClassLoader.java │ │ │ ├── RSCDriver.java │ │ │ ├── RSCDriverBootstrapper.java │ │ │ ├── SparkEntries.java │ │ │ ├── Statement.java │ │ │ └── StatementState.java │ │ │ ├── package-info.java │ │ │ └── rpc │ │ │ ├── KryoMessageCodec.java │ │ │ ├── README.md │ │ │ ├── Rpc.java │ │ │ ├── RpcDispatcher.java │ │ │ ├── RpcException.java │ │ │ ├── RpcServer.java │ │ │ └── SaslHandler.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.apache.livy.LivyClientFactory │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── livy │ │ └── rsc │ │ ├── TestJobHandle.java │ │ ├── TestSparkClient.java │ │ ├── driver │ │ └── TestRSCDriver.java │ │ └── rpc │ │ ├── TestKryoMessageCodec.java │ │ └── TestRpc.java │ └── resources │ └── log4j.properties ├── scala-api ├── pom.xml ├── scala-2.11 │ └── pom.xml ├── scala-2.12 │ └── pom.xml └── src │ ├── main │ ├── resources │ │ └── build.marker │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── scalaapi │ │ ├── LivyScalaClient.scala │ │ ├── ScalaJobContext.scala │ │ ├── ScalaJobHandle.scala │ │ └── package.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── org │ └── apache │ └── livy │ └── scalaapi │ ├── ScalaClientTest.scala │ ├── ScalaClientTestUtils.scala │ └── ScalaJobHandleTest.scala ├── scala └── pom.xml ├── scalastyle.xml ├── server ├── pom.xml └── src │ ├── main │ ├── resources │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── server │ │ │ └── ui │ │ │ └── static │ │ │ ├── css │ │ │ ├── bootstrap.min.css │ │ │ ├── dataTables.bootstrap.min.css │ │ │ └── livy-ui.css │ │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ ├── glyphicons-halflings-regular.woff │ │ │ └── glyphicons-halflings-regular.woff2 │ │ │ ├── html │ │ │ ├── batches-table.html │ │ │ ├── sessions-table.html │ │ │ └── statements-table.html │ │ │ ├── img │ │ │ └── livy-mini-logo.png │ │ │ └── js │ │ │ ├── all-sessions.js │ │ │ ├── bootstrap.min.js │ │ │ ├── dataTables.bootstrap.min.js │ │ │ ├── jquery-3.4.1.min.js │ │ │ ├── jquery.dataTables.min.js │ │ │ ├── livy-ui.js │ │ │ ├── session-log.js │ │ │ └── session.js │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ ├── LivyConf.scala │ │ ├── package.scala │ │ ├── server │ │ ├── AccessFilter.scala │ │ ├── AccessManager.scala │ │ ├── ApiVersioningSupport.scala │ │ ├── ApiVersions.scala │ │ ├── CsrfFilter.scala │ │ ├── JsonServlet.scala │ │ ├── LivyServer.scala │ │ ├── SecurityHeadersFilter.scala │ │ ├── SessionServlet.scala │ │ ├── ThriftServerFactory.scala │ │ ├── WebServer.scala │ │ ├── auth │ │ │ ├── LdapAuthenticationHandlerImpl.scala │ │ │ └── LdapUtils.scala │ │ ├── batch │ │ │ ├── BatchSession.scala │ │ │ ├── BatchSessionServlet.scala │ │ │ └── CreateBatchRequest.scala │ │ ├── interactive │ │ │ ├── CreateInteractiveRequest.scala │ │ │ ├── InteractiveSession.scala │ │ │ ├── InteractiveSessionServlet.scala │ │ │ └── SessionHeartbeat.scala │ │ ├── recovery │ │ │ ├── BlackholeStateStore.scala │ │ │ ├── FileSystemStateStore.scala │ │ │ ├── SessionStore.scala │ │ │ ├── StateStore.scala │ │ │ ├── ZooKeeperManager.scala │ │ │ └── ZooKeeperStateStore.scala │ │ └── ui │ │ │ └── UIServlet.scala │ │ ├── sessions │ │ ├── Session.scala │ │ └── SessionManager.scala │ │ └── utils │ │ ├── Clock.scala │ │ ├── LineBufferedProcess.scala │ │ ├── LineBufferedStream.scala │ │ ├── LivySparkUtils.scala │ │ ├── LivyUncaughtException.java │ │ ├── SparkApp.scala │ │ ├── SparkKubernetesApp.scala │ │ ├── SparkProcApp.scala │ │ ├── SparkProcessBuilder.scala │ │ └── SparkYarnApp.scala │ └── test │ ├── resources │ ├── log4j.properties │ └── spark-blacklist.conf │ └── scala │ └── org │ └── apache │ └── livy │ ├── server │ ├── AccessManagerSuite.scala │ ├── ApiVersioningSupportSpec.scala │ ├── BaseJsonServletSpec.scala │ ├── BaseSessionServletSpec.scala │ ├── JsonServletSpec.scala │ ├── SecurityHeadersFilterSpec.scala │ ├── SessionServletSpec.scala │ ├── auth │ │ └── TestLdapAuthenticationHandlerImpl.scala │ ├── batch │ │ ├── BatchServletSpec.scala │ │ ├── BatchSessionSpec.scala │ │ └── CreateBatchRequestSpec.scala │ ├── interactive │ │ ├── BaseInteractiveServletSpec.scala │ │ ├── CreateInteractiveRequestSpec.scala │ │ ├── InteractiveSessionServletSpec.scala │ │ ├── InteractiveSessionSpec.scala │ │ ├── JobApiSpec.scala │ │ └── SessionHeartbeatSpec.scala │ └── recovery │ │ ├── BlackholeStateStoreSpec.scala │ │ ├── FileSystemStateStoreSpec.scala │ │ ├── SessionStoreSpec.scala │ │ ├── StateStoreSpec.scala │ │ └── ZooKeeperStateStoreSpec.scala │ ├── sessions │ ├── MockSession.scala │ ├── SessionManagerSpec.scala │ └── SessionSpec.scala │ └── utils │ ├── LivySparkUtilsSuite.scala │ ├── SparkKubernetesAppSpec.scala │ └── SparkYarnAppSpec.scala ├── test-lib ├── pom.xml └── src │ └── main │ ├── java │ └── org │ │ └── apache │ │ └── livy │ │ └── test │ │ ├── apps │ │ ├── FailingApp.java │ │ └── SimpleSparkApp.java │ │ └── jobs │ │ ├── Echo.java │ │ ├── Failure.java │ │ ├── FileReader.java │ │ ├── GetCurrentUser.java │ │ ├── SQLGetTweets.java │ │ ├── SharedVariableCounter.java │ │ ├── Sleeper.java │ │ ├── SmallCount.java │ │ └── VoidJob.java │ ├── resources │ └── testweet.json │ ├── scala │ └── org │ │ └── apache │ │ └── livy │ │ └── test │ │ └── jobs │ │ ├── ScalaEcho.scala │ │ └── ScalaSharedVariableCounter.scala │ └── spark2 │ └── java │ └── org │ └── apache │ └── livy │ └── test │ └── jobs │ └── spark2 │ ├── DatasetTest.java │ └── SparkSessionTest.java └── thriftserver ├── client └── pom.xml ├── server ├── pom.xml └── src │ ├── main │ ├── resources │ │ └── org │ │ │ └── apache │ │ │ └── livy │ │ │ └── server │ │ │ └── ui │ │ │ └── static │ │ │ ├── html │ │ │ └── thrift-sessions-table.html │ │ │ └── js │ │ │ └── thrift-sessions.js │ └── scala │ │ └── org │ │ └── apache │ │ └── livy │ │ └── thriftserver │ │ ├── LivyCLIService.scala │ │ ├── LivyExecuteStatementOperation.scala │ │ ├── LivyOperationManager.scala │ │ ├── LivyThriftServer.scala │ │ ├── LivyThriftSessionManager.scala │ │ ├── SessionInfo.scala │ │ ├── SessionStates.scala │ │ ├── ThriftServerAudit.scala │ │ ├── ThriftServerFactoryImpl.scala │ │ ├── ThriftService.scala │ │ ├── auth │ │ ├── AuthBridgeServer.scala │ │ ├── AuthFactory.scala │ │ ├── AuthenticationProvider.scala │ │ ├── LdapAuthenticationProviderImpl.scala │ │ ├── LivyDelegationTokenSecretManager.scala │ │ ├── PlainSaslServer.scala │ │ └── ldap │ │ │ ├── ChainFilter.scala │ │ │ ├── DirSearchFactory.scala │ │ │ ├── Filter.scala │ │ │ ├── LdapSearchFactory.scala │ │ │ └── UserFilter.scala │ │ ├── cli │ │ ├── ThreadPoolExecutorWithOomHook.scala │ │ ├── ThriftBinaryCLIService.scala │ │ ├── ThriftCLIService.scala │ │ ├── ThriftHttpCLIService.scala │ │ └── ThriftHttpServlet.scala │ │ ├── operation │ │ ├── GetCatalogsOperation.scala │ │ ├── GetColumnsOperation.scala │ │ ├── GetFunctionsOperation.scala │ │ ├── GetSchemasOperation.scala │ │ ├── GetTableTypesOperation.scala │ │ ├── GetTablesOperation.scala │ │ ├── GetTypeInfoOperation.scala │ │ ├── MetadataOperation.scala │ │ ├── Operation.scala │ │ └── SparkCatalogOperation.scala │ │ ├── rpc │ │ └── RpcClient.scala │ │ ├── serde │ │ └── ThriftResultSet.scala │ │ ├── types │ │ ├── DataTypeUtils.scala │ │ └── Schema.scala │ │ └── ui │ │ └── ThriftJsonServlet.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── org │ └── apache │ └── livy │ └── thriftserver │ ├── TestLivyThriftSessionManager.scala │ ├── ThriftServerBaseTest.scala │ ├── ThriftServerSuites.scala │ └── auth │ └── TestLdapAuthenticationProviderImpl.scala └── session ├── pom.xml └── src ├── main └── java │ └── org │ └── apache │ └── livy │ └── thriftserver │ └── session │ ├── CatalogJobState.java │ ├── CleanupStatementJob.java │ ├── ColumnBuffer.java │ ├── DataType.java │ ├── FetchResultJob.java │ ├── FetchResultSchemaJob.java │ ├── GetColumnsJob.java │ ├── GetFunctionsJob.java │ ├── GetSchemasJob.java │ ├── GetTablesJob.java │ ├── RegisterSessionJob.java │ ├── ResultSet.java │ ├── ScalaIterator.java │ ├── SparkCatalogJob.java │ ├── SparkUtils.java │ ├── SqlJob.java │ ├── StatementState.java │ ├── ThriftSessionState.java │ └── UnregisterSessionJob.java └── test └── java └── org └── apache └── livy └── thriftserver └── session ├── ColumnBufferTest.java └── ThriftSessionTest.java /.asf.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # See: https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features 19 | 20 | github: 21 | description: "Apache Livy is an open source REST interface for interacting with Apache Spark from anywhere." 22 | homepage: https://livy.apache.org/ 23 | labels: 24 | - livy 25 | - apachelivy 26 | - bigdata 27 | - spark 28 | enabled_merge_buttons: 29 | squash: true 30 | merge: false 31 | rebase: false 32 | protected_branches: 33 | master: 34 | required_pull_request_reviews: 35 | required_approving_review_count: 1 36 | features: 37 | issues: true 38 | discussions: true 39 | wiki: true 40 | projects: true 41 | notifications: 42 | commits: commits@livy.apache.org 43 | issues: commits@livy.apache.org 44 | pullrequests: commits@livy.apache.org 45 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE: -------------------------------------------------------------------------------- 1 | ## What changes were proposed in this pull request? 2 | 3 | (Please fill in changes proposed in this fix) 4 | (Include a link to the associated JIRA and make sure to add a link to this pr on the JIRA as well) 5 | 6 | ## How was this patch tested? 7 | 8 | (Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests) 9 | (If this patch involves UI changes, please attach a screenshot; otherwise, remove this) 10 | 11 | Please review https://livy.incubator.apache.org/community/ before opening a pull request. 12 | -------------------------------------------------------------------------------- /.github/workflows/build-ci-image.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | name: 'Build CI images' 18 | on: 19 | push: 20 | branches: ["master"] 21 | paths: 22 | - 'dev/docker/livy-dev-base/Dockerfile' 23 | jobs: 24 | docker-build: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - 28 | name: Checkout 29 | uses: actions/checkout@v3 30 | - 31 | name: Set up Docker Buildx 32 | uses: docker/setup-buildx-action@v2 33 | - 34 | name: Login to the GitHub Container Registry 35 | uses: docker/login-action@v2 36 | with: 37 | registry: ghcr.io 38 | username: ${{ github.repository_owner }} 39 | password: ${{ secrets.GITHUB_TOKEN }} 40 | - 41 | name: Build and push image 42 | id: docker_build 43 | uses: docker/build-push-action@v4 44 | with: 45 | push: true 46 | context: ./dev/docker/livy-dev-base 47 | tags: | 48 | ghcr.io/${{ github.repository_owner }}/livy-ci:latest 49 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | name: Unit Tests 18 | on: [push] 19 | env: 20 | ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true 21 | MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 22 | jobs: 23 | build: 24 | runs-on: ubuntu-24.04 25 | # TODO: Possibly point to the ./build-ci-image.yaml with the "uses" key 26 | container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest 27 | strategy: 28 | matrix: 29 | maven_profile: 30 | - "-Pscala-2.11 -Pspark2" 31 | - "-Pscala-2.12 -Pspark2" 32 | - "-Pscala-2.12 -Pspark3" 33 | steps: 34 | - 35 | name: Checkout 36 | uses: actions/checkout@v3 37 | - 38 | name: Cache local Maven repository 39 | uses: actions/cache@v3 40 | with: 41 | path: | 42 | /root/.m2/repository 43 | !/root/.m2/repository/org/apache/livy 44 | key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml', '*/pom.xml', 'thriftserver/*/pom.xml', 'core/*/pom.xml', 'repl/*/pom.xml', 'scala-api/*/pom.xml') }} 45 | restore-keys: | 46 | ${{ runner.os }}-maven- 47 | - 48 | name: Build with Maven 49 | run: mvn -Pthriftserver ${{ matrix.maven_profile }} -DskipITs -Dmaven.javadoc.skip=true -B -V -e verify 50 | - 51 | name: Upload coverage to codecov 52 | uses: codecov/codecov-action@v3 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.ipr 2 | *.iml 3 | *.iws 4 | *.pyc 5 | *.pyo 6 | .idea/ 7 | .idea_modules/ 8 | .settings 9 | .cache 10 | .DS_Store 11 | .project 12 | .classpath 13 | .scala_dependencies 14 | conf/*.conf 15 | conf/*.properties 16 | conf/*.sh 17 | docs/api/ 18 | lib_managed/ 19 | logs/ 20 | src_managed/ 21 | target/ 22 | reports/ 23 | metastore_db/ 24 | derby.log 25 | dependency-reduced-pom.xml 26 | release-staging/ 27 | venv/ 28 | .sdkmanrc 29 | 30 | # For python setup.py, which pollutes the source dirs. 31 | python-api/dist 32 | .eggs 33 | *.egg-info 34 | *.egg 35 | -------------------------------------------------------------------------------- /.rat-excludes: -------------------------------------------------------------------------------- 1 | .rat-excludes 2 | .github/* 3 | logs/* 4 | **/logs/** 5 | **/*.conf 6 | **/*.json 7 | **/*.md 8 | *.iml 9 | **/*.iml 10 | **/build.marker 11 | **/dependency-reduced-pom.xml 12 | **/target/** 13 | **/bootstrap.min.css 14 | **/bootstrap.min.js 15 | **/dataTables.bootstrap.min.css 16 | **/dataTables.bootstrap.min.js 17 | **/jquery.dataTables.min.js 18 | **/jquery-3.4.1.min.js 19 | **/fonts/** 20 | **/livy_python_api.egg-info/* 21 | **/*.txt 22 | **/MANIFEST.in 23 | **/setup.cfg 24 | **/derby.log 25 | **/metastore_db/** 26 | **/.eggs/** 27 | **/.pytest_cache/** 28 | **/Gemfile.lock 29 | **/jquery-2.1.1.min.js 30 | docs/**/*.html 31 | docs/**/JB/** 32 | venv/* 33 | -------------------------------------------------------------------------------- /DISCLAIMER: -------------------------------------------------------------------------------- 1 | Apache Livy is an effort undergoing incubation at the Apache Software 2 | Foundation (ASF), sponsored by the Apache Incubator PMC. 3 | 4 | Incubation is required of all newly accepted projects until a further review 5 | indicates that the infrastructure, communications, and decision making process 6 | have stabilized in a manner consistent with other successful ASF projects. 7 | 8 | While incubation status is not necessarily a reflection of the completeness 9 | or stability of the code, it does indicate that the project has yet to be 10 | fully endorsed by the ASF. 11 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Livy 2 | Copyright 2018-2023 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/livy/Job.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy; 19 | 20 | import java.io.Serializable; 21 | 22 | /** 23 | * Interface for a Spark remote job. 24 | */ 25 | public interface Job extends Serializable { 26 | 27 | T call(JobContext jc) throws Exception; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/livy/LivyClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy; 19 | 20 | import java.net.URI; 21 | import java.util.Properties; 22 | 23 | import org.apache.livy.annotations.Private; 24 | 25 | /** 26 | * A factory for Livy clients. Client implementations can register themselves by using the 27 | * Java services mechanism, providing implementations of this interface. 28 | *

29 | * Client applications do not need to use this interface directly. Instead, use 30 | * {@link LivyClientBuilder}. 31 | * 32 | * @see java.util.ServiceLoader 33 | */ 34 | @Private 35 | public interface LivyClientFactory { 36 | 37 | /** 38 | * Instantiates a new client if the given URI is supported by the implementation. 39 | * 40 | * @param uri URI pointing at the livy backend to use. 41 | * @param config Livy client configs. 42 | * @return The newly created LivyClient or null if an unsupported URI 43 | */ 44 | LivyClient createClient(URI uri, Properties config); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/livy/annotations/Private.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.annotations; 19 | 20 | import java.lang.annotation.Documented; 21 | import java.lang.annotation.Retention; 22 | import java.lang.annotation.RetentionPolicy; 23 | 24 | /** 25 | * Indicates an API that is considered private to Livy and should not be used by client 26 | * applications. 27 | */ 28 | @Documented 29 | @Retention(RetentionPolicy.CLASS) 30 | public @interface Private { } 31 | -------------------------------------------------------------------------------- /api/src/main/java/org/apache/livy/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | * Livy programmatic Java API 20 | */ 21 | package org.apache.livy; 22 | -------------------------------------------------------------------------------- /api/src/test/resources/META-INF/services/org.apache.livy.LivyClientFactory: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | org.apache.livy.TestClientFactory 18 | -------------------------------------------------------------------------------- /api/src/test/resources/livy-client.conf: -------------------------------------------------------------------------------- 1 | livy.uri=match 2 | spark.config=override -------------------------------------------------------------------------------- /api/src/test/resources/spark-defaults.conf: -------------------------------------------------------------------------------- 1 | spark.config=default -------------------------------------------------------------------------------- /checkstyle-suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /client-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | 21 | org.apache.livy 22 | livy-main 23 | 0.9.0-incubating-SNAPSHOT 24 | 25 | 26 | org.apache.livy 27 | livy-client-common 28 | 0.9.0-incubating-SNAPSHOT 29 | jar 30 | 31 | 32 | 33 | org.apache.livy 34 | livy-api 35 | ${project.version} 36 | 37 | 38 | 39 | com.esotericsoftware 40 | kryo-shaded 41 | 42 | 43 | com.fasterxml.jackson.core 44 | jackson-databind 45 | test 46 | 47 | 48 | org.slf4j 49 | slf4j-reload4j 50 | provided 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /client-common/src/main/java/org/apache/livy/client/common/BufferUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.client.common; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.apache.livy.annotations.Private; 23 | 24 | /** 25 | * Utility methods for dealing with byte buffers and byte arrays. 26 | */ 27 | @Private 28 | public class BufferUtils { 29 | 30 | public static byte[] toByteArray(ByteBuffer buf) { 31 | byte[] bytes; 32 | if (buf.hasArray() && buf.arrayOffset() == 0 && 33 | buf.remaining() == buf.array().length) { 34 | bytes = buf.array(); 35 | } else { 36 | bytes = new byte[buf.remaining()]; 37 | buf.get(bytes); 38 | } 39 | return bytes; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /client-common/src/test/java/org/apache/livy/client/common/TestBufferUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.client.common; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.Test; 23 | import static org.junit.Assert.*; 24 | 25 | public class TestBufferUtils { 26 | 27 | @Test 28 | public void testWrappedArray() { 29 | byte[] array = new byte[] { 0x1b, 0x2b }; 30 | byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array)); 31 | assertSame(array, unwrapped); 32 | } 33 | 34 | @Test 35 | public void testShortArray() { 36 | byte[] array = new byte[] { 0x1b, 0x2b }; 37 | byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 0, 1)); 38 | assertNotSame(array, unwrapped); 39 | assertEquals(1, unwrapped.length); 40 | } 41 | 42 | @Test 43 | public void testOffsetArray() { 44 | byte[] array = new byte[] { 0x1b, 0x2b }; 45 | byte[] unwrapped = BufferUtils.toByteArray(ByteBuffer.wrap(array, 1, 1)); 46 | assertNotSame(array, unwrapped); 47 | assertEquals(1, unwrapped.length); 48 | } 49 | 50 | @Test 51 | public void testDirectBuffer() { 52 | ByteBuffer direct = ByteBuffer.allocateDirect(1); 53 | direct.put((byte) 0x1b); 54 | assertFalse(direct.hasArray()); 55 | direct.flip(); 56 | 57 | byte[] unwrapped = BufferUtils.toByteArray(direct); 58 | assertEquals(0x1b, unwrapped[0]); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /client-common/src/test/java/org/apache/livy/client/common/TestTestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.client.common; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.junit.Test; 23 | import static org.junit.Assert.*; 24 | 25 | public class TestTestUtils { 26 | 27 | @Test 28 | public void testJacocoArgs() { 29 | String args1 = TestUtils.getJacocoArgs(); 30 | String expected1 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-1.exec"); 31 | assertEquals(expected1, args1); 32 | 33 | String args2 = TestUtils.getJacocoArgs(); 34 | String expected2 = System.getProperty("jacoco.args").replace("main.exec", "jacoco-2.exec"); 35 | assertEquals(expected2, args2); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /client-http/src/main/java/org/apache/livy/client/http/HttpClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.client.http; 19 | 20 | import java.net.URI; 21 | import java.util.Properties; 22 | 23 | import org.apache.livy.LivyClient; 24 | import org.apache.livy.LivyClientFactory; 25 | 26 | /** 27 | * Factory for HTTP Livy clients. 28 | */ 29 | public final class HttpClientFactory implements LivyClientFactory { 30 | 31 | @Override 32 | public LivyClient createClient(URI uri, Properties config) { 33 | if (!"http".equals(uri.getScheme()) && !"https".equals(uri.getScheme())) { 34 | return null; 35 | } 36 | 37 | return new HttpClient(uri, new HttpConf(config)); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /client-http/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | org.apache.livy.client.http.HttpClientFactory 18 | -------------------------------------------------------------------------------- /client-http/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | test.appender=file 20 | log4j.rootCategory=INFO, ${test.appender} 21 | log4j.appender.file=org.apache.log4j.FileAppender 22 | log4j.appender.file.append=true 23 | log4j.appender.file.file=target/unit-tests.log 24 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 26 | 27 | # Tests that launch java subprocesses can set the "test.appender" system property to 28 | # "console" to avoid having the child process's logs overwrite the unit test's 29 | # log file. 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%t: %m%n 34 | 35 | # Ignore messages below warning level from Jetty, because it's a bit verbose 36 | log4j.logger.org.apache.livy.shaded=INFO 37 | log4j.logger.org.eclipse.jetty=WARN 38 | -------------------------------------------------------------------------------- /conf/livy-env.sh.template: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # LIVY ENVIRONMENT VARIABLES 19 | # 20 | # - JAVA_HOME Java runtime to use. By default use "java" from PATH. 21 | # - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. 22 | # - SPARK_HOME Spark which you would like to use in Livy. 23 | # - SPARK_CONF_DIR Optional directory where the Spark configuration lives. 24 | # (Default: $SPARK_HOME/conf) 25 | # - LIVY_LOG_DIR Where log files are stored. (Default: ${LIVY_HOME}/logs) 26 | # - LIVY_PID_DIR Where the pid file is stored. (Default: /tmp) 27 | # - LIVY_SERVER_JAVA_OPTS Java Opts for running livy server (You can set jvm related setting here, 28 | # like jvm memory/gc algorithm and etc.) 29 | # - LIVY_IDENT_STRING A name that identifies the Livy server instance, used to generate log file 30 | # names. (Default: name of the user starting Livy). 31 | # - LIVY_MAX_LOG_FILES Max number of log file to keep in the log directory. (Default: 5.) 32 | # - LIVY_NICENESS Niceness of the Livy server process when running in the background. (Default: 0.) 33 | # - LIVY_CLASSPATH Override if the additional classpath is required. 34 | -------------------------------------------------------------------------------- /conf/log4j.properties.template: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # The default Livy logging configuration. 18 | log4j.rootCategory=INFO, console 19 | log4j.appender.console=org.apache.log4j.ConsoleAppender 20 | log4j.appender.console.target=System.err 21 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 22 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 23 | 24 | log4j.logger.org.eclipse.jetty=WARN 25 | -------------------------------------------------------------------------------- /conf/spark-blacklist.conf.template: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # 18 | # Configuration override / blacklist. Defines a list of properties that users are not allowed 19 | # to override when starting Spark sessions. 20 | # 21 | # This file takes a list of property names (one per line). Empty lines and lines starting with "#" 22 | # are ignored. 23 | # 24 | 25 | # Disallow overriding the master and the deploy mode. 26 | spark.master 27 | spark.submit.deployMode 28 | 29 | # Disallow overriding the location of Spark cached jars. 30 | spark.yarn.jar 31 | spark.yarn.jars 32 | spark.yarn.archive 33 | 34 | # Don't allow users to override the RSC timeout. 35 | livy.rsc.server.idle-timeout 36 | -------------------------------------------------------------------------------- /core/scala-2.11/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | org.apache.livy 21 | livy-core_2.11 22 | 0.9.0-incubating-SNAPSHOT 23 | jar 24 | 25 | 26 | org.apache.livy 27 | livy-core-parent 28 | 0.9.0-incubating-SNAPSHOT 29 | ../pom.xml 30 | 31 | 32 | 33 | 34 | 35 | org.apache.maven.plugins 36 | maven-jar-plugin 37 | 38 | 39 | 40 | test-jar 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /core/scala-2.12/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | org.apache.livy 21 | livy-core_2.12 22 | 0.9.0-incubating-SNAPSHOT 23 | jar 24 | 25 | 26 | org.apache.livy 27 | livy-core-parent 28 | 0.9.0-incubating-SNAPSHOT 29 | ../pom.xml 30 | 31 | 32 | 33 | 34 | 35 | org.apache.maven.plugins 36 | maven-jar-plugin 37 | 38 | 39 | 40 | test-jar 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /core/src/main/resources/build.marker: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/core/src/main/resources/build.marker -------------------------------------------------------------------------------- /core/src/main/scala/org/apache/livy/Logging.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy 19 | 20 | import org.slf4j.LoggerFactory 21 | 22 | trait Logging { 23 | lazy val logger = LoggerFactory.getLogger(this.getClass) 24 | 25 | def trace(message: => Any): Unit = { 26 | if (logger.isTraceEnabled) { 27 | logger.trace(message.toString) 28 | } 29 | } 30 | 31 | def debug(message: => Any): Unit = { 32 | if (logger.isDebugEnabled) { 33 | logger.debug(message.toString) 34 | } 35 | } 36 | 37 | def info(message: => Any): Unit = { 38 | if (logger.isInfoEnabled) { 39 | logger.info(message.toString) 40 | } 41 | } 42 | 43 | def warn(message: => Any): Unit = { 44 | logger.warn(message.toString) 45 | } 46 | 47 | def warn(message: => Any, t: Throwable): Unit = { 48 | logger.warn(message.toString, t) 49 | } 50 | 51 | def error(message: => Any, t: Throwable): Unit = { 52 | logger.error(message.toString, t) 53 | } 54 | 55 | def error(message: => Any): Unit = { 56 | logger.error(message.toString) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /core/src/test/scala/org/apache/livy/EOLUtilsSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy 19 | 20 | import org.scalatest.FunSuite 21 | 22 | class EOLUtilsSuite extends FunSuite with LivyBaseUnitTestSuite { 23 | 24 | test("check EOL") { 25 | val s1 = "test\r\ntest" 26 | assert(!EOLUtils.Mode.hasUnixEOL(s1)) 27 | assert(!EOLUtils.Mode.hasOldMacEOL(s1)) 28 | assert(EOLUtils.Mode.hasWindowsEOL(s1)) 29 | 30 | val s2 = "test\ntest" 31 | assert(EOLUtils.Mode.hasUnixEOL(s2)) 32 | assert(!EOLUtils.Mode.hasOldMacEOL(s2)) 33 | assert(!EOLUtils.Mode.hasWindowsEOL(s2)) 34 | 35 | val s3 = "test\rtest" 36 | assert(!EOLUtils.Mode.hasUnixEOL(s3)) 37 | assert(EOLUtils.Mode.hasOldMacEOL(s3)) 38 | assert(!EOLUtils.Mode.hasWindowsEOL(s3)) 39 | 40 | val s4 = "testtest" 41 | assert(!EOLUtils.Mode.hasUnixEOL(s4)) 42 | assert(!EOLUtils.Mode.hasOldMacEOL(s4)) 43 | assert(!EOLUtils.Mode.hasWindowsEOL(s4)) 44 | } 45 | 46 | test("convert EOL") { 47 | val s1 = "test\r\ntest" 48 | val s2 = "test\ntest" 49 | val s3 = "test\rtest" 50 | val s4 = "testtest" 51 | 52 | assert(EOLUtils.convertToSystemEOL(s1) === EOLUtils.convertToSystemEOL(s2)) 53 | assert(EOLUtils.convertToSystemEOL(s1) === EOLUtils.convertToSystemEOL(s3)) 54 | assert(EOLUtils.convertToSystemEOL(s2) === EOLUtils.convertToSystemEOL(s3)) 55 | assert(EOLUtils.convertToSystemEOL(s4) === s4) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /core/src/test/scala/org/apache/livy/LivyBaseUnitTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy 19 | 20 | import org.scalatest.{Outcome, TestSuite} 21 | 22 | trait LivyBaseUnitTestSuite extends TestSuite with Logging { 23 | 24 | protected override def withFixture(test: NoArgTest): Outcome = { 25 | val testName = test.name 26 | val suiteName = this.getClass.getName 27 | try { 28 | info(s"\n\n==== TEST OUTPUT FOR $suiteName: '$testName' ====\n") 29 | test() 30 | } finally { 31 | info(s"\n\n==== FINISHED $suiteName: '$testName' ====\n") 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dev/beeline: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | export LIVY_HOME=$(cd $(dirname $0)/.. && pwd) 19 | CLASS="org.apache.hive.beeline.BeeLine" 20 | 21 | LIVY_CONF_DIR=${LIVY_CONF_DIR:-"$LIVY_HOME/conf"} 22 | 23 | if [ -f "${LIVY_CONF_DIR}/livy-env.sh" ]; then 24 | # Promote all variable declarations to environment (exported) variables 25 | set -a 26 | . "${LIVY_CONF_DIR}/livy-env.sh" 27 | set +a 28 | fi 29 | 30 | # Find the java binary 31 | if [ -n "${JAVA_HOME}" ]; then 32 | RUNNER="${JAVA_HOME}/bin/java" 33 | elif [ `command -v java` ]; then 34 | RUNNER="java" 35 | else 36 | echo "JAVA_HOME is not set" >&2 37 | exit 1 38 | fi 39 | 40 | 41 | LIBDIR="$LIVY_HOME/thriftserver/client/target/jars" 42 | 43 | if [ ! -d "$LIBDIR" ]; then 44 | echo "Could not find Livy jars directory." 1>&2 45 | exit 1 46 | fi 47 | 48 | exec $RUNNER $BEELINE_JAVA_OPTS -cp $LIBDIR/*:$LIVY_CONF_DIR:$CLASSPATH $CLASS "$@" 49 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/livy/livy-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export LIVY_SERVER_JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,address=9010,suspend=n" 19 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/livy/livy.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | livy.spark.master = spark://master:7077 19 | livy.file.local-dir-whitelist = / 20 | livy.spark.driver.memory=1g 21 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/livy/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | log4j.rootCategory=INFO, console, DRFA 19 | 20 | log4j.appender.console=org.apache.log4j.ConsoleAppender 21 | log4j.appender.console.target=System.err 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 24 | 25 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 26 | log4j.appender.DRFA.File=/logs/livy-server.log 27 | # Rollver at midnight 28 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 29 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 30 | # Pattern format: Date LogLevel LoggerName LogMessage 31 | log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 32 | 33 | log4j.logger.org.eclipse.jetty=WARN 34 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/master/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | log4j.rootCategory=INFO, console, DRFA 19 | log4jspark.log.dir=/logs 20 | log4jspark.log.file=spark-master.log 21 | 22 | log4j.appender.console=org.apache.log4j.ConsoleAppender 23 | log4j.appender.console.target=System.err 24 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 26 | 27 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 28 | log4j.appender.DRFA.File=${log4jspark.log.dir}/${log4jspark.log.file} 29 | # Rollver at midnight 30 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 31 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 32 | # Pattern format: Date LogLevel LoggerName LogMessage 33 | log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 34 | 35 | log4j.logger.org.eclipse.jetty=WARN 36 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/master/spark-default.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | spark.driver.port 7001 19 | spark.fileserver.port 7002 20 | spark.broadcast.port 7003 21 | spark.replClassServer.port 7004 22 | spark.blockManager.port 7005 23 | spark.driver.memory 1024m 24 | 25 | spark.broadcast.factory org.apache.spark.broadcast.HttpBroadcastFactory 26 | spark.port.maxRetries 4 27 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/master/spark-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Nothing here yet 19 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/worker/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | log4j.rootCategory=INFO, console, DRFA 19 | log4jspark.log.dir=/logs 20 | log4jspark.log.file=spark-worker.log 21 | 22 | log4j.appender.console=org.apache.log4j.ConsoleAppender 23 | log4j.appender.console.target=System.err 24 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.console.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 26 | 27 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 28 | log4j.appender.DRFA.File=${log4jspark.log.dir}/${log4jspark.log.file} 29 | # Rollver at midnight 30 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 31 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 32 | # Pattern format: Date LogLevel LoggerName LogMessage 33 | log4j.appender.DRFA.layout.ConversionPattern=%d %p %c{1} [%t]: %m%n 34 | 35 | log4j.logger.org.eclipse.jetty=WARN 36 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/worker/spark-default.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | spark.fileserver.port 7012 19 | spark.broadcast.port 7013 20 | spark.replClassServer.port 7014 21 | spark.blockManager.port 7015 22 | 23 | spark.broadcast.factory org.apache.spark.broadcast.HttpBroadcastFactory 24 | spark.port.maxRetries 4 25 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-cluster/conf/worker/spark-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Nothing here yet 19 | -------------------------------------------------------------------------------- /dev/docker/livy-dev-server/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | FROM livy-dev-spark:latest 19 | 20 | ARG LIVY_VERSION=0.9.0-incubating-SNAPSHOT 21 | ARG ROOT_PATH=/opt 22 | 23 | RUN apt-get update \ 24 | && apt-get install -y unzip 25 | 26 | ENV LIVY_HOME=${ROOT_PATH}/livy 27 | ENV LIVY_PACKAGE=apache-livy-${LIVY_VERSION}-bin 28 | 29 | COPY ${LIVY_PACKAGE}.zip ${LIVY_PACKAGE}.zip 30 | 31 | RUN unzip ${LIVY_PACKAGE}.zip 1>/dev/null \ 32 | && mv ${LIVY_PACKAGE} ${ROOT_PATH}/ \ 33 | && ln -s ${ROOT_PATH}/${LIVY_PACKAGE} ${LIVY_HOME} \ 34 | && chown -R root:root ${LIVY_HOME} \ 35 | && rm ${LIVY_PACKAGE}.zip 36 | 37 | # Uncomment following line or add more such lines to replace the default jars with private builds. 38 | # COPY livy-core_2.12-0.9.0-incubating-SNAPSHOT.jar ${SPARK_HOME}/repl_2.12-jars/livy-core_2.12-0.9.0-incubating-SNAPSHOT.jar 39 | 40 | WORKDIR ${LIVY_HOME} 41 | -------------------------------------------------------------------------------- /dev/livy-build-info.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # This script generates the build info for Livy and places it into the livy-version-info.properties file. 21 | # Arguments: 22 | # build_tgt_directory - The target directory where properties file would be created. [./server/target/extra-resources] 23 | # livy_version - The current version of livy 24 | 25 | RESOURCE_DIR="$1" 26 | mkdir -p "$RESOURCE_DIR" 27 | LIVY_BUILD_INFO="${RESOURCE_DIR}"/livy-version-info.properties 28 | 29 | echo_build_properties() { 30 | echo version=$1 31 | echo user=$USER 32 | echo revision=$(git rev-parse HEAD) 33 | echo branch=$(git rev-parse --abbrev-ref HEAD) 34 | echo date=$(date -u +%Y-%m-%dT%H:%M:%SZ) 35 | echo url=$(git config --get remote.origin.url) 36 | } 37 | 38 | echo_build_properties $2 > "$LIVY_BUILD_INFO" 39 | -------------------------------------------------------------------------------- /dev/third-party-missing-license.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | commons-beanutils--commons-beanutils--1.7.0=Apache License, Version 2.0 17 | javax.servlet.jsp--jsp-api--2.1=CDDL 1.1 18 | javax.transaction--jta--1.1=CDDL 1.1 19 | org.antlr--antlr-runtime--3.4=BSD License 20 | org.apache.zookeeper--zookeeper--3.4.6=Apache License, Version 2.0 21 | oro--oro--2.0.8=Apache License, Version 2.0 22 | asm--asm--3.1=The BSD 3-Clause license 23 | org.codehaus.jettison--jettison--1.1=Apache License, Version 2.0 24 | ldapsdk--ldapsdk--4.1= 25 | javax.servlet--servlet-api--2.5=CDDL 1.1 26 | javax.transaction--transaction-api--1.1=CDDL 1.0 27 | javax.servlet--jsp-api--2.0=CDDL 1.1 28 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | source 'https://rubygems.org' 17 | gem 'github-pages' 18 | gem 'rouge' 19 | gem 'jekyll-oembed', :require => 'jekyll_oembed' 20 | # End Gemfile 21 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | markdown: kramdown 17 | 18 | destination: target 19 | exclude: [README.md,Gemfile*] 20 | 21 | # End _config.yml 22 | 23 | -------------------------------------------------------------------------------- /docs/_data/navigation.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | topnav: 17 | 18 | - title: Documentation 19 | subcategories: 20 | - title: REST API 21 | url: rest-api.html 22 | - title: Programmatic API 23 | url: programmatic-api.html 24 | - title: Java API Docs 25 | url: api/java/index.html 26 | - title: Scala API Docs 27 | url: api/scala/index.html#org.apache.livy.scalaapi.package 28 | 29 | - title: Apache 30 | subcategories: 31 | - title: Apache Software Foundation 32 | url: http://www.apache.org/foundation/how-it-works.html 33 | - title: Apache License 34 | url: http://www.apache.org/licenses/ 35 | - title: Sponsorship 36 | url: http://www.apache.org/foundation/sponsorship 37 | - title: Thanks 38 | url: http://www.apache.org/foundation/thanks.html 39 | -------------------------------------------------------------------------------- /docs/_data/project.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # Apache Project configurations 17 | # 18 | name: Apache Livy 19 | version: 0.9.0-incubating-SNAPSHOT 20 | 21 | podling: true 22 | -------------------------------------------------------------------------------- /docs/_includes/JB/setup: -------------------------------------------------------------------------------- 1 | {% capture jbcache %} 2 | 5 | {% if site.JB.setup.provider == "custom" %} 6 | {% include custom/setup %} 7 | {% else %} 8 | {% if site.safe and site.JB.BASE_PATH and site.JB.BASE_PATH != '' %} 9 | {% assign BASE_PATH = site.JB.BASE_PATH %} 10 | {% assign HOME_PATH = site.JB.BASE_PATH %} 11 | {% else %} 12 | {% assign BASE_PATH = nil %} 13 | {% assign HOME_PATH = "/" %} 14 | {% endif %} 15 | 16 | {% if site.JB.ASSET_PATH %} 17 | {% assign ASSET_PATH = site.JB.ASSET_PATH %} 18 | {% else %} 19 | {% capture ASSET_PATH %}{{ BASE_PATH }}/assets/themes/{{ layout.theme.name }}{% endcapture %} 20 | {% endif %} 21 | {% endif %} 22 | {% endcapture %}{% assign jbcache = nil %} 23 | -------------------------------------------------------------------------------- /docs/_includes/themes/apache/default.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{ page.title }} 6 | {% if page.description %}{% endif %} 7 | 8 | 9 | 10 | 11 | 12 | 13 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 28 | 29 | 30 | 31 | 32 | 33 | {% include themes/apache/_navigation.html %} 34 | 35 |

36 | {{ content }} 37 |
38 |
39 | 40 | {% include themes/apache/footer.html %} 41 |
42 |
43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /docs/_includes/themes/apache/footer.html: -------------------------------------------------------------------------------- 1 | 37 | -------------------------------------------------------------------------------- /docs/_includes/themes/apache/page.html: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |
8 | {{ content }} 9 |
10 |
11 | -------------------------------------------------------------------------------- /docs/_includes/themes/apache/settings.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to you under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | theme : 17 | name : apache 18 | -------------------------------------------------------------------------------- /docs/_layouts/default.html: -------------------------------------------------------------------------------- 1 | --- 2 | theme : 3 | name : apache 4 | --- 5 | {% include JB/setup %} 6 | {% include themes/apache/default.html %} 7 | -------------------------------------------------------------------------------- /docs/_layouts/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | {% include JB/setup %} 5 | {% include themes/apache/page.html %} 6 | -------------------------------------------------------------------------------- /docs/_layouts/post.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | --- 4 | {% include JB/setup %} 5 | {% include themes/apache/post.html %} 6 | -------------------------------------------------------------------------------- /docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/bootstrap/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /docs/assets/themes/apache/bootstrap/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/bootstrap/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /docs/assets/themes/apache/bootstrap/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/bootstrap/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /docs/assets/themes/apache/img/egg-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/img/egg-logo.png -------------------------------------------------------------------------------- /docs/assets/themes/apache/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/docs/assets/themes/apache/img/logo.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Livy Docs 4 | tagline: Documentation 5 | --- 6 | 24 | 25 | {% include JB/setup %} 26 | 27 | # Apache Livy Documentation 28 | 29 | ### [REST API](rest-api.html) 30 | 31 | ### [Programmatic API](programmatic-api.html) -------------------------------------------------------------------------------- /examples/src/main/python/pi_app.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | from __future__ import print_function 19 | 20 | import sys 21 | from random import random 22 | from operator import add 23 | 24 | from livy.client import HttpClient 25 | 26 | if __name__ == "__main__": 27 | """ 28 | Usage: pi_app [livy url] [slices] 29 | 30 | To run this Python script you need to install livy-python-api-*version*.tar.gz with 31 | easy_install first. 32 | 33 | python /pathTo/pi_app.py http://:8998 2 34 | """ 35 | 36 | if len(sys.argv) != 3: 37 | print("Usage: pi_app ", file=sys.stderr) 38 | exit(-1) 39 | 40 | slices = int(sys.argv[2]) 41 | samples = 100000 * slices 42 | 43 | client = HttpClient(sys.argv[1]) 44 | 45 | def f(_): 46 | x = random() * 2 - 1 47 | y = random() * 2 - 1 48 | return 1 if x ** 2 + y ** 2 <= 1 else 0 49 | 50 | def pi_job(context): 51 | count = context.sc.parallelize(range(1, samples + 1), slices).map(f).reduce(add) 52 | return 4.0 * count / samples 53 | 54 | pi = client.submit(pi_job).result() 55 | 56 | print("Pi is roughly %f" % pi) 57 | client.stop(True) 58 | 59 | -------------------------------------------------------------------------------- /integration-test/src/test/resources/batch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env /python 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | import sys 20 | from pyspark import SparkContext 21 | 22 | output = sys.argv[1] 23 | sc = SparkContext(appName="PySpark Test") 24 | try: 25 | sc.parallelize(range(100), 10).map(lambda x: (x, x * 2)).saveAsTextFile(output) 26 | finally: 27 | sc.stop() 28 | -------------------------------------------------------------------------------- /integration-test/src/test/resources/cluster.spec.template: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # type of cluster you are running integration tests against 19 | # use mini to test against the mini cluster, use external to test 20 | # against a real external cluster 21 | cluster.type=external 22 | 23 | # The authentication scheme used by your external cluster 24 | # The potential values for authScheme are kerberos for kerberos auth, 25 | # basic for basic auth, or nothing for no authentication 26 | authScheme= 27 | 28 | # The location of hadoop config files 29 | configDir= 30 | 31 | # The location in hdfs to store temporary files 32 | hdfsScratchDir= 33 | 34 | # Endpoint for talking to livy 35 | livyEndpoint= 36 | 37 | # username and password for authentication 38 | user= 39 | password= 40 | 41 | # SSL Certificate Path 42 | sslCertPath= 43 | 44 | # principal and keytab Path for kerberos authentication 45 | principal= 46 | keytabPath= 47 | -------------------------------------------------------------------------------- /integration-test/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | test.appender=file 20 | log4j.rootCategory=DEBUG, ${test.appender} 21 | log4j.appender.file=org.apache.log4j.FileAppender 22 | log4j.appender.file.append=true 23 | log4j.appender.file.file=target/unit-tests.log 24 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 26 | 27 | # Tests that launch java subprocesses can set the "test.appender" system property to 28 | # "console" to avoid having the child process's logs overwrite the unit test's 29 | # log file. 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t: %m%n 34 | 35 | # Silence 3rd party libraries. 36 | log4j.logger.org.apache.livy.rsc=INFO 37 | log4j.logger.org.apache.livy.shaded=INFO 38 | log4j.logger.com.decodified=WARN 39 | log4j.logger.com.ning.http=WARN 40 | log4j.logger.org.apache.hadoop=WARN 41 | log4j.logger.org.eclipse.jetty=WARN 42 | log4j.logger.net.schmizz=WARN 43 | -------------------------------------------------------------------------------- /integration-test/src/test/resources/rtest.R: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | library(SparkR) 19 | 20 | # Initialize SparkSession 21 | sparkR.session(appName = "SparkR-DataFrame-example") 22 | 23 | # Create a simple local data.frame 24 | localDF <- data.frame(name=c("John", "Smith", "Sarah"), age=c(19, 23, 18)) 25 | 26 | # Convert local data frame to a SparkDataFrame 27 | df <- createDataFrame(localDF) 28 | 29 | # Print its schema 30 | printSchema(df) 31 | 32 | # Stop the SparkContext now 33 | sparkR.session.stop() 34 | -------------------------------------------------------------------------------- /integration-test/src/thriftserver/main/scala/org/apache/livy/test/framework/BaseThriftIntegrationTestSuite.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.framework 19 | 20 | import java.sql.{Connection, DriverManager, ResultSet} 21 | 22 | class BaseThriftIntegrationTestSuite extends BaseIntegrationTestSuite { 23 | private var jdbcUri: String = _ 24 | 25 | override def beforeAll(): Unit = { 26 | cluster = Cluster.get() 27 | // The JDBC endpoint must contain a valid value 28 | assert(cluster.jdbcEndpoint.isDefined) 29 | jdbcUri = cluster.jdbcEndpoint.get 30 | } 31 | 32 | def checkQuery(connection: Connection, query: String)(validate: ResultSet => Unit): Unit = { 33 | val ps = connection.prepareStatement(query) 34 | try { 35 | val rs = ps.executeQuery() 36 | try { 37 | validate(rs) 38 | } finally { 39 | rs.close() 40 | } 41 | } finally { 42 | ps.close() 43 | } 44 | } 45 | 46 | def withConnection[T](f: Connection => T): T = { 47 | val connection = DriverManager.getConnection(jdbcUri) 48 | try { 49 | f(connection) 50 | } finally { 51 | connection.close() 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /python-api/.gitattributes: -------------------------------------------------------------------------------- 1 | livy/_version.py export-subst 2 | src/main/python/livy/_version.py export-subst 3 | -------------------------------------------------------------------------------- /python-api/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versioneer.py 2 | include livy/_version.py 3 | include src/main/python/livy/_version.py 4 | -------------------------------------------------------------------------------- /python-api/setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [tool:pytest] 5 | addopts = --verbose 6 | python_files = src/test/python/*/*.py 7 | cache_dir = target/.pytest_cache 8 | -------------------------------------------------------------------------------- /python-api/setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | from setuptools import setup 19 | 20 | DESCRIPTION = "A simple Python API for Livy powered by requests" 21 | 22 | CLASSIFIERS = [ 23 | 'Development Status :: 1 - Planning', 24 | 'Intended Audience :: Developers', 25 | 'Operating System :: OS Independent', 26 | 'Programming Language :: Python :: 2.7', 27 | 'Topic :: Software Development :: Libraries :: Python Modules', 28 | ] 29 | 30 | requirements = [ 31 | 'cloudpickle>=0.2.1', 32 | 'configparser>=3.5.0', 33 | 'future>=0.15.2', 34 | 'futures>=3.0.5', 35 | 'mock~=3.0.5', 36 | 'requests>=2.10.0', 37 | 'responses>=0.5.1', 38 | 'requests-kerberos>=0.11.0', 39 | ] 40 | 41 | setup( 42 | name='livy-python-api', 43 | version="0.9.0-incubating-SNAPSHOT", 44 | packages=["livy", "livy-tests"], 45 | package_dir={ 46 | "": "src/main/python", 47 | "livy-tests": "src/test/python/livy-tests", 48 | }, 49 | url='https://github.com/apache/incubator-livy', 50 | author_email='user@livy.incubator.apache.org', 51 | license='Apache License, Version 2.0', 52 | description=DESCRIPTION, 53 | platforms=['any'], 54 | keywords='livy pyspark development', 55 | classifiers=CLASSIFIERS, 56 | install_requires=requirements, 57 | setup_requires=['pytest-runner', 'flake8'], 58 | tests_require=['pytest'] 59 | ) 60 | -------------------------------------------------------------------------------- /python-api/src/main/python/livy/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | -------------------------------------------------------------------------------- /python-api/src/test/python/livy-tests/resources/livy-client.conf: -------------------------------------------------------------------------------- 1 | livy.uri=match 2 | spark.config=override -------------------------------------------------------------------------------- /python-api/src/test/python/livy-tests/resources/spark-defaults.conf: -------------------------------------------------------------------------------- 1 | spark.config=default -------------------------------------------------------------------------------- /python-api/src/test/python/livy-tests/resources/text_file.txt: -------------------------------------------------------------------------------- 1 | 100 2 | -------------------------------------------------------------------------------- /repl/scala-2.11/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 21 | 4.0.0 22 | org.apache.livy 23 | livy-repl_2.11 24 | 0.9.0-incubating-SNAPSHOT 25 | jar 26 | 27 | 28 | org.apache.livy 29 | livy-repl-parent 30 | 0.9.0-incubating-SNAPSHOT 31 | ../pom.xml 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /repl/scala-2.12/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 21 | 4.0.0 22 | org.apache.livy 23 | livy-repl_2.12 24 | 0.9.0-incubating-SNAPSHOT 25 | jar 26 | 27 | 28 | org.apache.livy 29 | livy-repl-parent 30 | 0.9.0-incubating-SNAPSHOT 31 | ../pom.xml 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /repl/src/main/resources/build.marker: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/repl/src/main/resources/build.marker -------------------------------------------------------------------------------- /repl/src/main/scala/org/apache/livy/repl/BypassPySparkJob.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.repl 18 | 19 | import java.nio.charset.StandardCharsets 20 | 21 | import org.apache.livy.{Job, JobContext} 22 | import org.apache.livy.sessions._ 23 | 24 | class BypassPySparkJob( 25 | serializedJob: Array[Byte], 26 | pi: PythonInterpreter) extends Job[Array[Byte]] { 27 | 28 | override def call(jc: JobContext): Array[Byte] = { 29 | val resultByteArray = pi.pysparkJobProcessor.processBypassJob(serializedJob) 30 | val resultString = new String(resultByteArray, StandardCharsets.UTF_8) 31 | if (resultString.startsWith("Client job error:")) { 32 | throw new PythonJobException(resultString) 33 | } 34 | resultByteArray 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /repl/src/main/scala/org/apache/livy/repl/Interpreter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.repl 19 | 20 | import org.json4s.JArray 21 | import org.json4s.JObject 22 | 23 | object Interpreter { 24 | abstract class ExecuteResponse 25 | 26 | case class ExecuteSuccess(content: JObject) extends ExecuteResponse 27 | case class ExecuteError(ename: String, 28 | evalue: String, 29 | traceback: Seq[String] = Seq()) extends ExecuteResponse 30 | case class ExecuteIncomplete() extends ExecuteResponse 31 | case class ExecuteAborted(message: String) extends ExecuteResponse 32 | } 33 | 34 | trait Interpreter { 35 | import Interpreter._ 36 | 37 | def kind: String 38 | 39 | /** 40 | * Start the Interpreter. 41 | */ 42 | def start(): Unit 43 | 44 | /** 45 | * Execute the code and return the result, it may 46 | * take some time to execute. 47 | */ 48 | protected[repl] def execute(code: String): ExecuteResponse 49 | 50 | protected[repl] def complete(code: String, cursor: Int): Array[String] 51 | = Array() 52 | 53 | /** Shut down the interpreter. */ 54 | def close(): Unit 55 | } 56 | -------------------------------------------------------------------------------- /repl/src/main/scala/org/apache/livy/repl/PySparkJobProcessor.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.repl 18 | 19 | trait PySparkJobProcessor { 20 | def processBypassJob(job: Array[Byte]): Array[Byte] 21 | 22 | def addFile(path: String) 23 | 24 | def addPyFile(path: String) 25 | 26 | def getLocalTmpDirPath: String 27 | } 28 | -------------------------------------------------------------------------------- /repl/src/main/scala/org/apache/livy/repl/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy 19 | 20 | import org.json4s.JField 21 | 22 | package object repl { 23 | type MimeTypeMap = List[JField] 24 | 25 | val APPLICATION_JSON = "application/json" 26 | val APPLICATION_LIVY_TABLE_JSON = "application/vnd.livy.table.v1+json" 27 | val IMAGE_PNG = "image/png" 28 | val TEXT_PLAIN = "text/plain" 29 | } 30 | -------------------------------------------------------------------------------- /repl/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | test.appender=file 20 | log4j.rootCategory=INFO, ${test.appender} 21 | log4j.appender.file=org.apache.log4j.FileAppender 22 | log4j.appender.file.append=true 23 | log4j.appender.file.file=target/unit-tests.log 24 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 26 | 27 | # Tests that launch java subprocesses can set the "test.appender" system property to 28 | # "console" to avoid having the child process's logs overwrite the unit test's 29 | # log file. 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%t: %m%n 34 | 35 | # Ignore messages below warning level from Jetty, because it's a bit verbose 36 | log4j.logger.org.spark-project.jetty=WARN 37 | org.spark-project.jetty.LEVEL=WARN 38 | -------------------------------------------------------------------------------- /repl/src/test/scala/org/apache/livy/repl/BaseInterpreterSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.repl 19 | 20 | import org.scalatest.{FlatSpec, Matchers} 21 | 22 | import org.apache.livy.LivyBaseUnitTestSuite 23 | 24 | abstract class BaseInterpreterSpec extends FlatSpec with Matchers with LivyBaseUnitTestSuite { 25 | 26 | def createInterpreter(): Interpreter 27 | 28 | def withInterpreter(testCode: Interpreter => Any): Unit = { 29 | val interpreter = createInterpreter() 30 | try { 31 | interpreter.start() 32 | testCode(interpreter) 33 | } finally { 34 | interpreter.close() 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/BypassJobStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc; 19 | 20 | import org.apache.livy.JobHandle; 21 | 22 | public class BypassJobStatus { 23 | 24 | public final JobHandle.State state; 25 | public final byte[] result; 26 | public final String error; 27 | 28 | public BypassJobStatus(JobHandle.State state, byte[] result, String error) { 29 | this.state = state; 30 | this.result = result; 31 | this.error = error; 32 | } 33 | 34 | BypassJobStatus() { 35 | this(null, null, null); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/ContextInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc; 19 | 20 | /** 21 | * Information about a running RSC instance. 22 | */ 23 | class ContextInfo { 24 | 25 | final String remoteAddress; 26 | final int remotePort; 27 | final String clientId; 28 | final String secret; 29 | 30 | ContextInfo(String remoteAddress, int remotePort, String clientId, String secret) { 31 | this.remoteAddress = remoteAddress; 32 | this.remotePort = remotePort; 33 | this.clientId = clientId; 34 | this.secret = secret; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/DriverProcessInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc; 19 | 20 | import io.netty.util.concurrent.Promise; 21 | 22 | /** 23 | * Information about driver process and @{@link ContextInfo} 24 | */ 25 | public class DriverProcessInfo { 26 | 27 | private Promise contextInfo; 28 | private transient Process driverProcess; 29 | 30 | public DriverProcessInfo(Promise contextInfo, Process driverProcess) { 31 | this.contextInfo = contextInfo; 32 | this.driverProcess = driverProcess; 33 | } 34 | 35 | public Promise getContextInfo() { 36 | return contextInfo; 37 | } 38 | 39 | public Process getDriverProcess() { 40 | return driverProcess; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/FutureListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc; 19 | 20 | /** A simplified future listener for netty futures. See Utils.addListener(). */ 21 | public abstract class FutureListener { 22 | 23 | public void onSuccess(T result) throws Exception { } 24 | 25 | public void onFailure(Throwable error) throws Exception { } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/PingJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | /** A job that can be used to check for liveness of the remote context. */ 24 | public class PingJob implements Job { 25 | 26 | @Override 27 | public Void call(JobContext jc) { 28 | return null; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/ReplJobResults.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.rsc; 18 | 19 | import org.apache.livy.rsc.driver.Statement; 20 | 21 | public class ReplJobResults { 22 | public final Statement[] statements; 23 | 24 | public ReplJobResults(Statement[] statements) { 25 | this.statements = statements; 26 | } 27 | 28 | public ReplJobResults() { 29 | this(null); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/driver/AddFileJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.rsc.driver; 18 | 19 | import org.apache.livy.Job; 20 | import org.apache.livy.JobContext; 21 | 22 | public class AddFileJob implements Job { 23 | 24 | private final String path; 25 | 26 | AddFileJob() { 27 | this(null); 28 | } 29 | 30 | public AddFileJob(String path) { 31 | this.path = path; 32 | } 33 | 34 | @Override 35 | public Object call(JobContext jc) throws Exception { 36 | JobContextImpl jobContextImpl = (JobContextImpl)jc; 37 | jobContextImpl.addFile(path); 38 | return null; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/driver/AddJarJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.driver; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | public class AddJarJob implements Job { 24 | 25 | private final String path; 26 | 27 | // For serialization. 28 | private AddJarJob() { 29 | this(null); 30 | } 31 | 32 | public AddJarJob(String path) { 33 | this.path = path; 34 | } 35 | 36 | @Override 37 | public Object call(JobContext jc) throws Exception { 38 | JobContextImpl jobContextImpl = (JobContextImpl)jc; 39 | jobContextImpl.addJarOrPyFile(path); 40 | return null; 41 | } 42 | } -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/driver/BypassJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.driver; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import org.apache.livy.Job; 23 | import org.apache.livy.JobContext; 24 | import org.apache.livy.client.common.BufferUtils; 25 | import org.apache.livy.client.common.Serializer; 26 | 27 | class BypassJob implements Job { 28 | 29 | private final Serializer serializer; 30 | private final byte[] serializedJob; 31 | 32 | BypassJob(Serializer serializer, byte[] serializedJob) { 33 | this.serializer = serializer; 34 | this.serializedJob = serializedJob; 35 | } 36 | 37 | @Override 38 | public byte[] call(JobContext jc) throws Exception { 39 | Job job = (Job) serializer.deserialize(ByteBuffer.wrap(serializedJob)); 40 | Object result = job.call(jc); 41 | byte[] serializedResult; 42 | if (result != null) { 43 | ByteBuffer data = serializer.serialize(result); 44 | serializedResult = BufferUtils.toByteArray(data); 45 | } else { 46 | serializedResult = null; 47 | } 48 | return serializedResult; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/driver/MutableClassLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.driver; 19 | 20 | import java.net.URL; 21 | import java.net.URLClassLoader; 22 | 23 | class MutableClassLoader extends URLClassLoader { 24 | 25 | MutableClassLoader(ClassLoader parent) { 26 | super(new URL[] { }, parent); 27 | } 28 | 29 | @Override 30 | public void addURL(URL url) { 31 | super.addURL(url); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/driver/Statement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.driver; 19 | 20 | import java.util.concurrent.atomic.AtomicReference; 21 | 22 | import com.fasterxml.jackson.annotation.JsonRawValue; 23 | 24 | public class Statement { 25 | public final Integer id; 26 | public final String code; 27 | public final AtomicReference state; 28 | @JsonRawValue 29 | public volatile String output; 30 | public double progress; 31 | public long started = 0; 32 | public long completed = 0; 33 | 34 | public Statement(Integer id, String code, StatementState state, String output) { 35 | this.id = id; 36 | this.code = code; 37 | this.state = new AtomicReference<>(state); 38 | this.output = output; 39 | this.progress = 0.0; 40 | } 41 | 42 | public Statement() { 43 | this(null, null, null, null); 44 | } 45 | 46 | public boolean compareAndTransit(final StatementState from, final StatementState to) { 47 | if (state.compareAndSet(from, to)) { 48 | StatementState.validate(from, to); 49 | return true; 50 | } 51 | return false; 52 | } 53 | 54 | public void updateProgress(double p) { 55 | if (this.state.get().isOneOf(StatementState.Cancelled, StatementState.Available)) { 56 | this.progress = 1.0; 57 | } else { 58 | this.progress = p; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /** 19 | * Livy Remote Spark Context used by Livy's Interactive Sessions 20 | */ 21 | package org.apache.livy.rsc; 22 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/rpc/README.md: -------------------------------------------------------------------------------- 1 | Spark Client RPC 2 | ================ 3 | 4 | Basic flow of events: 5 | 6 | - Client side creates an RPC server 7 | - Client side spawns RemoteDriver, which manages the SparkContext, and provides a secret 8 | - Client side sets up a timer to wait for RemoteDriver to connect back 9 | - RemoteDriver connects back to client, SASL handshake ensues 10 | - Connection is established and now there's a session between the client and the driver. 11 | 12 | Features of the RPC layer: 13 | 14 | - All messages serialized via Kryo 15 | - All messages are replied to. It's either an empty "ack" or an actual response - that depends 16 | on the message. 17 | - RPC send API is asynchronous - callers get a future that can be used to wait for the message. 18 | - Currently, no connection retry. If connection goes down, both sides tear down the session. 19 | 20 | Notes: 21 | 22 | - Because serialization is using Kryo, types need explicit empty constructors or things will 23 | fail to deserialize. This can be seen in the way exceptions are propagated - the throwing 24 | side sends just a string stack trace to the remote, because certain fields on exceptions 25 | don't have empty constructors. 26 | - The above is especially important because at the moment there's no way to register custom 27 | serializers in the RPC library. 28 | 29 | Future work: 30 | 31 | - Random initial RPC id + id wrapping. 32 | - SSL / security in general. 33 | -------------------------------------------------------------------------------- /rsc/src/main/java/org/apache/livy/rsc/rpc/RpcException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.rpc; 19 | 20 | public class RpcException extends RuntimeException { 21 | 22 | RpcException(String remoteStackTrace) { 23 | super(remoteStackTrace); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /rsc/src/main/resources/META-INF/services/org.apache.livy.LivyClientFactory: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | org.apache.livy.rsc.RSCClientFactory 18 | -------------------------------------------------------------------------------- /rsc/src/test/java/org/apache/livy/rsc/driver/TestRSCDriver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.rsc.driver; 19 | 20 | import org.apache.spark.SparkConf; 21 | import org.junit.Test; 22 | 23 | import org.apache.livy.rsc.BaseProtocol; 24 | import org.apache.livy.rsc.RSCConf; 25 | 26 | public class TestRSCDriver { 27 | @Test(expected = IllegalArgumentException.class) 28 | public void testCancelJobAfterInitializeFailed() 29 | throws Exception { 30 | //use empty Conf to trigger initialize throw IllegalArgumentException 31 | RSCDriver rscDriver = new RSCDriver(new SparkConf(), new RSCConf()); 32 | //add asynchronous dummy job request to trigger cancel job failure 33 | rscDriver.handle(null, new BaseProtocol.BypassJobRequest("RequestId", null, null, false)); 34 | rscDriver.run(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /rsc/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | log4j.rootCategory=WARN, console 20 | log4j.appender.console=org.apache.log4j.ConsoleAppender 21 | log4j.appender.console.target=System.err 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 24 | 25 | # Silence some verbose logs. 26 | log4j.logger.org.spark-project.jetty=WARN 27 | -------------------------------------------------------------------------------- /scala-api/scala-2.11/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | org.apache.livy 21 | livy-scala-api_2.11 22 | 0.9.0-incubating-SNAPSHOT 23 | jar 24 | 25 | 26 | org.apache.livy 27 | livy-scala-api-parent 28 | 0.9.0-incubating-SNAPSHOT 29 | ../pom.xml 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /scala-api/scala-2.12/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 4.0.0 20 | org.apache.livy 21 | livy-scala-api_2.12 22 | 0.9.0-incubating-SNAPSHOT 23 | jar 24 | 25 | 26 | org.apache.livy 27 | livy-scala-api-parent 28 | 0.9.0-incubating-SNAPSHOT 29 | ../pom.xml 30 | 31 | 32 | -------------------------------------------------------------------------------- /scala-api/src/main/resources/build.marker: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/scala-api/src/main/resources/build.marker -------------------------------------------------------------------------------- /scala-api/src/main/scala/org/apache/livy/scalaapi/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy 19 | 20 | import java.util.concurrent.{ExecutionException, Future => JFuture, TimeUnit} 21 | 22 | import scala.concurrent.duration.Duration 23 | 24 | package object scalaapi { 25 | 26 | /** 27 | * A Scala Client for Livy which is a wrapper over the Java client. 28 | * @constructor Creates a Scala client. 29 | * @param livyJavaClient the Java client of Livy. 30 | * {{{ 31 | * import org.apache.livy._ 32 | * import org.apache.livy.scalaapi._ 33 | * val url = "http://example.com" 34 | * val livyJavaClient = new LivyClientBuilder(false).setURI(new URI(url))).build() 35 | * val livyScalaClient = livyJavaClient.asScalaClient 36 | * }}} 37 | */ 38 | implicit class ScalaWrapper(livyJavaClient: LivyClient) { 39 | def asScalaClient: LivyScalaClient = new LivyScalaClient(livyJavaClient) 40 | } 41 | 42 | private[livy] def getJavaFutureResult[T](jFuture: JFuture[T], 43 | atMost: Duration = Duration.Undefined): T = { 44 | try { 45 | if (!atMost.isFinite()) jFuture.get else jFuture.get(atMost.toMillis, TimeUnit.MILLISECONDS) 46 | } catch { 47 | case executionException: ExecutionException => throw executionException.getCause 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /scala-api/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # Set everything to be logged to the file target/unit-tests.log 18 | test.appender=file 19 | log4j.rootCategory=INFO, ${test.appender} 20 | log4j.appender.file=org.apache.log4j.FileAppender 21 | log4j.appender.file.append=true 22 | log4j.appender.file.file=target/unit-tests.log 23 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 24 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 25 | # Tests that launch java subprocesses can set the "test.appender" system property to 26 | # "console" to avoid having the child process's logs overwrite the unit test's 27 | # log file. 28 | log4j.appender.console=org.apache.log4j.ConsoleAppender 29 | log4j.appender.console.target=System.err 30 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 31 | log4j.appender.console.layout.ConversionPattern=%t: %m%n 32 | # Ignore messages below warning level from Jetty, because it's a bit verbose 33 | log4j.logger.org.spark-project.jetty=WARN 34 | org.spark-project.jetty.LEVEL=WARN 35 | -------------------------------------------------------------------------------- /scala-api/src/test/scala/org/apache/livy/scalaapi/ScalaClientTestUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.scalaapi 18 | 19 | import java.util.Random 20 | import java.util.concurrent.{CountDownLatch, TimeUnit} 21 | 22 | import scala.collection.mutable.ArrayBuffer 23 | import scala.concurrent.{Await, Future} 24 | import scala.concurrent.duration._ 25 | 26 | import org.scalatest.FunSuite 27 | 28 | import org.apache.livy.LivyBaseUnitTestSuite 29 | 30 | object ScalaClientTestUtils extends FunSuite with LivyBaseUnitTestSuite { 31 | 32 | val Timeout = 40 33 | 34 | def helloJob(context: ScalaJobContext): String = "hello" 35 | 36 | def throwExceptionJob(context: ScalaJobContext): Unit = throw new CustomTestFailureException 37 | 38 | def simpleSparkJob(context: ScalaJobContext): Long = { 39 | val r = new Random 40 | val count = 5 41 | val partitions = Math.min(r.nextInt(10) + 1, count) 42 | val buffer = new ArrayBuffer[Int]() 43 | for (a <- 1 to count) { 44 | buffer += r.nextInt() 45 | } 46 | context.sc.parallelize(buffer, partitions).count() 47 | } 48 | 49 | def assertAwait(lock: CountDownLatch): Unit = { 50 | assert(lock.await(Timeout, TimeUnit.SECONDS) == true) 51 | } 52 | 53 | def assertTestPassed[T](future: Future[T], expectedValue: T): Unit = { 54 | val result = Await.result(future, Timeout second) 55 | assert(result === expectedValue) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/css/livy-ui.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | body { 19 | padding: 15px 0; 20 | } 21 | 22 | pre { 23 | font-size: 12px; 24 | line-height: 18px; 25 | padding: 6px; 26 | margin: 0; 27 | word-break: break-word; 28 | white-space: pre-wrap; 29 | } 30 | 31 | #session-log pre { 32 | max-height: 80%; 33 | overflow: scroll; 34 | } 35 | 36 | #session-log h4 div { 37 | display: inline-block; 38 | } 39 | 40 | td .progress { 41 | margin: 0; 42 | } 43 | 44 | .with-scroll-bar { 45 | display: block; 46 | overflow-y: scroll; 47 | max-height: 200px; 48 | } 49 | 50 | #session-summary { 51 | margin: 20px 0; 52 | } -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/server/src/main/resources/org/apache/livy/server/ui/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/img/livy-mini-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-livy/d6822aaa04df87fb17e7e061d39cbc05cedae5c3/server/src/main/resources/org/apache/livy/server/ui/static/img/livy-mini-logo.png -------------------------------------------------------------------------------- /server/src/main/resources/org/apache/livy/server/ui/static/js/dataTables.bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | DataTables Bootstrap 3 integration 3 | ©2011-2015 SpryMedia Ltd - datatables.net/license 4 | */ 5 | (function(b){"function"===typeof define&&define.amd?define(["jquery","datatables.net"],function(a){return b(a,window,document)}):"object"===typeof exports?module.exports=function(a,d){a||(a=window);if(!d||!d.fn.dataTable)d=require("datatables.net")(a,d).$;return b(d,a,a.document)}:b(jQuery,window,document)})(function(b,a,d,m){var f=b.fn.dataTable;b.extend(!0,f.defaults,{dom:"<'row'<'col-sm-6'l><'col-sm-6'f>><'row'<'col-sm-12'tr>><'row'<'col-sm-5'i><'col-sm-7'p>>",renderer:"bootstrap"});b.extend(f.ext.classes, 6 | {sWrapper:"dataTables_wrapper form-inline dt-bootstrap",sFilterInput:"form-control input-sm",sLengthSelect:"form-control input-sm",sProcessing:"dataTables_processing panel panel-default"});f.ext.renderer.pageButton.bootstrap=function(a,h,r,s,j,n){var o=new f.Api(a),t=a.oClasses,k=a.oLanguage.oPaginate,u=a.oLanguage.oAria.paginate||{},e,g,p=0,q=function(d,f){var l,h,i,c,m=function(a){a.preventDefault();!b(a.currentTarget).hasClass("disabled")&&o.page()!=a.data.action&&o.page(a.data.action).draw("page")}; 7 | l=0;for(h=f.length;l",{"class":t.sPageButton+" "+g,id:0===r&&"string"===typeof c?a.sTableId+"_"+c:null}).append(b("",{href:"#", 8 | "aria-controls":a.sTableId,"aria-label":u[c],"data-dt-idx":p,tabindex:a.iTabIndex}).html(e)).appendTo(d),a.oApi._fnBindAction(i,{action:c},m),p++)}},i;try{i=b(h).find(d.activeElement).data("dt-idx")}catch(v){}q(b(h).empty().html('
    ').children("ul"),s);i!==m&&b(h).find("[data-dt-idx="+i+"]").focus()};return f}); 9 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/AccessFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.server 19 | 20 | import javax.servlet._ 21 | import javax.servlet.http.{HttpServletRequest, HttpServletResponse} 22 | 23 | private[livy] class AccessFilter(accessManager: AccessManager) extends Filter { 24 | 25 | override def init(filterConfig: FilterConfig): Unit = {} 26 | 27 | override def doFilter(request: ServletRequest, 28 | response: ServletResponse, 29 | chain: FilterChain): Unit = { 30 | val httpRequest = request.asInstanceOf[HttpServletRequest] 31 | val remoteUser = httpRequest.getRemoteUser 32 | if (accessManager.isUserAllowed(remoteUser)) { 33 | chain.doFilter(request, response) 34 | } else { 35 | val httpServletResponse = response.asInstanceOf[HttpServletResponse] 36 | httpServletResponse.sendError(HttpServletResponse.SC_FORBIDDEN, 37 | "User not authorised to use Livy.") 38 | } 39 | } 40 | 41 | override def destroy(): Unit = {} 42 | } 43 | 44 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/ApiVersions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.server 19 | 20 | /** 21 | * This enum defines Livy's API versions. 22 | * [[org.apache.livy.server.AbstractApiVersioningSupport]] uses this for API version checking. 23 | * 24 | * Version is defined as .. 25 | * When making backward compatible change (adding methods/fields), bump minor version. 26 | * When making backward incompatible change (renaming/removing methods/fields), bump major version. 27 | * This ensures our users can safely migrate to a newer API version if major version is unchanged. 28 | */ 29 | object ApiVersions extends Enumeration { 30 | type ApiVersions = Value 31 | // ApiVersions are ordered and the ordering is defined by the order of Value() calls. 32 | // Please make sure API version is defined in ascending order (Older API before newer). 33 | // AbstractApiVersioningSupport relies on the ordering. 34 | val v0_1 = Value("0.1") 35 | } 36 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/CsrfFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | package org.apache.livy.server 20 | 21 | import javax.servlet._ 22 | import javax.servlet.http.{HttpServletRequest, HttpServletResponse} 23 | 24 | class CsrfFilter extends Filter { 25 | 26 | val METHODS_TO_IGNORE = Set("GET", "OPTIONS", "HEAD"); 27 | 28 | val HEADER_NAME = "X-Requested-By"; 29 | 30 | override def init(filterConfig: FilterConfig): Unit = {} 31 | 32 | override def doFilter(request: ServletRequest, 33 | response: ServletResponse, 34 | chain: FilterChain): Unit = { 35 | val httpRequest = request.asInstanceOf[HttpServletRequest] 36 | 37 | if (!METHODS_TO_IGNORE.contains(httpRequest.getMethod) 38 | && httpRequest.getHeader(HEADER_NAME) == null) { 39 | response.asInstanceOf[HttpServletResponse].sendError(HttpServletResponse.SC_BAD_REQUEST, 40 | "Missing Required Header for CSRF protection.") 41 | } else { 42 | chain.doFilter(request, response) 43 | } 44 | } 45 | 46 | override def destroy(): Unit = {} 47 | } 48 | 49 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/ThriftServerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.server 19 | 20 | import javax.servlet.Servlet 21 | 22 | import org.apache.livy.LivyConf 23 | import org.apache.livy.server.recovery.SessionStore 24 | import org.apache.livy.sessions.InteractiveSessionManager 25 | 26 | /** 27 | * Its implementation starts Livy ThriftServer 28 | */ 29 | trait ThriftServerFactory { 30 | def start( 31 | livyConf: LivyConf, 32 | livySessionManager: InteractiveSessionManager, 33 | sessionStore: SessionStore, 34 | accessManager: AccessManager): Unit 35 | 36 | def stop(): Unit 37 | 38 | def getServlet(basePath: String): Servlet 39 | 40 | def getServletMappings: Seq[String] 41 | } 42 | 43 | object ThriftServerFactory { 44 | def getInstance: ThriftServerFactory = { 45 | Class.forName("org.apache.livy.thriftserver.ThriftServerFactoryImpl").newInstance() 46 | .asInstanceOf[ThriftServerFactory] 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/recovery/BlackholeStateStore.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.server.recovery 19 | 20 | import scala.reflect.ClassTag 21 | 22 | import org.apache.livy.LivyConf 23 | 24 | /** 25 | * This is a blackhole implementation of StateStore. 26 | * Livy will use this when session recovery is disabled. 27 | */ 28 | class BlackholeStateStore(livyConf: LivyConf) extends StateStore(livyConf) { 29 | def set(key: String, value: Object): Unit = {} 30 | 31 | def get[T: ClassTag](key: String): Option[T] = None 32 | 33 | def getChildren(key: String): Seq[String] = List.empty[String] 34 | 35 | def remove(key: String): Unit = {} 36 | } 37 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/server/recovery/ZooKeeperStateStore.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.server.recovery 18 | 19 | import scala.reflect.ClassTag 20 | 21 | import org.apache.livy.LivyConf 22 | 23 | class ZooKeeperStateStore( 24 | livyConf: LivyConf, 25 | zkManager: ZooKeeperManager) 26 | extends StateStore(livyConf) { 27 | 28 | private val zkKeyPrefix = livyConf.get(LivyConf.RECOVERY_ZK_STATE_STORE_KEY_PREFIX) 29 | private def prefixKey(key: String) = s"/$zkKeyPrefix/$key" 30 | 31 | override def set(key: String, value: Object): Unit = { 32 | zkManager.set(prefixKey(key), value) 33 | } 34 | 35 | override def get[T: ClassTag](key: String): Option[T] = { 36 | zkManager.get(prefixKey(key)) 37 | } 38 | 39 | override def getChildren(key: String): Seq[String] = { 40 | zkManager.getChildren(prefixKey(key)) 41 | } 42 | 43 | override def remove(key: String): Unit = { 44 | zkManager.remove(prefixKey(key)) 45 | } 46 | 47 | def getZooKeeperManager(): ZooKeeperManager = { 48 | zkManager 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/utils/Clock.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.utils 18 | 19 | /** 20 | * A lot of Livy code relies on time related functions like Thread.sleep. 21 | * To timing effects from unit test, this class is created to mock out time. 22 | * 23 | * Code in Livy should not call Thread.sleep() directly. It should call this class instead. 24 | */ 25 | object Clock { 26 | private var _sleep: Long => Unit = Thread.sleep 27 | 28 | def withSleepMethod(mockSleep: Long => Unit)(f: => Unit): Unit = { 29 | try { 30 | _sleep = mockSleep 31 | f 32 | } finally { 33 | _sleep = Thread.sleep 34 | } 35 | } 36 | 37 | def sleep: Long => Unit = _sleep 38 | } 39 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/utils/LineBufferedProcess.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.utils 19 | 20 | import org.apache.livy.{Logging, Utils} 21 | 22 | class LineBufferedProcess(process: Process, logSize: Int) extends Logging { 23 | 24 | private[this] val _inputStream = new LineBufferedStream(process.getInputStream, logSize) 25 | private[this] val _errorStream = new LineBufferedStream(process.getErrorStream, logSize) 26 | 27 | def inputLines: IndexedSeq[String] = _inputStream.lines 28 | def errorLines: IndexedSeq[String] = _errorStream.lines 29 | 30 | def inputIterator: Iterator[String] = _inputStream.iterator 31 | def errorIterator: Iterator[String] = _errorStream.iterator 32 | 33 | def destroy(): Unit = { 34 | process.destroy() 35 | } 36 | 37 | /** Returns if the process is still actively running. */ 38 | def isAlive: Boolean = Utils.isProcessAlive(process) 39 | 40 | def exitValue(): Int = { 41 | process.exitValue() 42 | } 43 | 44 | def waitFor(): Int = { 45 | val returnCode = process.waitFor() 46 | _inputStream.waitUntilClose() 47 | _errorStream.waitUntilClose() 48 | returnCode 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /server/src/main/scala/org/apache/livy/utils/LivyUncaughtException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.utils; 19 | 20 | public class LivyUncaughtException extends Exception { 21 | public LivyUncaughtException(String remoteStackTrace) { 22 | super(remoteStackTrace); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /server/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | test.appender=file 20 | log4j.rootCategory=INFO, ${test.appender} 21 | log4j.appender.file=org.apache.log4j.FileAppender 22 | log4j.appender.file.append=true 23 | log4j.appender.file.file=target/unit-tests.log 24 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 26 | 27 | # Tests that launch java subprocesses can set the "test.appender" system property to 28 | # "console" to avoid having the child process's logs overwrite the unit test's 29 | # log file. 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%t: %m%n 34 | 35 | # Silence some noisy libraries. 36 | log4j.logger.org.apache.http=WARN 37 | log4j.logger.org.apache.spark=INFO 38 | log4j.logger.org.eclipse.jetty=WARN 39 | log4j.logger.org.spark-project.jetty=WARN 40 | -------------------------------------------------------------------------------- /server/src/test/resources/spark-blacklist.conf: -------------------------------------------------------------------------------- 1 | spark.do_not_set 2 | -------------------------------------------------------------------------------- /server/src/test/scala/org/apache/livy/sessions/MockSession.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.sessions 19 | 20 | import org.apache.livy.LivyConf 21 | 22 | class MockSession(id: Int, owner: String, conf: LivyConf, name: Option[String] = None, 23 | ttl: Option[String] = None, idleTimeout: Option[String] = None) 24 | extends Session(id, name, owner, ttl, idleTimeout, conf) { 25 | case class RecoveryMetadata(id: Int) extends Session.RecoveryMetadata() 26 | 27 | override val proxyUser = None 28 | 29 | override def start(): Unit = { 30 | startedOn = Some(System.nanoTime()) 31 | } 32 | 33 | var stopped = false 34 | override protected def stopSession(): Unit = { 35 | stopped = true 36 | } 37 | 38 | override def logLines(): IndexedSeq[String] = IndexedSeq() 39 | 40 | var serverState: SessionState = SessionState.Idle 41 | override def state: SessionState = serverState 42 | 43 | override def recoveryMetadata: RecoveryMetadata = RecoveryMetadata(0) 44 | } 45 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/apps/FailingApp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.apps; 19 | 20 | import org.apache.hadoop.conf.Configuration; 21 | import org.apache.hadoop.fs.FileSystem; 22 | import org.apache.hadoop.fs.Path; 23 | 24 | public class FailingApp { 25 | 26 | public static void main(String[] args) throws Exception { 27 | if (args.length != 1) { 28 | throw new IllegalArgumentException("Missing output path."); 29 | } 30 | String output = args[0]; 31 | 32 | FileSystem fs = FileSystem.get(new Configuration()); 33 | Path out = new Path(output); 34 | fs.create(out).close(); 35 | 36 | throw new IllegalStateException("This app always fails."); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/Echo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | public class Echo implements Job { 24 | 25 | private final T value; 26 | 27 | public Echo(T value) { 28 | this.value = value; 29 | } 30 | 31 | @Override 32 | public T call(JobContext jc) { 33 | return value; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/Failure.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | public class Failure implements Job { 24 | 25 | @Override 26 | public Void call(JobContext jc) { 27 | throw new JobFailureException(); 28 | } 29 | 30 | public static class JobFailureException extends RuntimeException { 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/GetCurrentUser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import org.apache.hadoop.security.UserGroupInformation; 21 | 22 | import org.apache.livy.Job; 23 | import org.apache.livy.JobContext; 24 | 25 | public class GetCurrentUser implements Job { 26 | 27 | @Override 28 | public String call(JobContext jc) throws Exception { 29 | return UserGroupInformation.getCurrentUser().getUserName(); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/SharedVariableCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import java.util.NoSuchElementException; 21 | 22 | import org.apache.livy.Job; 23 | import org.apache.livy.JobContext; 24 | 25 | public class SharedVariableCounter implements Job { 26 | 27 | private final String name; 28 | 29 | public SharedVariableCounter(String name) { 30 | this.name = name; 31 | } 32 | 33 | @Override 34 | public Integer call(JobContext jc) { 35 | Integer value = -1; 36 | 37 | try { 38 | value = jc.getSharedObject(name); 39 | } catch (NoSuchElementException e) { 40 | jc.setSharedObject(name, value); 41 | } 42 | 43 | Integer newValue = value + 1; 44 | jc.setSharedObject(name, newValue); 45 | 46 | return newValue; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/Sleeper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | public class Sleeper implements Job { 24 | 25 | private final long millis; 26 | 27 | public Sleeper(long millis) { 28 | this.millis = millis; 29 | } 30 | 31 | @Override 32 | public Void call(JobContext jc) throws Exception { 33 | Thread.sleep(millis); 34 | return null; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/SmallCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.Random; 23 | 24 | import org.apache.livy.Job; 25 | import org.apache.livy.JobContext; 26 | 27 | public class SmallCount implements Job { 28 | 29 | private final int count; 30 | 31 | public SmallCount(int count) { 32 | this.count = count; 33 | } 34 | 35 | @Override 36 | public Long call(JobContext jc) { 37 | Random r = new Random(); 38 | int partitions = Math.min(r.nextInt(10) + 1, count); 39 | 40 | List elements = new ArrayList<>(count); 41 | for (int i = 0; i < count; i++) { 42 | elements.add(r.nextInt()); 43 | } 44 | 45 | return jc.sc().parallelize(elements, partitions).count(); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /test-lib/src/main/java/org/apache/livy/test/jobs/VoidJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | public class VoidJob implements Job { 24 | @Override 25 | public Void call(JobContext jc) { 26 | return null; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test-lib/src/main/resources/testweet.json: -------------------------------------------------------------------------------- 1 | {"createdAt":"Nov 4, 2014 4:56:59 PM","id":529799371026485248,"text":"Adventures With Coffee, Code, and Writing.","source":"\u003ca href\u003d\"http://twitter.com\" rel\u003d\"nofollow\"\u003eTwitter Web Client\u003c/a\u003e","isTruncated":false,"inReplyToStatusId":-1,"inReplyToUserId":-1,"isFavorited":false,"retweetCount":0,"isPossiblySensitive":false,"contributorsIDs":[],"userMentionEntities":[],"urlEntities":[],"hashtagEntities":[],"mediaEntities":[],"currentUserRetweetId":-1,"user":{"id":15594928,"name":"Holden Karau","screenName":"holdenkarau","location":"","description":"","descriptionURLEntities":[],"isContributorsEnabled":false,"profileImageUrl":"http://pbs.twimg.com/profile_images/3005696115/2036374bbadbed85249cdd50aac6e170_normal.jpeg","profileImageUrlHttps":"https://pbs.twimg.com/profile_images/3005696115/2036374bbadbed85249cdd50aac6e170_normal.jpeg","isProtected":false,"followersCount":1231,"profileBackgroundColor":"C0DEED","profileTextColor":"333333","profileLinkColor":"0084B4","profileSidebarFillColor":"DDEEF6","profileSidebarBorderColor":"FFFFFF","profileUseBackgroundImage":true,"showAllInlineMedia":false,"friendsCount":600,"createdAt":"Aug 5, 2011 9:42:44 AM","favouritesCount":1095,"utcOffset":-3,"profileBackgroundImageUrl":"","profileBackgroundImageUrlHttps":"","profileBannerImageUrl":"","profileBackgroundTiled":true,"lang":"en","statusesCount":6234,"isGeoEnabled":true,"isVerified":false,"translator":false,"listedCount":0,"isFollowRequestSent":false}} 2 | -------------------------------------------------------------------------------- /test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaEcho.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs 19 | 20 | import scala.reflect.ClassTag 21 | 22 | import org.apache.livy.{Job, JobContext} 23 | 24 | case class ValueHolder[T](value: T) 25 | 26 | class ScalaEcho[T: ClassTag](val value: T)(implicit val tag: ClassTag[T]) extends Job[T] { 27 | 28 | override def call(jc: JobContext): T = { 29 | jc.sc().sc.parallelize(Seq(value), 1)(tag).collect()(0) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /test-lib/src/main/scala/org/apache/livy/test/jobs/ScalaSharedVariableCounter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs 19 | 20 | import org.apache.livy.{Job, JobContext} 21 | 22 | class ScalaSharedVariableCounter(name: String) extends Job[Int] { 23 | 24 | override def call(jc: JobContext): Int = { 25 | val value = try { 26 | jc.getSharedObject(name) 27 | } catch { 28 | case e: NoSuchElementException => 29 | jc.setSharedObject(name, -1) 30 | -1 31 | } 32 | 33 | val newValue = value + 1 34 | jc.setSharedObject(name, newValue) 35 | 36 | newValue 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /test-lib/src/main/spark2/java/org/apache/livy/test/jobs/spark2/SparkSessionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.test.jobs.spark2; 19 | 20 | import java.util.Arrays; 21 | 22 | import org.apache.spark.api.java.JavaSparkContext; 23 | import org.apache.spark.sql.SparkSession; 24 | 25 | import org.apache.livy.Job; 26 | import org.apache.livy.JobContext; 27 | 28 | public class SparkSessionTest implements Job { 29 | 30 | @Override 31 | public Long call(JobContext jc) throws Exception { 32 | // Make sure SparkSession and SparkContext is callable 33 | SparkSession session = jc.sparkSession(); 34 | 35 | JavaSparkContext sc = new JavaSparkContext(session.sparkContext()); 36 | return sc.parallelize(Arrays.asList(1, 2, 3)).count(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/html/thrift-sessions-table.html: -------------------------------------------------------------------------------- 1 | 17 | 18 |

    JDBC/ODBC Open Sessions

    19 | 20 | 22 | 23 | 24 | 30 | 36 | 41 | 47 | 48 | 49 | 50 | 51 |
    25 | 27 | Session Id 28 | 29 | 31 | 33 | Livy Session Id 34 | 35 | 37 | 38 | Owner 39 | 40 | 42 | 44 | Created At 45 | 46 |
    52 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/resources/org/apache/livy/server/ui/static/js/thrift-sessions.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | function loadThriftSessionsTable(sessions) { 19 | $.each(sessions, function(index, session) { 20 | $("#thrift-sessions-table .sessions-table-body").append( 21 | "" + 22 | tdWrap(session.sessionId) + 23 | tdWrap(uiLink("session/" + session.livySessionId, session.livySessionId)) + 24 | tdWrap(session.owner) + 25 | tdWrap(session.createdAt) + 26 | "" 27 | ); 28 | }); 29 | } 30 | 31 | var numSessions = 0; 32 | 33 | $(document).ready(function () { 34 | var sessionsReq = $.getJSON(location.origin + prependBasePath("/thriftserver/sessions"), function(response) { 35 | if (response && response.total > 0) { 36 | $("#thrift-sessions").load(prependBasePath("/static/html/thrift-sessions-table.html .sessions-template"), function() { 37 | loadThriftSessionsTable(response.sessions); 38 | $("#thrift-sessions-table").DataTable(); 39 | $('#thrift-sessions [data-toggle="tooltip"]').tooltip(); 40 | }); 41 | } 42 | numSessions = response.total; 43 | }); 44 | 45 | $.when(sessionsReq).done(function () { 46 | if (numSessions == 0) { 47 | $("#thrift-sessions").append('

    No open JDBC/ODBC sessions.

    '); 48 | } 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/SessionStates.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver 19 | 20 | object SessionStates extends Enumeration { 21 | type SessionStates = Value 22 | 23 | val CREATION_SUCCESS = Value("success") 24 | val CREATION_FAILED = Value("failed") 25 | val CREATION_IN_PROGRESS = Value("in_progress") 26 | } 27 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerAudit.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver 19 | 20 | import org.apache.livy.Logging 21 | 22 | object ThriftServerAudit extends Logging { 23 | 24 | def audit( 25 | user: String, 26 | ipAddress: String, 27 | query: String, 28 | startTime: Long, 29 | endTime: Long): Unit = { 30 | info( 31 | s"user: $user ipAddress: $ipAddress query: ${query.replace('\n', ' ')} " + 32 | s"start time: ${startTime} end time: ${endTime} " + 33 | s"time spent: ${Math.round((endTime - startTime) / 1000)}s") 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/ThriftServerFactoryImpl.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver 19 | 20 | import javax.servlet.Servlet 21 | 22 | import org.apache.livy.LivyConf 23 | import org.apache.livy.server.{AccessManager, ThriftServerFactory} 24 | import org.apache.livy.server.recovery.SessionStore 25 | import org.apache.livy.sessions.InteractiveSessionManager 26 | import org.apache.livy.thriftserver.ui.ThriftJsonServlet 27 | 28 | class ThriftServerFactoryImpl extends ThriftServerFactory { 29 | override def start( 30 | livyConf: LivyConf, 31 | livySessionManager: InteractiveSessionManager, 32 | sessionStore: SessionStore, 33 | accessManager: AccessManager): Unit = { 34 | if (LivyThriftServer.getInstance.isDefined) { 35 | throw new RuntimeException(s"A ${classOf[LivyThriftServer].getName} has been already " + 36 | s"started, so a new one cannot be started.") 37 | } 38 | LivyThriftServer.start(livyConf, livySessionManager, sessionStore, accessManager) 39 | } 40 | 41 | override def stop(): Unit = { 42 | assert(LivyThriftServer.getInstance.isDefined) 43 | LivyThriftServer.getInstance.foreach(_.stop()) 44 | } 45 | 46 | override def getServlet(basePath: String): Servlet = new ThriftJsonServlet(basePath) 47 | 48 | override def getServletMappings: Seq[String] = Seq("/thriftserver/*") 49 | } 50 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/ldap/ChainFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.thriftserver.auth.ldap 18 | 19 | import javax.security.sasl.AuthenticationException 20 | 21 | /** 22 | * Applies all the filters passed as param to the `ChainFilter` 23 | */ 24 | class ChainFilter(val chainedFilters: Seq[Filter]) extends Filter { 25 | @throws[AuthenticationException] 26 | def apply(user: String): Unit = { 27 | chainedFilters.foreach { f => f(user) } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/ldap/DirSearchFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.thriftserver.auth.ldap 18 | 19 | import javax.naming.directory.InitialDirContext 20 | import javax.security.sasl.AuthenticationException 21 | 22 | import org.apache.livy.LivyConf 23 | 24 | /** 25 | * A factory for DirSearch. 26 | */ 27 | trait DirSearchFactory { 28 | /** 29 | * Returns an instance of DirSearch. 30 | * 31 | * @param conf Livy configuration 32 | * @param user username 33 | * @param password user password 34 | */ 35 | @throws[AuthenticationException] 36 | def getInstance(conf: LivyConf, user: String, password: String): InitialDirContext 37 | } 38 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/ldap/Filter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.thriftserver.auth.ldap 18 | 19 | import javax.security.sasl.AuthenticationException 20 | 21 | /** 22 | * The object that filters LDAP users. 23 | * The assumption is that this user was already authenticated by a previous bind operation. 24 | */ 25 | trait Filter { 26 | /** 27 | * Applies this filter to the authenticated user. 28 | * 29 | * @param user username 30 | * @throws AuthenticationException 31 | */ 32 | @throws[AuthenticationException] 33 | def apply(user: String): Unit 34 | } 35 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/auth/ldap/UserFilter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.thriftserver.auth.ldap 18 | 19 | import javax.security.sasl.AuthenticationException 20 | 21 | import org.apache.livy.{LivyConf, Logging} 22 | import org.apache.livy.server.auth.LdapUtils 23 | 24 | /** 25 | * Filter out all users that are not in the provided in Livy configuration list. 26 | */ 27 | class UserFilter(conf: LivyConf) extends Filter with Logging { 28 | private val userFilterStr = conf.get(LivyConf.THRIFT_LDAP_AUTHENTICATION_USERFILTER) 29 | private val userFilter: Set[String] = 30 | if (userFilterStr != null) userFilterStr.split(",").toSet else Set() 31 | 32 | @throws[AuthenticationException] 33 | def apply(user: String): Unit = { 34 | if (!userFilter.isEmpty) { 35 | info("Authenticating user '{}' using user filter", user) 36 | val userName = LdapUtils.extractUserName(user).toLowerCase 37 | if (!userFilter.contains(userName)) { 38 | info("Authentication failed based on user membership") 39 | throw new AuthenticationException( 40 | "Authentication failed: User not a member of specified list") 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/cli/ThreadPoolExecutorWithOomHook.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.cli 19 | 20 | import java.util.concurrent._ 21 | 22 | /** 23 | * This class is taken from Hive, because it is package private so it cannot be accessed. 24 | * If it will become public we can remove this from here. 25 | */ 26 | class ThreadPoolExecutorWithOomHook( 27 | corePoolSize: Int, 28 | maximumPoolSize: Int, 29 | keepAliveTime: Long, 30 | unit: TimeUnit, 31 | workQueue: BlockingQueue[Runnable], 32 | threadFactory: ThreadFactory, 33 | val oomHook: Runnable) 34 | extends ThreadPoolExecutor( 35 | corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory) { 36 | 37 | override protected def afterExecute(r: Runnable, t: Throwable): Unit = { 38 | super.afterExecute(r, t) 39 | if (t == null && r.isInstanceOf[Future[_]] ) { 40 | try { 41 | val future: Future[_] = r.asInstanceOf[Future[_]] 42 | if (future.isDone) { 43 | future.get 44 | } 45 | } catch { 46 | case _: InterruptedException => Thread.currentThread.interrupt() 47 | case _: OutOfMemoryError => oomHook.run() 48 | case _: Throwable => // Do nothing 49 | } 50 | } else if (t.isInstanceOf[OutOfMemoryError]) { 51 | oomHook.run() 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /thriftserver/server/src/main/scala/org/apache/livy/thriftserver/operation/MetadataOperation.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.operation 19 | 20 | import org.apache.hive.service.cli.{FetchOrientation, HiveSQLException, OperationState, OperationType, SessionHandle} 21 | 22 | import org.apache.livy.thriftserver.serde.ThriftResultSet 23 | 24 | /** 25 | * MetadataOperation is the base class for operations which do not perform any call on Spark side 26 | */ 27 | abstract class MetadataOperation(sessionHandle: SessionHandle, opType: OperationType) 28 | extends Operation(sessionHandle, opType) { 29 | setHasResultSet(true) 30 | 31 | protected def rowSet: ThriftResultSet 32 | 33 | @throws[HiveSQLException] 34 | override def close(): Unit = { 35 | setState(OperationState.CLOSED) 36 | } 37 | 38 | @throws[HiveSQLException] 39 | override def cancel(stateAfterCancel: OperationState): Unit = { 40 | throw new UnsupportedOperationException("MetadataOperation.cancel()") 41 | } 42 | 43 | @throws(classOf[HiveSQLException]) 44 | override def getNextRowSet(orientation: FetchOrientation, maxRows: Long): ThriftResultSet = { 45 | assertState(Seq(OperationState.FINISHED)) 46 | validateFetchOrientation(orientation) 47 | if (orientation.equals(FetchOrientation.FETCH_FIRST)) { 48 | rowSet.setRowOffset(0) 49 | } 50 | rowSet.extractSubset(maxRows.toInt) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /thriftserver/server/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the file target/unit-tests.log 19 | test.appender=file 20 | log4j.rootCategory=WARN, ${test.appender} 21 | log4j.appender.file=org.apache.log4j.FileAppender 22 | log4j.appender.file.append=true 23 | log4j.appender.file.file=target/unit-tests.log 24 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 25 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n 26 | 27 | # Tests that launch java subprocesses can set the "test.appender" system property to 28 | # "console" to avoid having the child process's logs overwrite the unit test's 29 | # log file. 30 | log4j.appender.console=org.apache.log4j.ConsoleAppender 31 | log4j.appender.console.target=System.err 32 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 33 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t: %m%n 34 | 35 | # Enable DEBUG logs for Livy classes (with some exceptions). 36 | log4j.logger.org.apache.livy=DEBUG 37 | log4j.logger.org.apache.livy.rsc=INFO 38 | log4j.logger.org.apache.livy.shaded=INFO 39 | log4j.logger.org.apache.livy.thriftserver.LivyCLIService=INFO 40 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/CatalogJobState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import java.util.Iterator; 21 | 22 | public class CatalogJobState { 23 | final Iterator iter; 24 | 25 | public CatalogJobState(Iterator iter) { 26 | this.iter = iter; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/CleanupStatementJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | /** 24 | * Job used to clean up state held for a statement. 25 | */ 26 | public class CleanupStatementJob implements Job { 27 | 28 | private final String sessionId; 29 | private final String statementId; 30 | 31 | public CleanupStatementJob() { 32 | this(null, null); 33 | } 34 | 35 | public CleanupStatementJob(String sessionId, String statementId) { 36 | this.sessionId = sessionId; 37 | this.statementId = statementId; 38 | } 39 | 40 | @Override 41 | public Boolean call(JobContext ctx) { 42 | ThriftSessionState session = ThriftSessionState.get(ctx, sessionId); 43 | return session.cleanupStatement(statementId); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/DataType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.livy.thriftserver.session; 18 | 19 | /** 20 | * Enum representing the way user data is encoded on the wire between the Livy session and 21 | * the server. 22 | */ 23 | public enum DataType { 24 | 25 | BOOLEAN, 26 | BYTE, 27 | SHORT, 28 | INTEGER, 29 | LONG, 30 | FLOAT, 31 | DOUBLE, 32 | BINARY, 33 | DECIMAL, 34 | TIMESTAMP, 35 | DATE, 36 | STRING; 37 | 38 | } 39 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/FetchResultSchemaJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | /** 24 | * Job used to fetch the schema of query's results. 25 | */ 26 | public class FetchResultSchemaJob implements Job { 27 | 28 | private final String sessionId; 29 | private final String statementId; 30 | 31 | public FetchResultSchemaJob() { 32 | this(null, null); 33 | } 34 | 35 | public FetchResultSchemaJob(String sessionId, String statementId) { 36 | this.sessionId = sessionId; 37 | this.statementId = statementId; 38 | } 39 | 40 | @Override 41 | public String call(JobContext ctx) { 42 | ThriftSessionState session = ThriftSessionState.get(ctx, sessionId); 43 | return session.findStatement(statementId).schema; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/GetSchemasJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import static scala.collection.JavaConversions.seqAsJavaList; 24 | 25 | import org.apache.spark.sql.Row; 26 | import org.apache.spark.sql.catalyst.catalog.SessionCatalog; 27 | import org.apache.spark.sql.catalyst.expressions.GenericRow; 28 | 29 | public class GetSchemasJob extends SparkCatalogJob { 30 | private final String schemaPattern; 31 | 32 | public GetSchemasJob( 33 | String schemaPattern, 34 | String sessionId, 35 | String jobId, 36 | DataType[] resultTypes) { 37 | super(sessionId, jobId, resultTypes); 38 | this.schemaPattern = convertSchemaPattern(schemaPattern); 39 | } 40 | 41 | @Override 42 | protected List fetchCatalogObjects(SessionCatalog catalog) { 43 | List databases = seqAsJavaList(catalog.listDatabases(schemaPattern)); 44 | List schemas = new ArrayList<>(); 45 | for (String db : databases) { 46 | schemas.add(new GenericRow(new Object[] { 47 | db, 48 | DEFAULT_HIVE_CATALOG, 49 | })); 50 | } 51 | return schemas; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/RegisterSessionJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | /** 24 | * Job used to register a new Thrift session. Initializes the session state and stores it in the 25 | * job context. 26 | */ 27 | public class RegisterSessionJob implements Job { 28 | private final String sessionId; 29 | 30 | public RegisterSessionJob() { 31 | this(null); 32 | } 33 | 34 | public RegisterSessionJob(String sessionId) { 35 | this.sessionId = sessionId; 36 | } 37 | 38 | @Override 39 | public Boolean call(JobContext ctx) throws Exception { 40 | ThriftSessionState.create(ctx, sessionId); 41 | return true; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/ScalaIterator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import java.util.Iterator; 21 | 22 | /** 23 | * Wrapper that provides a Java iterator interface over a Scala iterator. 24 | */ 25 | class ScalaIterator implements Iterator { 26 | 27 | private final scala.collection.Iterator it; 28 | 29 | ScalaIterator(scala.collection.Iterator it) { 30 | this.it = it; 31 | } 32 | 33 | @Override 34 | public boolean hasNext() { 35 | return it.hasNext(); 36 | } 37 | 38 | @Override 39 | public T next() { 40 | return it.next(); 41 | } 42 | 43 | @Override 44 | public void remove() { 45 | throw new UnsupportedOperationException(); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/StatementState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import java.util.Iterator; 21 | 22 | import org.apache.spark.sql.Row; 23 | import org.apache.spark.sql.types.StructType; 24 | 25 | /** 26 | * State related to one user statement. 27 | */ 28 | class StatementState { 29 | 30 | final String schema; 31 | final Iterator iter; 32 | final DataType[] types; 33 | 34 | StatementState(StructType schema, Iterator iter) { 35 | this.schema = schema.json(); 36 | this.iter = iter; 37 | this.types = SparkUtils.translateSchema(schema); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/UnregisterSessionJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.livy.thriftserver.session; 19 | 20 | import org.apache.livy.Job; 21 | import org.apache.livy.JobContext; 22 | 23 | /** 24 | * Job used to clean up state held for a Thrift session. 25 | */ 26 | public class UnregisterSessionJob implements Job { 27 | 28 | private final String sessionId; 29 | 30 | public UnregisterSessionJob() { 31 | this(null); 32 | } 33 | 34 | public UnregisterSessionJob(String sessionId) { 35 | this.sessionId = sessionId; 36 | } 37 | 38 | @Override 39 | public Void call(JobContext ctx) { 40 | if (sessionId == null) { 41 | throw new IllegalStateException("No session ID."); 42 | } 43 | 44 | ThriftSessionState.get(ctx, sessionId).dispose(); 45 | return null; 46 | } 47 | 48 | } 49 | --------------------------------------------------------------------------------