├── .cirrus.yml ├── .clang-format ├── .cspell.json ├── .dockerignore ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md ├── build-docs.sh ├── codecov.yml ├── free-disk-space.sh └── workflows │ ├── build-archlinux-latest.yml │ ├── build-centos-latest.yaml │ ├── build-compatibility.yml │ ├── build-test-graph.yml │ ├── build-test.yml │ ├── build-vineyardd-and-wheels-linux.yaml │ ├── build-vineyardd-and-wheels-macos.yaml │ ├── docs.yaml │ ├── java-ci.yaml │ ├── release-latest.yml │ ├── rust-ci.yml │ ├── spelling.yaml │ ├── stale.yaml │ └── vineyard-operator.yaml ├── .gitignore ├── .gitleaks.toml ├── .gitmodules ├── .pre-commit-config.yaml ├── .pylintrc ├── .stoat └── config.yaml ├── CITATION.cff ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.rst ├── CPPLINT.cfg ├── GOVERNANCE.md ├── LICENSE ├── MAINTAINERS.md ├── NOTICE.txt ├── OWNERS ├── README.rst ├── SECURITY.md ├── benchmark ├── CMakeLists.txt ├── alloc_test │ ├── .gitignore │ ├── CMakeLists.txt │ ├── README.md │ ├── alloc_test.h │ └── bench_allocator.cc └── blob_test │ ├── CMakeLists.txt │ ├── README.md │ └── blob_test.cc ├── charts ├── .gitignore ├── artifacthub-repo.yml └── vineyard-operator │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── backup-crd.yaml │ ├── csidriver-crd.yaml │ ├── deployment.yaml │ ├── globalobject-crd.yaml │ ├── kube-scheduler-rbac.yaml │ ├── leader-election-rbac.yaml │ ├── localobject-crd.yaml │ ├── manager-rbac.yaml │ ├── metrics-reader-rbac.yaml │ ├── metrics-service.yaml │ ├── mutating-webhook-configuration.yaml │ ├── operation-crd.yaml │ ├── proxy-rbac.yaml │ ├── recover-crd.yaml │ ├── scheduler-plugin-rbac.yaml │ ├── scheduler-rbac.yaml │ ├── sidecar-crd.yaml │ ├── validating-webhook-configuration.yaml │ ├── vineyardd-crd.yaml │ ├── volume-scheduler-rbac.yaml │ └── webhook-service.yaml │ └── values.yaml ├── cmake ├── BuildEtcdCpp.cmake ├── BuildHiredis.cmake ├── BuildRedisCpp.cmake ├── CheckGCCABI.cmake ├── CheckLinkerFlag.cmake ├── DetermineImplicitIncludes.cmake ├── FindArrow.cmake ├── FindFUSE3.cmake ├── FindGFlags.cmake ├── FindGlog.cmake ├── FindGperftools.cmake ├── FindLibUV.cmake ├── FindLibUnwind.cmake ├── FindParquet.cmake ├── FindPythonExecutable.cmake ├── FindRdkafka.cmake ├── GenerateVineyard.cmake └── GenerateVineyardJava.cmake ├── docker ├── Dockerfile.vineyard-python ├── Dockerfile.vineyard-python-dev ├── Dockerfile.vineyardd ├── Makefile ├── dev │ ├── Dockerfile.dev │ └── build_scripts │ │ ├── install-arrow.sh │ │ ├── install-deps.sh │ │ ├── install-miscs.sh │ │ └── requirements.txt ├── pypa │ ├── Dockerfile.manylinux1 │ └── Dockerfile.manylinux1-wheel ├── vineyard-fluid-fuse │ ├── Dockerfile │ └── entrypoint.sh └── vineyardd │ └── Dockerfile.alpine-builder ├── docs ├── .gitignore ├── .nojekyll ├── CNAME ├── Doxyfile ├── Makefile ├── _static │ ├── css │ │ ├── brands.min.css │ │ ├── custom.css │ │ ├── index.css │ │ ├── panels.css │ │ ├── termynal.css │ │ └── v4-shims.min.css │ ├── js │ │ └── termynal.js │ └── webfonts │ │ ├── fa-brands-400.ttf │ │ └── fa-brands-400.woff2 ├── _templates │ ├── base.html │ ├── index.html │ ├── layout.html │ └── page.html ├── artifacts │ └── vineyard-sigmod-2023.pdf ├── cncf │ ├── 2022-vineyard-annual.md │ └── 2023-vineyard-annual.md ├── conf.py ├── docs.rst ├── images │ ├── airflow_etl.jpg │ ├── cncf-color.svg │ ├── cncf-sandbox-color.svg │ ├── cncf-small.png │ ├── cncf-tiny.png │ ├── cncf.png │ ├── dask-tf.jpg │ ├── data_sharing_with_deployment.jpg │ ├── data_sharing_with_sidecar.jpg │ ├── fraud-detection-job.jpg │ ├── kubeflow_create_run.png │ ├── kubeflow_upload_pipeline.png │ ├── sandbox-small.png │ ├── sandbox-tiny.png │ ├── sandbox.png │ ├── vineyard-logo-h.png │ ├── vineyard-logo-notext.png │ ├── vineyard-logo-rect.png │ ├── vineyard-logo-v6d.png │ ├── vineyard-logo.png │ ├── vineyard.ico │ ├── vineyard_arch.jpg │ ├── vineyard_compare.png │ ├── vineyard_composable.jpg │ ├── vineyard_distributed_tensor.jpg │ ├── vineyard_logo.png │ └── vineyard_operator_arch.png ├── notes │ ├── architecture.rst │ ├── cloud-native │ │ ├── deploy-kubernetes.rst │ │ ├── vineyard-operator.rst │ │ └── vineyardctl.md │ ├── developers.rst │ ├── developers │ │ ├── build-from-source.rst │ │ ├── contributing.rst │ │ ├── faq.rst │ │ ├── roadmap.rst │ │ └── troubleshooting.rst │ ├── getting-started.rst │ ├── integration-bigdata.rst │ ├── integration-orchestration.rst │ ├── integration │ │ ├── airflow.rst │ │ ├── dask.rst │ │ ├── kedro.md │ │ ├── ml.rst │ │ └── ray.rst │ ├── key-concepts.rst │ ├── key-concepts │ │ ├── data-accessing.rst │ │ ├── io-drivers.rst │ │ ├── objects.rst │ │ ├── streams.rst │ │ └── vcdl.rst │ ├── references.rst │ └── references │ │ ├── cpp-api.rst │ │ ├── crds.md │ │ └── python-api.rst ├── summer.html └── tutorials │ ├── data-processing.rst │ ├── data-processing │ ├── accelerate-data-sharing-in-kedro.rst │ ├── distributed-learning.ipynb │ ├── gpu-memory-sharing.rst │ ├── python-sharedmemory.rst │ └── using-objects-python.rst │ ├── extending.rst │ ├── extending │ ├── define-datatypes-cpp.rst │ └── define-datatypes-python.rst │ ├── kubernetes.rst │ ├── kubernetes │ ├── data-sharing-with-vineyard-on-kubernetes.rst │ ├── efficient-data-sharing-in-kubeflow-with-vineyard-csi-driver.rst │ ├── ml-pipeline-mars-pytorch.rst │ ├── using-vineyard-operator.rst │ └── vineyard-on-fluid.rst │ └── tutorials.rst ├── go.work ├── go └── vineyard │ ├── .gitignore │ ├── .golangci.yaml │ ├── Makefile │ ├── README.md │ ├── go.mod │ ├── go.sum │ ├── hack │ └── boilerplate.go.txt │ └── pkg │ ├── client │ ├── blob.go │ ├── client_base.go │ ├── io │ │ └── io.go │ ├── ipc_client.go │ ├── ipc_client_test.go │ ├── object.go │ ├── object_meta.go │ ├── rpc_client.go │ └── rpc_client_test.go │ ├── common │ ├── config.go │ ├── json.go │ ├── log │ │ ├── deleg.go │ │ └── log.go │ ├── memory │ │ ├── fd.go │ │ └── slice.go │ ├── protocol.go │ ├── protocol_test.go │ ├── status.go │ ├── types │ │ ├── id.go │ │ ├── id_test.go │ │ ├── payload.go │ │ └── traits.go │ └── util.go │ └── ds │ └── basic │ ├── array.go │ ├── array_test.go │ └── arrow │ ├── array.go │ ├── array_32.go │ ├── array_32_test.go │ ├── array_64.go │ ├── array_64_test.go │ ├── array_string.go │ ├── array_string_test.go │ ├── record_batch.go │ ├── record_batch_test.go │ ├── table.go │ └── table_test.go ├── java ├── .gitignore ├── .mvn │ └── jvm.config ├── apache-2.0.txt ├── core │ ├── .gitignore │ ├── CMakeLists.txt │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── cpp │ │ │ └── ffi │ │ │ │ ├── ffi.cc │ │ │ │ └── object_meta.cc │ │ └── java │ │ │ ├── com │ │ │ └── google │ │ │ │ └── common │ │ │ │ └── base │ │ │ │ └── StopwatchContext.java │ │ │ └── io │ │ │ └── v6d │ │ │ └── core │ │ │ ├── FFI.java │ │ │ ├── client │ │ │ ├── Client.java │ │ │ ├── ClusterStatus.java │ │ │ ├── Context.java │ │ │ ├── IPCClient.java │ │ │ ├── InstanceStatus.java │ │ │ ├── RPCClient.java │ │ │ └── ds │ │ │ │ ├── Blob.java │ │ │ │ ├── Buffer.java │ │ │ │ ├── BufferSet.java │ │ │ │ ├── Object.java │ │ │ │ ├── ObjectBase.java │ │ │ │ ├── ObjectBuilder.java │ │ │ │ ├── ObjectFactory.java │ │ │ │ ├── ObjectMeta.java │ │ │ │ └── ffi │ │ │ │ ├── FFIResolver.java │ │ │ │ └── ObjectMeta.java │ │ │ └── common │ │ │ ├── memory │ │ │ ├── Payload.java │ │ │ └── ffi │ │ │ │ └── Fling.java │ │ │ └── util │ │ │ ├── Env.java │ │ │ ├── InstanceID.java │ │ │ ├── JSON.java │ │ │ ├── ObjectID.java │ │ │ ├── Protocol.java │ │ │ ├── Signature.java │ │ │ └── VineyardException.java │ │ └── test │ │ ├── java │ │ └── io │ │ │ └── v6d │ │ │ └── core │ │ │ ├── client │ │ │ ├── IPCClientTest.java │ │ │ └── ds │ │ │ │ ├── ObjectFactoryTest.java │ │ │ │ ├── ObjectMetaTest.java │ │ │ │ ├── StringObject.java │ │ │ │ ├── StringObjectBuilder.java │ │ │ │ └── StringObjectResolver.java │ │ │ └── common │ │ │ └── util │ │ │ ├── IdTest.java │ │ │ └── ProtocolTest.java │ │ └── resources │ │ └── logback.xml ├── hive │ ├── README.rst │ ├── conf │ │ ├── hive-log4j2.properties │ │ └── hive-site.xml │ ├── docker │ │ ├── .env │ │ ├── build.sh │ │ ├── dependency │ │ │ ├── images │ │ │ │ ├── Dockerfile │ │ │ │ ├── bootstrap.sh │ │ │ │ ├── hadoop-config │ │ │ │ │ ├── core-site.xml │ │ │ │ │ ├── log4j.properties │ │ │ │ │ ├── mapred-site.xml │ │ │ │ │ ├── yarn-hosts-exclude │ │ │ │ │ ├── yarn-hosts-include │ │ │ │ │ └── yarn-site.xml │ │ │ │ ├── hive-config-distributed │ │ │ │ │ ├── hive-env.sh │ │ │ │ │ └── hive-site.xml │ │ │ │ ├── hive-config │ │ │ │ │ ├── hive-env.sh │ │ │ │ │ └── hive-site.xml │ │ │ │ ├── mysql-connector-java-5.1.49 │ │ │ │ │ └── mysql-connector-java-5.1.49-bin.jar │ │ │ │ ├── spark-config-distributed │ │ │ │ │ └── spark-defaults.conf │ │ │ │ ├── spark-config │ │ │ │ │ └── spark-defaults.conf │ │ │ │ └── tez-config │ │ │ │ │ └── tez-site.xml │ │ │ └── mysql │ │ │ │ └── mysql-compose.yaml │ │ ├── docker-compose-distributed.yaml │ │ └── docker-compose.yaml │ ├── pom.xml │ ├── src │ │ ├── main │ │ │ ├── java │ │ │ │ ├── com │ │ │ │ │ └── google │ │ │ │ │ │ └── common │ │ │ │ │ │ ├── base │ │ │ │ │ │ ├── CharMatcher.java │ │ │ │ │ │ ├── Objects.java │ │ │ │ │ │ └── Splitter.java │ │ │ │ │ │ ├── jimfs │ │ │ │ │ │ └── FIleSystemState.java │ │ │ │ │ │ └── util │ │ │ │ │ │ └── concurrent │ │ │ │ │ │ ├── AbstractCheckedFuture.java │ │ │ │ │ │ ├── CheckedFuture.java │ │ │ │ │ │ ├── Futures.java │ │ │ │ │ │ ├── ListeningScheduledExecutorService.java │ │ │ │ │ │ └── MoreExecutors.java │ │ │ │ └── io │ │ │ │ │ └── v6d │ │ │ │ │ ├── hadoop │ │ │ │ │ └── fs │ │ │ │ │ │ ├── FileSystem.java │ │ │ │ │ │ └── VineyardFileSystem.java │ │ │ │ │ └── hive │ │ │ │ │ └── ql │ │ │ │ │ └── io │ │ │ │ │ ├── CloseableReentrantLock.java │ │ │ │ │ ├── HiveTypeResolver.java │ │ │ │ │ ├── HiveTypeTransformer.java │ │ │ │ │ ├── RecordWrapperWritable.java │ │ │ │ │ ├── TypeContext.java │ │ │ │ │ ├── VineyardInputFormat.java │ │ │ │ │ ├── VineyardOutputFormat.java │ │ │ │ │ ├── VineyardSerDe.java │ │ │ │ │ ├── VineyardSplit.java │ │ │ │ │ └── VineyardStorageFormatDescriptor.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── org.apache.hadoop.fs.FileSystem │ │ │ │ └── org.apache.hadoop.hive.ql.io.StorageFormatDescriptor │ │ └── test │ │ │ └── resources │ │ │ └── logback.xml │ └── test │ │ ├── distributed-test.sh │ │ ├── expected │ │ ├── test_all_primitive_types.q.out │ │ ├── test_hive_dynamic_partition.q.out │ │ ├── test_hive_static_partition.q.out │ │ ├── test_insert.q.out │ │ └── test_nested_types.q.out │ │ ├── query │ │ ├── test_all_primitive_types.q │ │ ├── test_hive_dynamic_partition.q │ │ ├── test_hive_static_partition.q │ │ ├── test_insert.q │ │ └── test_nested_types.q │ │ ├── spark-hive-distributed-test.sh │ │ ├── spark-hive-test.sh │ │ ├── spark-query │ │ └── test_all_primitive_types.scala │ │ └── test.sh ├── modules │ ├── basic │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── io │ │ │ │ └── v6d │ │ │ │ └── modules │ │ │ │ └── basic │ │ │ │ ├── array │ │ │ │ └── Array.java │ │ │ │ ├── arrow │ │ │ │ ├── Array.java │ │ │ │ ├── ArrayBuilder.java │ │ │ │ ├── Arrow.java │ │ │ │ ├── BooleanArray.java │ │ │ │ ├── BooleanArrayBuilder.java │ │ │ │ ├── Buffer.java │ │ │ │ ├── BufferBuilder.java │ │ │ │ ├── DateArray.java │ │ │ │ ├── DateArrayBuilder.java │ │ │ │ ├── DecimalArray.java │ │ │ │ ├── DecimalArrayBuilder.java │ │ │ │ ├── DoubleArray.java │ │ │ │ ├── DoubleArrayBuilder.java │ │ │ │ ├── FloatArray.java │ │ │ │ ├── FloatArrayBuilder.java │ │ │ │ ├── Int16Array.java │ │ │ │ ├── Int16ArrayBuilder.java │ │ │ │ ├── Int32Array.java │ │ │ │ ├── Int32ArrayBuilder.java │ │ │ │ ├── Int64Array.java │ │ │ │ ├── Int64ArrayBuilder.java │ │ │ │ ├── Int8Array.java │ │ │ │ ├── Int8ArrayBuilder.java │ │ │ │ ├── LargeStringArray.java │ │ │ │ ├── LargeStringArrayBuilder.java │ │ │ │ ├── ListArray.java │ │ │ │ ├── ListArrayBuilder.java │ │ │ │ ├── MapArray.java │ │ │ │ ├── MapArrayBuilder.java │ │ │ │ ├── NullArray.java │ │ │ │ ├── NullArrayBuilder.java │ │ │ │ ├── RecordBatch.java │ │ │ │ ├── RecordBatchBuilder.java │ │ │ │ ├── Schema.java │ │ │ │ ├── SchemaBuilder.java │ │ │ │ ├── StringArray.java │ │ │ │ ├── StringArrayBuilder.java │ │ │ │ ├── StructArray.java │ │ │ │ ├── StructArrayBuilder.java │ │ │ │ ├── Table.java │ │ │ │ ├── TableBuilder.java │ │ │ │ ├── TimestampArray.java │ │ │ │ ├── TimestampArrayBuilder.java │ │ │ │ ├── VarBinaryArray.java │ │ │ │ ├── VarBinaryArrayBuilder.java │ │ │ │ └── util │ │ │ │ │ ├── ArrowBufSeekableByteChannel.java │ │ │ │ │ ├── ArrowVectorUtils.java │ │ │ │ │ ├── BufferedOutputStreamWritableByteChannel.java │ │ │ │ │ ├── ObjectResolver.java │ │ │ │ │ ├── ObjectTransformer.java │ │ │ │ │ └── SchemaSerializer.java │ │ │ │ ├── columnar │ │ │ │ ├── ArrowFieldVectorAccessor.java │ │ │ │ ├── ColumnarData.java │ │ │ │ └── ColumnarDataBuilder.java │ │ │ │ ├── dataframe │ │ │ │ └── DataFrame.java │ │ │ │ ├── filesystem │ │ │ │ ├── VineyardFile.java │ │ │ │ ├── VineyardFileStat.java │ │ │ │ └── VineyardFileUtils.java │ │ │ │ ├── scalar │ │ │ │ ├── BooleanValue.java │ │ │ │ ├── BooleanValueBuilder.java │ │ │ │ ├── FloatValue.java │ │ │ │ ├── FloatValueBuilder.java │ │ │ │ ├── StringValue.java │ │ │ │ └── StringValueBuilder.java │ │ │ │ ├── stream │ │ │ │ └── RecordBatchStream.java │ │ │ │ └── tensor │ │ │ │ └── Tensor.java │ │ │ └── test │ │ │ ├── java │ │ │ └── io │ │ │ │ └── v6d │ │ │ │ └── modules │ │ │ │ └── basic │ │ │ │ └── arrow │ │ │ │ ├── ArrowTest.java │ │ │ │ └── BufferTest.java │ │ │ └── resources │ │ │ └── logback.xml │ ├── graph │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── io │ │ │ │ └── v6d │ │ │ │ └── contrib │ │ │ │ └── graph │ │ │ │ └── Graph.java │ │ │ └── test │ │ │ └── java │ │ │ └── io │ │ │ └── v6d │ │ │ └── contrib │ │ │ └── graph │ │ │ └── GraphTest.java │ └── pom.xml ├── pom.xml └── spark │ ├── pom.xml │ └── src │ ├── main │ └── scala │ │ ├── io │ │ └── v6d │ │ │ └── spark │ │ │ ├── SparkApp.scala │ │ │ └── rdd │ │ │ ├── DataFrameBuilder.scala │ │ │ ├── GlobalDataFrameRDD.scala │ │ │ ├── TableRDD.scala │ │ │ └── VineyardRDD.scala │ │ └── org │ │ └── apache │ │ └── spark │ │ ├── scheduler │ │ └── cluster │ │ │ └── vineyard │ │ │ └── DeployContext.scala │ │ └── sql │ │ └── vineyard │ │ └── DataContext.scala │ └── test │ └── scala │ ├── io │ └── v6d │ │ └── spark │ │ └── rdd │ │ ├── TestCustomRDD.scala │ │ ├── TestTableRDD.scala │ │ └── TestVineyardRDD.scala │ └── samples │ └── scalatest.scala ├── k8s ├── .dockerignore ├── .gitignore ├── .golangci.yaml ├── Dockerfile ├── Makefile ├── PROJECT ├── README.md ├── apis │ └── k8s │ │ └── v1alpha1 │ │ ├── README.md │ │ ├── backup_types.go │ │ ├── backup_webhook.go │ │ ├── csidriver_types.go │ │ ├── csidriver_webhook.go │ │ ├── globalobject_types.go │ │ ├── globalobject_webhook.go │ │ ├── groupversion_info.go │ │ ├── localobject_types.go │ │ ├── localobject_webhook.go │ │ ├── operation_types.go │ │ ├── operation_webhook.go │ │ ├── recover_types.go │ │ ├── recover_webhook.go │ │ ├── sidecar_types.go │ │ ├── sidecar_webhook.go │ │ ├── vineyardd_types.go │ │ ├── vineyardd_webhook.go │ │ ├── webhook_test.go │ │ └── zz_generated.deepcopy.go ├── cmd │ ├── README.md │ ├── commands │ │ ├── client │ │ │ ├── get.go │ │ │ ├── get_blob.go │ │ │ ├── get_cluster_info.go │ │ │ ├── get_metadata.go │ │ │ ├── get_object.go │ │ │ ├── ls.go │ │ │ ├── ls_blobs.go │ │ │ ├── ls_metadatas.go │ │ │ ├── ls_objects.go │ │ │ └── put.go │ │ ├── create │ │ │ ├── create.go │ │ │ ├── create_backup.go │ │ │ ├── create_backup_test.go │ │ │ ├── create_operation.go │ │ │ ├── create_operation_test.go │ │ │ ├── create_recover.go │ │ │ └── create_recover_test.go │ │ ├── csi │ │ │ ├── csi.go │ │ │ └── csi_test.go │ │ ├── delete │ │ │ ├── delete.go │ │ │ ├── delete_backup.go │ │ │ ├── delete_csidriver.go │ │ │ ├── delete_operation.go │ │ │ ├── delete_operator.go │ │ │ ├── delete_recover.go │ │ │ ├── delete_vineyard_cluster.go │ │ │ ├── delete_vineyard_deployment.go │ │ │ ├── delete_vineyard_deployment_test.go │ │ │ └── delete_vineyardd.go │ │ ├── deploy │ │ │ ├── deploy.go │ │ │ ├── deploy_backup_job.go │ │ │ ├── deploy_backup_job_test.go │ │ │ ├── deploy_csidriver.go │ │ │ ├── deploy_operator.go │ │ │ ├── deploy_recover_job.go │ │ │ ├── deploy_recover_job_test.go │ │ │ ├── deploy_vineyard_cluster.go │ │ │ ├── deploy_vineyard_deployment.go │ │ │ ├── deploy_vineyard_deployment_test.go │ │ │ ├── deploy_vineyardd.go │ │ │ └── deploy_vineyardd_test.go │ │ ├── flags │ │ │ ├── backup_flags.go │ │ │ ├── client_flags.go │ │ │ ├── csi_flags.go │ │ │ ├── csidriver_flags.go │ │ │ ├── global_flags.go │ │ │ ├── inject_argo_workflow_flags.go │ │ │ ├── inject_flags.go │ │ │ ├── managers_flags.go │ │ │ ├── operation_flags.go │ │ │ ├── recover_flags.go │ │ │ ├── scheduler_workflow_flags.go │ │ │ ├── scheduler_workload_flags.go │ │ │ └── vineyardd_flags.go │ │ ├── inject │ │ │ ├── inject.go │ │ │ ├── inject_argo_workflow.go │ │ │ ├── inject_argo_workflow_test.go │ │ │ └── inject_test.go │ │ ├── manager │ │ │ └── manager.go │ │ ├── schedule │ │ │ ├── schedule.go │ │ │ ├── schedule_workflow.go │ │ │ ├── schedule_workload.go │ │ │ └── schedule_workload_test.go │ │ └── util │ │ │ ├── arg.go │ │ │ ├── arg_test.go │ │ │ ├── cert.go │ │ │ ├── client.go │ │ │ ├── client_test.go │ │ │ ├── connect.go │ │ │ ├── format.go │ │ │ ├── io.go │ │ │ ├── io_test.go │ │ │ ├── manifests.go │ │ │ ├── manifests_test.go │ │ │ ├── objects.go │ │ │ ├── operator.go │ │ │ ├── output.go │ │ │ ├── parse.go │ │ │ ├── parse_test.go │ │ │ ├── portforward.go │ │ │ ├── template.go │ │ │ ├── template_test.go │ │ │ ├── usage │ │ │ ├── doc.go │ │ │ ├── json.go │ │ │ ├── md_docs.go │ │ │ └── types.go │ │ │ └── workload.go │ └── main.go ├── config.yaml ├── config │ ├── certmanager │ │ ├── certificate.yaml │ │ ├── kustomization.yaml │ │ └── kustomizeconfig.yaml │ ├── crd │ │ ├── bases │ │ │ ├── k8s.v6d.io_backups.yaml │ │ │ ├── k8s.v6d.io_csidrivers.yaml │ │ │ ├── k8s.v6d.io_globalobjects.yaml │ │ │ ├── k8s.v6d.io_localobjects.yaml │ │ │ ├── k8s.v6d.io_operations.yaml │ │ │ ├── k8s.v6d.io_recovers.yaml │ │ │ ├── k8s.v6d.io_sidecars.yaml │ │ │ └── k8s.v6d.io_vineyardds.yaml │ │ ├── kustomization.yaml │ │ ├── kustomizeconfig.yaml │ │ └── patches │ │ │ ├── cainjection_in_backups.yaml │ │ │ ├── cainjection_in_csidrivers.yaml │ │ │ ├── cainjection_in_globalobjects.yaml │ │ │ ├── cainjection_in_localobjects.yaml │ │ │ ├── cainjection_in_operations.yaml │ │ │ ├── cainjection_in_recovers.yaml │ │ │ ├── cainjection_in_sidecars.yaml │ │ │ ├── cainjection_in_vineyardds.yaml │ │ │ ├── webhook_in_backups.yaml │ │ │ ├── webhook_in_csidrivers.yaml │ │ │ ├── webhook_in_globalobjects.yaml │ │ │ ├── webhook_in_localobjects.yaml │ │ │ ├── webhook_in_operations.yaml │ │ │ ├── webhook_in_recovers.yaml │ │ │ ├── webhook_in_sidecars.yaml │ │ │ └── webhook_in_vineyardds.yaml │ ├── default │ │ ├── kustomization.yaml │ │ ├── manager_auth_proxy_patch.yaml │ │ ├── manager_webhook_patch.yaml │ │ └── webhookcainjection_patch.yaml │ ├── embed.go │ ├── manager │ │ ├── kustomization.yaml │ │ └── manager.yaml │ ├── prometheus │ │ ├── kustomization.yaml │ │ └── monitor.yaml │ ├── rbac │ │ ├── auth_proxy_client_clusterrole.yaml │ │ ├── auth_proxy_role.yaml │ │ ├── auth_proxy_role_binding.yaml │ │ ├── auth_proxy_service.yaml │ │ ├── globalobject_editor_role.yaml │ │ ├── globalobject_viewer_role.yaml │ │ ├── k8s_backup_editor_role.yaml │ │ ├── k8s_backup_viewer_role.yaml │ │ ├── k8s_csidriver_editor_role.yaml │ │ ├── k8s_csidriver_viewer_role.yaml │ │ ├── k8s_operation_editor_role.yaml │ │ ├── k8s_operation_viewer_role.yaml │ │ ├── k8s_recover_editor_role.yaml │ │ ├── k8s_recover_viewer_role.yaml │ │ ├── k8s_sidecar_editor_role.yaml │ │ ├── k8s_sidecar_viewer_role.yaml │ │ ├── kustomization.yaml │ │ ├── leader_election_role.yaml │ │ ├── leader_election_role_binding.yaml │ │ ├── localobject_editor_role.yaml │ │ ├── localobject_viewer_role.yaml │ │ ├── manager_account.yaml │ │ ├── role.yaml │ │ ├── role_binding.yaml │ │ ├── vineyardd_editor_role.yaml │ │ └── vineyardd_viewer_role.yaml │ ├── samples │ │ ├── k8s_v1alpha1_backup.yaml │ │ ├── k8s_v1alpha1_csidriver.yaml │ │ ├── k8s_v1alpha1_globalobject.yaml │ │ ├── k8s_v1alpha1_localobject.yaml │ │ ├── k8s_v1alpha1_operation.yaml │ │ ├── k8s_v1alpha1_recover.yaml │ │ ├── k8s_v1alpha1_sidecar.yaml │ │ ├── k8s_v1alpha1_vineyardd.yaml │ │ └── kustomization.yaml │ ├── scheduler │ │ └── config.yaml │ └── webhook │ │ ├── kustomization.yaml │ │ ├── kustomizeconfig.yaml │ │ ├── manifests.yaml │ │ ├── operation_namespace_selector_patch.yaml │ │ ├── scheduling_selector_patch.yaml │ │ ├── service.yaml │ │ └── sidecar_selector_patch.yaml ├── controllers │ └── k8s │ │ ├── backup_controller.go │ │ ├── common.go │ │ ├── controller_test.go │ │ ├── csidriver_controller.go │ │ ├── globalobject_controller.go │ │ ├── localobject_controller.go │ │ ├── operation_controller.go │ │ ├── recover_controller.go │ │ ├── sidecar_controller.go │ │ ├── utils.go │ │ └── vineyardd_controller.go ├── examples │ ├── vineyard-csidriver │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── pipeline-kfp-v2-with-vineyard.py │ │ ├── pipeline-kfp-v2-with-vineyard.yaml │ │ ├── pipeline-kfp-v2.py │ │ ├── pipeline-kfp-v2.yaml │ │ ├── pipeline-with-vineyard.py │ │ ├── pipeline-with-vineyard.yaml │ │ ├── pipeline.py │ │ ├── pipeline.yaml │ │ ├── prepare-data.yaml │ │ ├── prepare-data │ │ │ └── prepare-data.py │ │ ├── preprocess │ │ │ └── preprocess.py │ │ ├── rbac.yaml │ │ ├── test │ │ │ └── test.py │ │ └── train │ │ │ └── train.py │ ├── vineyard-kubeflow │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── pipeline-with-vineyard.py │ │ ├── pipeline-with-vineyard.yaml │ │ ├── pipeline.py │ │ ├── pipeline.yaml │ │ ├── prepare-data.yaml │ │ ├── prepare-data │ │ │ └── prepare-data.py │ │ ├── preprocess │ │ │ └── preprocess.py │ │ ├── rbac.yaml │ │ ├── readme.md │ │ ├── test │ │ │ └── test.py │ │ └── train │ │ │ └── train.py │ └── vineyard-mars-pytorch │ │ ├── Dockerfile.mars-with-vineyard │ │ ├── Makefile │ │ ├── prepare-data │ │ ├── prepare-data.py │ │ └── resources │ │ │ ├── job.yaml │ │ │ ├── role-binding.yaml │ │ │ ├── role.yaml │ │ │ └── service-account.yaml │ │ ├── process-data │ │ ├── process-data.py │ │ └── resources │ │ │ ├── cluster-role-binding.yaml │ │ │ ├── cluster-role.yaml │ │ │ ├── job.yaml │ │ │ └── service-account.yaml │ │ └── train-data │ │ ├── resources │ │ ├── job.yaml │ │ └── service.yaml │ │ └── train-data.py ├── go.mod ├── go.sum ├── hack │ └── boilerplate.go.txt ├── pkg │ ├── config │ │ ├── annotations │ │ │ └── annotations.go │ │ └── labels │ │ │ └── labels.go │ ├── csidriver │ │ ├── controller.go │ │ ├── driver.go │ │ ├── identity.go │ │ ├── node.go │ │ └── state.go │ ├── injector │ │ └── sidecar_injector.go │ ├── log │ │ └── log.go │ ├── operation │ │ ├── assembly.go │ │ ├── interface.go │ │ ├── repartition.go │ │ └── utils.go │ ├── schedulers │ │ ├── common.go │ │ ├── scheduler_inside_cluster.go │ │ ├── scheduler_outside_cluster.go │ │ ├── scheduling.go │ │ └── scheduling_strategy.go │ ├── templates │ │ ├── backup │ │ │ ├── backup-pv.yaml │ │ │ ├── backup-pvc.yaml │ │ │ ├── job.yaml │ │ │ ├── role-binding.yaml │ │ │ └── role.yaml │ │ ├── csidriver │ │ │ ├── daemonset.yaml │ │ │ ├── deployment.yaml │ │ │ └── storageclass.yaml │ │ ├── etcd │ │ │ ├── etcd.yaml │ │ │ └── service.yaml │ │ ├── operation │ │ │ ├── dask-repartition.yaml │ │ │ ├── distributed-assembly-job.yaml │ │ │ └── local-assembly-job.yaml │ │ ├── recover │ │ │ ├── job.yaml │ │ │ ├── role-binding.yaml │ │ │ └── role.yaml │ │ ├── sidecar │ │ │ └── injection-template.yaml │ │ ├── template.go │ │ └── vineyardd │ │ │ ├── deployment.yaml │ │ │ ├── etcd-service.yaml │ │ │ ├── service.yaml │ │ │ ├── spill-pv.yaml │ │ │ └── spill-pvc.yaml │ └── webhook │ │ ├── operation │ │ └── operation_webhook.go │ │ ├── scheduling │ │ └── scheduling-webhook.go │ │ └── sidecar │ │ └── sidecar_webhook.go └── test │ ├── csidriver │ ├── Dockerfile │ └── Makefile │ ├── e2e │ ├── Dockerfile │ ├── Makefile │ ├── airflow-integration │ │ └── e2e.yaml │ ├── assembly-demo │ │ ├── assembly-distributed.py │ │ ├── assembly-job-config.yaml │ │ ├── assembly-job.yaml │ │ ├── assembly-job1.py │ │ ├── assembly-job2.py │ │ ├── assembly-local.py │ │ ├── distributed-job1.py │ │ ├── distributed-job2.py │ │ └── distributed-job3.py │ ├── assembly │ │ ├── distributed-assembly-e2e.yaml │ │ └── local-assembly-e2e.yaml │ ├── autogenerated-helm-chart │ │ └── e2e.yaml │ ├── deploy-raw-backup-and-recover │ │ └── e2e.yaml │ ├── etcd-failover │ │ ├── consumer.yaml │ │ ├── five-etcd-nodes-failover-e2e.yaml │ │ ├── producer.yaml │ │ ├── three-etcd-nodes-failover-e2e.yaml │ │ └── vineyardd.yaml │ ├── failover-demo │ │ ├── backup.py │ │ ├── backup.yaml │ │ ├── build-distributed-object-step1.py │ │ ├── build-distributed-object-step2.py │ │ ├── build-local-object.py │ │ ├── failover-job-config.yaml │ │ ├── failover-job.yaml │ │ ├── get-distributed-object.py │ │ ├── get-local-object.py │ │ ├── recover.py │ │ └── recover.yaml │ ├── failover │ │ └── e2e.yaml │ ├── kind-with-local-registry.yaml │ ├── kind.yaml │ ├── mars-examples │ │ └── e2e.yaml │ ├── repartition-demo │ │ ├── Dockerfile.dask-worker-with-vineyard │ │ ├── dask-repartition.py │ │ ├── job1.py │ │ ├── job2.py │ │ ├── repartition-dask-helm-values.yaml │ │ ├── repartition-job-config.yaml │ │ └── repartition-job.yaml │ ├── repartition │ │ └── dask-repartition-e2e.yaml │ ├── schedule-workflow-demo │ │ └── test.yaml │ ├── schedule-workflow-without-crd-demo │ │ └── test.yaml │ ├── schedule-workflow-without-crd │ │ └── e2e.yaml │ ├── schedule-workflow │ │ └── e2e.yaml │ ├── schedule-workload-demo │ │ ├── consumer.yaml │ │ └── producer.yaml │ ├── schedule-workload │ │ └── e2e.yaml │ ├── serialize-demo │ │ ├── serialize-demo.yaml │ │ ├── serialize-pv.yaml │ │ ├── serialize-pvc.yaml │ │ └── serialize.py │ ├── serialize │ │ └── e2e.yaml │ ├── sidecar-demo │ │ ├── job.py │ │ ├── sidecar-with-custom-sidecar.yaml │ │ └── sidecar-with-default-sidecar.yaml │ ├── sidecar │ │ └── e2e.yaml │ ├── spill-demo │ │ ├── spill-demo.yaml │ │ └── vineyardd-with-spill.yaml │ ├── spill │ │ └── e2e.yaml │ ├── verify │ │ ├── basic-types-unit.yaml │ │ ├── dag-success.yaml │ │ ├── distributed-assembly-values.yaml │ │ ├── distributed-object-values.yaml │ │ ├── local-assembly-values.yaml │ │ ├── nodes.yaml │ │ ├── object-id.yaml │ │ ├── partition.yaml │ │ ├── serialize.yaml │ │ ├── serializedfile.yaml │ │ ├── spill.yaml │ │ ├── spilledfile.yaml │ │ └── values.yaml │ ├── vineyardctl-python-api │ │ └── e2e.yaml │ ├── vineyardctl │ │ └── e2e.yaml │ ├── vineyardd.yaml │ ├── workflow-demo │ │ ├── job1.py │ │ ├── job2.py │ │ ├── workflow-job1.yaml │ │ └── workflow-job2.yaml │ └── workflow │ │ └── e2e.yaml │ └── hack │ ├── build-kind-cluster-with-local-registry.sh │ ├── delete-kind-cluster-with-local-registry.sh │ └── prepare-e2e.sh ├── misc ├── cpplint.py ├── git-archive-submodules.sh ├── release-docker.sh ├── release-wheels.sh └── tardotgz-to-zip.py ├── modules ├── basic │ ├── CMakeLists.txt │ ├── ds │ │ ├── array.h │ │ ├── array.vineyard-mod │ │ ├── arrow.cc │ │ ├── arrow.h │ │ ├── arrow.vineyard-mod │ │ ├── arrow_shim │ │ │ ├── concatenate.cc │ │ │ ├── concatenate.h │ │ │ ├── memory_pool.cc │ │ │ └── memory_pool.h │ │ ├── arrow_utils.cc │ │ ├── arrow_utils.h │ │ ├── dataframe.cc │ │ ├── dataframe.h │ │ ├── dataframe.vineyard-mod │ │ ├── hashmap.h │ │ ├── hashmap.vineyard-mod │ │ ├── hashmap_mvcc.h │ │ ├── scalar.h │ │ ├── scalar.vineyard-mod │ │ ├── sequence.h │ │ ├── sequence.vineyard-mod │ │ ├── tensor.cc │ │ ├── tensor.h │ │ ├── tensor.vineyard-mod │ │ ├── types.cc │ │ └── types.h │ ├── stream │ │ ├── byte_stream.cc │ │ ├── byte_stream.h │ │ ├── dataframe_stream.cc │ │ ├── dataframe_stream.h │ │ ├── parallel_stream.cc │ │ ├── parallel_stream.h │ │ ├── parallel_stream.vineyard-mod │ │ ├── recordbatch_stream.cc │ │ └── recordbatch_stream.h │ └── utils.h ├── fuse │ ├── CMakeLists.txt │ ├── adaptors │ │ ├── arrow_ipc │ │ │ ├── deserializer_registry.cc │ │ │ ├── deserializer_registry.h │ │ │ ├── serializer_registry.cc │ │ │ └── serializer_registry.h │ │ ├── chunk_buffer │ │ │ ├── chunk_buffer.cc │ │ │ └── chunk_buffer.h │ │ ├── orc │ │ │ ├── orc.cc │ │ │ └── orc.h │ │ └── parquet │ │ │ ├── parquet.cc │ │ │ └── parquet.h │ ├── fuse_impl.cc │ ├── fuse_impl.h │ ├── fusermount.cc │ └── test │ │ ├── __init__.py │ │ ├── chunk_buffer_test.cc │ │ ├── conftest.py │ │ └── fuse_test.py ├── graph │ ├── CMakeLists.txt │ ├── README.rst │ ├── fragment │ │ ├── arrow_fragment.h │ │ ├── arrow_fragment.vineyard-mod │ │ ├── arrow_fragment_base.h │ │ ├── arrow_fragment_builder_impl.h │ │ ├── arrow_fragment_builder_int32_uint32.cc │ │ ├── arrow_fragment_builder_int32_uint64.cc │ │ ├── arrow_fragment_builder_int64_uint32.cc │ │ ├── arrow_fragment_builder_int64_uint64.cc │ │ ├── arrow_fragment_builder_string_uint32.cc │ │ ├── arrow_fragment_builder_string_uint64.cc │ │ ├── arrow_fragment_group.cc │ │ ├── arrow_fragment_group.h │ │ ├── arrow_fragment_impl.h │ │ ├── arrow_fragment_int32.cc │ │ ├── arrow_fragment_int64.cc │ │ ├── arrow_fragment_modifier.h │ │ ├── arrow_fragment_string.cc │ │ ├── fragment_traits.h │ │ ├── gar_fragment_builder.h │ │ ├── gar_fragment_builder_impl.h │ │ ├── gar_fragment_builder_int32.cc │ │ ├── gar_fragment_builder_int64.cc │ │ ├── graph_schema.cc │ │ ├── graph_schema.h │ │ ├── property_graph_types.h │ │ ├── property_graph_utils.h │ │ ├── property_graph_utils_impl.h │ │ ├── property_graph_utils_uint32.cc │ │ └── property_graph_utils_uint64.cc │ ├── loader │ │ ├── arrow_fragment_loader.cc │ │ ├── arrow_fragment_loader.h │ │ ├── arrow_fragment_loader_impl.h │ │ ├── arrow_fragment_loader_int32.cc │ │ ├── arrow_fragment_loader_int64.cc │ │ ├── arrow_fragment_loader_string.cc │ │ ├── basic_ev_fragment_loader.h │ │ ├── basic_ev_fragment_loader_impl.h │ │ ├── basic_ev_fragment_loader_int32.cc │ │ ├── basic_ev_fragment_loader_int64.cc │ │ ├── basic_ev_fragment_loader_string.cc │ │ ├── fragment_loader_utils.cc │ │ ├── fragment_loader_utils.h │ │ ├── gar_fragment_loader.cc │ │ ├── gar_fragment_loader.h │ │ ├── gar_fragment_loader_impl.h │ │ ├── gar_fragment_loader_int32.cc │ │ └── gar_fragment_loader_int64.cc │ ├── test │ │ ├── arrow_fragment_extend_test.cc │ │ ├── arrow_fragment_gar_test.cc │ │ └── arrow_fragment_test.cc │ ├── tools │ │ ├── fragment_loader_impl.h │ │ ├── fragment_loader_int32.cc │ │ ├── fragment_loader_int64.cc │ │ ├── fragment_loader_string.cc │ │ ├── graph_loader.cc │ │ └── graph_loader.h │ ├── utils │ │ ├── context_protocols.cc │ │ ├── context_protocols.h │ │ ├── error.h │ │ ├── grape_utils.h │ │ ├── mpi_utils.h │ │ ├── partitioner.h │ │ ├── string_collection.h │ │ ├── table_pipeline.h │ │ ├── table_shuffler.cc │ │ ├── table_shuffler.h │ │ ├── table_shuffler_beta.h │ │ ├── table_shuffler_impl.cc │ │ ├── table_shuffler_impl.h │ │ ├── thread_group.cc │ │ └── thread_group.h │ ├── vertex_map │ │ ├── arrow_local_vertex_map.h │ │ ├── arrow_local_vertex_map_impl.h │ │ ├── arrow_local_vertex_map_int32.cc │ │ ├── arrow_local_vertex_map_int64.cc │ │ ├── arrow_local_vertex_map_string.cc │ │ ├── arrow_vertex_map.h │ │ ├── arrow_vertex_map_builder.h │ │ ├── arrow_vertex_map_impl.h │ │ ├── arrow_vertex_map_int32.cc │ │ ├── arrow_vertex_map_int64.cc │ │ └── arrow_vertex_map_string.cc │ └── writer │ │ ├── arrow_fragment_writer.cc │ │ ├── arrow_fragment_writer.h │ │ ├── arrow_fragment_writer_impl.h │ │ ├── arrow_fragment_writer_int32.cc │ │ ├── arrow_fragment_writer_int64.cc │ │ ├── arrow_fragment_writer_string.cc │ │ ├── util.cc │ │ └── util.h ├── hosseinmoein-dataframe │ ├── CMakeLists.txt │ ├── hosseinmoein_dataframe.cc │ └── hosseinmoein_dataframe.h ├── io │ ├── .gitignore │ ├── CMakeLists.txt │ └── io │ │ ├── i_io_adaptor.h │ │ ├── io_factory.cc │ │ ├── io_factory.h │ │ ├── kafka_io_adaptor.cc │ │ ├── kafka_io_adaptor.h │ │ ├── local_io_adaptor.cc │ │ ├── local_io_adaptor.h │ │ └── utils.h ├── llm-cache │ ├── CMakeLists.txt │ ├── README.md │ ├── ds │ │ ├── config.h │ │ ├── kv_cache.cc │ │ ├── kv_cache.h │ │ ├── kv_cache_block.cc │ │ ├── kv_cache_block.h │ │ ├── kv_cache_manager.cc │ │ ├── kv_cache_manager.h │ │ ├── kv_tensor.h │ │ ├── refcnt_map.cc │ │ ├── refcnt_map.h │ │ ├── vineyard_file.cc │ │ └── vineyard_file.h │ ├── hash │ │ ├── hash_algorithm.h │ │ └── hasher.h │ ├── radix-tree │ │ ├── radix-tree.cc │ │ └── radix-tree.h │ ├── storage │ │ ├── blob_storage.cc │ │ ├── blob_storage.h │ │ ├── file_storage.cc │ │ ├── file_storage.h │ │ ├── local_file_storage.cc │ │ ├── local_file_storage.h │ │ ├── storage.h │ │ ├── vineyard_file_storage.cc │ │ └── vineyard_file_storage.h │ ├── tests │ │ ├── k8s-test │ │ │ ├── Dockerfile.master │ │ │ ├── Dockerfile.worker │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ ├── master.py │ │ │ ├── prompt-samples.txt │ │ │ ├── show_result.py │ │ │ ├── tokenize_prompt.py │ │ │ ├── worker.py │ │ │ └── yamls │ │ │ │ ├── master.yaml │ │ │ │ └── worker.yaml │ │ ├── kv_cache_benchmark_test.cc │ │ ├── kv_cache_hash_test.cc │ │ ├── kv_cache_local_file_test.cc │ │ ├── kv_cache_radix_tree_test.cc │ │ ├── kv_cache_test.cc │ │ ├── local_file_storage_gc_test.cc │ │ ├── refcnt_map_test.cc │ │ └── scripts │ │ │ ├── README.md │ │ │ └── dfs-fio.sh │ ├── thread_group.cc │ └── thread_group.h ├── malloc │ ├── CMakeLists.txt │ ├── malloc_wrapper.cc │ ├── malloc_wrapper.h │ ├── mimalloc_allocator.cc │ └── mimalloc_allocator.h └── msgpack │ ├── CMakeLists.txt │ ├── packed_object.cc │ ├── packed_object.h │ ├── pickle.cc │ ├── pickle.h │ ├── test │ ├── msgpack_test.cc │ └── pickle_test.cc │ ├── vbuffer.cc │ └── vbuffer.h ├── packages-java ├── .gitignore ├── CMakeLists.txt ├── excludes.txt ├── forward-headers.txt ├── pom.xml ├── src │ ├── main │ │ └── java │ │ │ └── .gitignore │ └── test │ │ └── java │ │ └── io │ │ └── v6d │ │ ├── arrow │ │ └── ArrowTest.java │ │ ├── std │ │ └── VectorTest.java │ │ └── vineyard │ │ └── ClientTest.java └── type-alias-oracles.txt ├── prometheus ├── Dockerfile.grok-exporter ├── grok_exporter.yml ├── prometheus.rules.yml └── prometheus.yml ├── pyproject.toml ├── python ├── client.cc ├── core.cc ├── error.cc ├── pybind11_docs.cc ├── pybind11_docs.h ├── pybind11_utils.cc ├── pybind11_utils.h ├── vineyard.cc └── vineyard │ ├── .gitignore │ ├── _C.pyi │ ├── __init__.py │ ├── __main__.py │ ├── bdist │ ├── .gitignore │ └── __init__.py │ ├── conftest.py │ ├── contrib │ ├── __init__.py │ ├── airflow │ │ ├── LICENSE │ │ ├── README.md │ │ ├── __init__.py │ │ ├── docker │ │ │ ├── Dockerfile │ │ │ ├── docker-compose.yaml │ │ │ └── docker-compose.yaml.diff │ │ ├── example_dags │ │ │ ├── __init__.py │ │ │ ├── v6d_dag.py │ │ │ ├── v6d_etl.py │ │ │ └── v6d_etl_pandas.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_base.py │ │ │ ├── test_pandas_dag.py │ │ │ └── test_python_dag.py │ │ ├── values.yaml │ │ └── xcom │ │ │ ├── __init__.py │ │ │ └── backend.py │ ├── dask │ │ ├── __init__.py │ │ ├── dask.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ └── test_dask.py │ ├── kedro │ │ ├── README.md │ │ ├── __init__.py │ │ ├── benchmark │ │ │ └── mlops │ │ │ │ ├── .gitignore │ │ │ │ ├── Dockerfile │ │ │ │ ├── Makefile │ │ │ │ ├── README.md │ │ │ │ ├── argo-aws-s3-benchmark.yml │ │ │ │ ├── argo-cloudpickle-benchmark.yml │ │ │ │ ├── argo-minio-s3-benchmark.yml │ │ │ │ ├── argo-vineyard-benchmark.yml │ │ │ │ ├── conf │ │ │ │ ├── README.md │ │ │ │ ├── aws-s3 │ │ │ │ │ ├── catalog.yml │ │ │ │ │ └── credentials.yml │ │ │ │ ├── base │ │ │ │ │ ├── catalog.yml │ │ │ │ │ ├── logging.yml │ │ │ │ │ └── parameters.yml │ │ │ │ ├── cloudpickle │ │ │ │ │ └── catalog.yml │ │ │ │ ├── local │ │ │ │ │ └── catalog.yml │ │ │ │ ├── minio-s3 │ │ │ │ │ ├── catalog.yml │ │ │ │ │ └── credentials.yml │ │ │ │ └── vineyard │ │ │ │ │ └── catalog.yml │ │ │ │ ├── data │ │ │ │ ├── 01_raw │ │ │ │ │ └── train.csv │ │ │ │ └── tmp │ │ │ │ │ └── .gitkeep │ │ │ │ ├── docs │ │ │ │ └── source │ │ │ │ │ ├── conf.py │ │ │ │ │ └── index.rst │ │ │ │ ├── minio-dev.yaml │ │ │ │ ├── notebooks │ │ │ │ └── .gitkeep │ │ │ │ ├── pyproject.toml │ │ │ │ ├── setup.cfg │ │ │ │ └── src │ │ │ │ ├── mlops │ │ │ │ ├── __init__.py │ │ │ │ ├── __main__.py │ │ │ │ ├── nodes.py │ │ │ │ ├── pipeline.py │ │ │ │ ├── pipeline_registry.py │ │ │ │ ├── pipelines │ │ │ │ │ └── __init__.py │ │ │ │ ├── settings.py │ │ │ │ └── utils.py │ │ │ │ ├── requirements.txt │ │ │ │ ├── setup.py │ │ │ │ └── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── pipelines │ │ │ │ └── __init__.py │ │ │ │ └── test_run.py │ │ ├── io │ │ │ ├── __init__.py │ │ │ └── dataset.py │ │ ├── plugins │ │ │ ├── __init__.py │ │ │ ├── argo.py │ │ │ ├── catalog.py │ │ │ ├── cli.py │ │ │ ├── docker.py │ │ │ ├── hook.py │ │ │ ├── starter.py │ │ │ └── templates │ │ │ │ ├── Dockerfile.tmpl │ │ │ │ └── argo_spec.tmpl │ │ └── runner │ │ │ ├── __init__.py │ │ │ ├── parallel_runner.py │ │ │ ├── sequential_runner.py │ │ │ └── thread_runner.py │ ├── ml │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dali.py │ │ ├── mxnet.py │ │ ├── tensorflow.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_dali.py │ │ │ ├── test_mxnet.py │ │ │ ├── test_tensorflow.py │ │ │ ├── test_torch.py │ │ │ ├── test_torcharrow.py │ │ │ └── test_xgboost.py │ │ ├── torch.py │ │ ├── torcharrow.py │ │ └── xgboost.py │ ├── pyspark │ │ ├── __init__.py │ │ ├── pyspark.py │ │ └── tests │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ └── test_pyspark.py │ └── ray │ │ ├── __init__.py │ │ ├── actor.py │ │ └── dataset.py │ ├── core │ ├── __init__.py │ ├── builder.py │ ├── client.py │ ├── codegen │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── cpp.py │ │ ├── java.py │ │ ├── main.py │ │ ├── parsing.py │ │ └── python.py │ ├── driver.py │ ├── resolver.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_client.py │ │ ├── test_context.py │ │ └── test_rpc_client.py │ └── utils.py │ ├── csi │ └── __init__.py │ ├── ctl.py │ ├── data │ ├── __init__.py │ ├── arrow.py │ ├── base.py │ ├── benchmarks │ │ ├── __init__.py │ │ └── test_tensor.py │ ├── dataframe.py │ ├── default.py │ ├── graph.py │ ├── index.py │ ├── pickle.py │ ├── series.py │ ├── tensor.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_arrow.py │ │ ├── test_base.py │ │ ├── test_dataframe.py │ │ ├── test_default.py │ │ ├── test_pickle.py │ │ ├── test_series.py │ │ ├── test_tensor.py │ │ └── test_velox.py │ ├── utils.py │ └── velox.py │ ├── deploy │ ├── __init__.py │ ├── __main__.py │ ├── _cobra.py │ ├── ctl.py │ ├── distributed.py │ ├── etcd.py │ ├── etcd.yaml │ ├── kubernetes.py │ ├── local.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_distributed.py │ │ ├── test_local.py │ │ ├── test_migration.py │ │ └── test_vineyardd_failure.py │ ├── utils.py │ └── vineyard.yaml.tpl │ ├── drivers │ ├── __init__.py │ └── io │ │ ├── README.rst │ │ ├── __init__.py │ │ ├── adaptors │ │ ├── __init__.py │ │ ├── deserializer.py │ │ ├── dump_dataframe.py │ │ ├── parse_bytes_to_dataframe.py │ │ ├── parse_dataframe_to_bytes.py │ │ ├── read_bytes.py │ │ ├── read_bytes_collection.py │ │ ├── read_orc.py │ │ ├── read_parquet.py │ │ ├── read_vineyard_dataframe.py │ │ ├── serializer.py │ │ ├── write_bytes.py │ │ ├── write_bytes_collection.py │ │ ├── write_orc.py │ │ ├── write_parquet.py │ │ └── write_vineyard_dataframe.py │ │ ├── fsspec_adaptors.py │ │ ├── kube_ssh.sh │ │ ├── ossfs.py │ │ ├── ssh.sh │ │ ├── stream.py │ │ └── tests │ │ ├── __init__.py │ │ ├── test_migrate_stream.py │ │ ├── test_open.py │ │ ├── test_ossfs.py │ │ └── test_serialize.py │ ├── io │ ├── __init__.py │ ├── byte.py │ ├── core.py │ ├── dataframe.py │ ├── parallel.py │ ├── recordbatch.py │ ├── scripts │ │ ├── __init__.py │ │ └── stream_bash.sh │ ├── serialization.py │ ├── stream.py │ ├── tests │ │ ├── __init__.py │ │ └── test_stream.py │ └── utils.py │ ├── launcher │ ├── __init__.py │ ├── launcher.py │ └── script.py │ ├── llm │ ├── __init__.py │ ├── cache.cc │ ├── cache.py │ └── tests │ │ ├── __init__.py │ │ └── test_llm.py │ ├── py.typed │ ├── shared_memory │ ├── __init__.py │ ├── shared_memory.py │ └── tests │ │ ├── __init__.py │ │ └── test_shared_memory.py │ ├── systemd │ ├── __init__.py │ └── __main__.py │ └── version.py.in ├── requirements-dev.txt ├── requirements-extra.txt ├── requirements-kubernetes.txt ├── requirements-setup.txt ├── requirements.txt ├── rust ├── .gitignore ├── Cargo.toml ├── README.md ├── deny.toml ├── vineyard-datafusion │ ├── Cargo.toml │ └── src │ │ ├── ds │ │ ├── dataframe.rs │ │ ├── dataframe_test.rs │ │ └── mod.rs │ │ └── lib.rs ├── vineyard-integration-testing │ ├── Cargo.toml │ └── src │ │ ├── ds │ │ ├── arrow_test.rs │ │ ├── mod.rs │ │ ├── numpy_test.rs │ │ ├── pandas_test.rs │ │ └── polars_test.rs │ │ └── lib.rs ├── vineyard-polars │ ├── Cargo.toml │ └── src │ │ ├── ds │ │ ├── dataframe.rs │ │ ├── dataframe_test.rs │ │ └── mod.rs │ │ └── lib.rs └── vineyard │ ├── Cargo.toml │ └── src │ ├── client │ ├── client.rs │ ├── ds │ │ ├── blob.rs │ │ ├── blob_test.rs │ │ ├── mod.rs │ │ ├── object.rs │ │ ├── object_factory.rs │ │ └── object_meta.rs │ ├── io.rs │ ├── ipc_client.rs │ ├── ipc_client_test.rs │ ├── mod.rs │ ├── rpc_client.rs │ └── rpc_client_test.rs │ ├── common │ ├── memory │ │ ├── fling.rs │ │ ├── mod.rs │ │ └── payload.rs │ ├── mod.rs │ └── util │ │ ├── arrow.rs │ │ ├── json.rs │ │ ├── mod.rs │ │ ├── protocol.rs │ │ ├── status.rs │ │ ├── typename.rs │ │ └── uuid.rs │ ├── ds │ ├── array.rs │ ├── array_test.rs │ ├── arrow.rs │ ├── arrow_test.rs │ ├── arrow_utils.rs │ ├── dataframe.rs │ ├── dataframe_test.rs │ ├── hashmap.rs │ ├── hashmap_test.rs │ ├── mod.rs │ ├── tensor.rs │ └── tensor_test.rs │ └── lib.rs ├── setup.cfg ├── setup.cfg.in ├── setup.py ├── setup_airflow.py ├── setup_bdist.py ├── setup_dask.py ├── setup_io.py ├── setup_kedro.py ├── setup_llm.py ├── setup_ml.py ├── setup_pyspark.py ├── setup_ray.py ├── src ├── client │ ├── client.cc │ ├── client.h │ ├── client_base.cc │ ├── client_base.h │ ├── ds │ │ ├── blob.cc │ │ ├── blob.h │ │ ├── collection.h │ │ ├── core_types.cc │ │ ├── core_types.h │ │ ├── factory │ │ │ └── registry.cc │ │ ├── i_object.cc │ │ ├── i_object.h │ │ ├── object_factory.cc │ │ ├── object_factory.h │ │ ├── object_meta.cc │ │ ├── object_meta.h │ │ ├── remote_blob.cc │ │ ├── remote_blob.h │ │ └── stream.h │ ├── io.cc │ ├── io.h │ ├── rpc_client.cc │ ├── rpc_client.h │ └── utils.h ├── common │ ├── backtrace │ │ ├── backtrace.hpp │ │ └── backtrace_on_terminate.hpp │ ├── compression │ │ ├── compressor.cc │ │ └── compressor.h │ ├── memory │ │ ├── cuda_ipc.cc │ │ ├── cuda_ipc.h │ │ ├── fling.cc │ │ ├── fling.h │ │ ├── memcpy.h │ │ ├── mimalloc.h │ │ ├── payload.cc │ │ └── payload.h │ ├── rdma │ │ ├── libfabric.map │ │ ├── rdma.cc │ │ ├── rdma.h │ │ ├── rdma_client.cc │ │ ├── rdma_client.h │ │ ├── rdma_server.cc │ │ ├── rdma_server.h │ │ └── util.h │ └── util │ │ ├── arrow.h │ │ ├── asio.h │ │ ├── base64.cc │ │ ├── base64.h │ │ ├── blocking_queue.h │ │ ├── callback.h │ │ ├── config.h │ │ ├── config.h.in │ │ ├── env.cc │ │ ├── env.h │ │ ├── flags.h │ │ ├── functions.h │ │ ├── json.cc │ │ ├── json.h │ │ ├── lifecycle.h │ │ ├── likely.h │ │ ├── logging.h │ │ ├── macros.h │ │ ├── protocols.cc │ │ ├── protocols.h │ │ ├── static_if.h │ │ ├── status.cc │ │ ├── status.h │ │ ├── typename.h │ │ ├── uuid.cc │ │ ├── uuid.h │ │ └── version.h └── server │ ├── async │ ├── ipc_server.cc │ ├── ipc_server.h │ ├── rpc_server.cc │ ├── rpc_server.h │ ├── socket_server.cc │ └── socket_server.h │ ├── memory │ ├── allocator.cc │ ├── allocator.h │ ├── cuda_allocator.cc │ ├── cuda_allocator.h │ ├── dlmalloc.cc │ ├── dlmalloc.h │ ├── malloc.cc │ ├── malloc.h │ ├── memory.cc │ ├── memory.h │ ├── mimalloc.cc │ ├── mimalloc.h │ ├── stream_store.cc │ ├── stream_store.h │ └── usage.h │ ├── server │ ├── vineyard_runner.cc │ ├── vineyard_runner.h │ ├── vineyard_server.cc │ └── vineyard_server.h │ ├── services │ ├── etcd_meta_service.cc │ ├── etcd_meta_service.h │ ├── local_meta_service.cc │ ├── local_meta_service.h │ ├── meta_service.cc │ ├── meta_service.h │ ├── redis_meta_service.cc │ └── redis_meta_service.h │ ├── util │ ├── etcd_launcher.cc │ ├── etcd_launcher.h │ ├── etcd_member.cc │ ├── etcd_member.h │ ├── file_io_adaptor.cc │ ├── file_io_adaptor.h │ ├── kubectl.cc │ ├── kubectl.h │ ├── meta_tree.cc │ ├── meta_tree.h │ ├── metrics.h │ ├── proc.cc │ ├── proc.h │ ├── redis_launcher.cc │ ├── redis_launcher.h │ ├── remote.cc │ ├── remote.h │ ├── spec_resolvers.cc │ ├── spec_resolvers.h │ ├── spill_file.cc │ └── spill_file.h │ └── vineyardd.cc ├── test ├── CMakeLists.txt ├── allocator_test.cc ├── array_test.cc ├── array_two_clients_test.cc ├── arrow_data_structure_test.cc ├── auth_test.cc ├── clear_test.cc ├── compressor_test.cc ├── concurrent_id_test.cc ├── concurrent_lru_spill_test.cc ├── concurrent_memcpy_test.cc ├── custom_vector_test.cc ├── dataframe_test.cc ├── delete_test.cc ├── etcd_member_test.cc ├── fetch_remote_object_test.cc ├── get_blob_disk_test.cc ├── get_blob_test.cc ├── get_object_test.cc ├── get_wait_test.cc ├── global_object_test.cc ├── gpumalloc_test.cu ├── hashmap_largefile_test.cc ├── hashmap_mvcc_test.cc ├── hashmap_test.cc ├── hosseinmoein_dataframe_test.cc ├── invalid_connect_test.cc ├── io_test.cc ├── json_utils_test.cc ├── large_meta_test.cc ├── list_object_test.cc ├── lru_spill_test.cc ├── lru_test.cc ├── mimalloc_test.cc ├── mutable_blob_test.cc ├── name_test.cc ├── object_meta_test.cc ├── perfect_hashmap_test.cc ├── persist_test.cc ├── plasma_test.cc ├── rdma_blob_perf_test.cc ├── rdma_test.cc ├── release_test.cc ├── remote_buffer_test.cc ├── remote_buffer_with_rdma_test.cc ├── rpc_delete_test.cc ├── rpc_get_object_test.cc ├── rpc_test.cc ├── runner.py ├── scalar_test.cc ├── sequence_test.cc ├── server_status_test.cc ├── session_test.cc ├── shallow_copy_test.cc ├── shared_memory_test.cc ├── signature_test.cc ├── spill_test.cc ├── stream_test.cc ├── tensor_test.cc ├── typename_test.cc ├── version_test.cc └── vineyard-cmake-example │ ├── CMakeLists.txt │ └── example.cc ├── thirdparty ├── MurmurHash3 │ ├── MurmurHash3.cc │ └── MurmurHash3.h ├── cityhash │ └── cityhash.hpp ├── dlmalloc │ └── dlmalloc.c ├── flat_hash_map │ ├── bytell_hash_map.hpp │ ├── flat_hash_map.hpp │ └── unordered_map.hpp ├── gulrak │ └── filesystem.hpp ├── nlohmann-json │ └── single_include │ │ └── nlohmann │ │ └── json.hpp ├── rax │ ├── radix.cc │ ├── radix.h │ └── rax_malloc.h ├── redis-plus-plus-shim │ └── recipes │ │ ├── redlock.cpp │ │ └── redlock.h └── wyhash │ └── wyhash.hpp ├── vineyard-config-version.in.cmake └── vineyard-config.in.cmake /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | DerivePointerAlignment: false 3 | PointerAlignment: Left 4 | Cpp11BracedListStyle: true 5 | IndentCaseLabels: false 6 | AllowShortBlocksOnASingleLine: true 7 | AllowShortLoopsOnASingleLine: false 8 | AllowShortIfStatementsOnASingleLine: false 9 | Standard: 'Cpp11' 10 | SpaceAfterCStyleCast: true 11 | AlignAfterOpenBracket: Align 12 | SortIncludes: true 13 | IncludeBlocks: Preserve 14 | ForEachMacros: 15 | - BOOST_FOREACH 16 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | **/.git 3 | 4 | # contents from .gitignore 5 | build/ 6 | build-mac/ 7 | install/ 8 | build-static/ 9 | build-static-mac/ 10 | install-static/ 11 | build-docker/ 12 | install-docker/ 13 | build-docker-gsa/ 14 | install-docker-gsa/ 15 | *.code-workspace 16 | .vscode 17 | .idea 18 | .DS_store 19 | cmake-build-debug/ 20 | modules/graph/thirdparty/**/build/ 21 | 22 | # for python packaging 23 | dist/ 24 | **/*.egg-info/ 25 | **/*.so 26 | **/*.dylib 27 | **/*.a 28 | **/*.pyc 29 | **/*.bin 30 | **/*.egg 31 | 32 | # generated files 33 | *.vineyard.h 34 | 35 | # etcd data directory during testing 36 | default.etcd/ 37 | 38 | # tmp data directory during building image 39 | docker/dist/ 40 | 41 | # docs 42 | docs/_build/ 43 | 44 | # go vendor 45 | /**/vendor/ 46 | 47 | # Rust build 48 | /**/target/ 49 | 50 | # k8s 51 | k8s/bin/ 52 | k8s/vendor/ 53 | k8s/examples/ 54 | k8s/test/ 55 | k8s/hack/ 56 | !k8s/config/scheduler/config.yaml 57 | 58 | # artifacts 59 | wheels/ 60 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Vineyard modules 2 | 3 | *.vineyard-mod linguist-language=C++ 4 | 5 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | What do these changes do? 6 | ------------------------- 7 | 8 | 9 | 10 | 11 | Related issue number 12 | -------------------- 13 | 14 | 15 | 16 | Fixes #issue number 17 | 18 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "50...80" 8 | 9 | status: 10 | project: off 11 | patch: off 12 | 13 | parsers: 14 | gcov: 15 | branch_detection: 16 | conditional: yes 17 | loop: yes 18 | method: no 19 | macro: no 20 | 21 | comment: 22 | layout: "reach,diff,flags,tree" 23 | behavior: default 24 | require_changes: no 25 | 26 | ignore: 27 | - "modules/**/*" 28 | - "python/*" 29 | - "python/**/*" 30 | - "test/*" 31 | - "test/**/*" 32 | - "thirdparty/**/*" 33 | -------------------------------------------------------------------------------- /.github/workflows/spelling.yaml: -------------------------------------------------------------------------------- 1 | name: Spell Check with CSpell 2 | on: 3 | - pull_request 4 | 5 | jobs: 6 | run: 7 | name: Spell Check with Typos 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | with: 12 | fetch-depth: 2 13 | 14 | - name: Get changed files 15 | id: changed-files 16 | uses: tj-actions/changed-files@v42 17 | with: 18 | since_last_remote_commit: true 19 | 20 | - name: List all changed files 21 | env: 22 | ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} 23 | run: | 24 | for file in ${ALL_CHANGED_FILES}; do 25 | echo "File $file was changed" 26 | done 27 | 28 | - name: Spell checking 29 | uses: streetsidesoftware/cspell-action@v5 30 | with: 31 | files: ${{ steps.changed-files.outputs.all_changed_files }} 32 | verbose: true 33 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/zricethezav/gitleaks 3 | rev: v8.15.0 4 | hooks: 5 | - id: gitleaks 6 | args: 7 | - '--verbose' 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Vineyard Community Code of Conduct 2 | 3 | Vineyard follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | 5 | In cases of abusive, harassing, or any unacceptable behaviors, please don't hesitate to contact the project team at [coc@v6d.io](mailto:coc@v6d.io). 6 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | # extensions=.vineyard-mod 2 | headers=hpp,cuh,hh,h,hxx,h++,vineyard-mod 3 | 4 | # Disable the unapproved checks, since this is a preference of the Chromium team. 5 | filter=-build/c++11 6 | 7 | # Non-const reference parameter is OK to us. 8 | filter=-runtime/references 9 | 10 | # C-style string to initialize static const std::string 11 | filter=-runtime/string 12 | 13 | # TODO or FIXME is fine 14 | filter=-readability/todo 15 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | # See: https://go.k8s.io/owners 2 | approvers: 3 | - acezen 4 | - andydiwenzhu 5 | - luoxiaojian 6 | - sighingnow 7 | - siyuan0322 8 | - wenyuanyu 9 | reviewers: 10 | - acezen 11 | - andydiwenzhu 12 | - luoxiaojian 13 | - sighingnow 14 | - siyuan0322 15 | - wenyuanyu 16 | -------------------------------------------------------------------------------- /benchmark/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(BUILD_VINEYARD_MALLOC) 2 | add_subdirectory(alloc_test) 3 | add_subdirectory(blob_test) 4 | endif() 5 | -------------------------------------------------------------------------------- /benchmark/alloc_test/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/benchmark/alloc_test/.gitignore -------------------------------------------------------------------------------- /benchmark/blob_test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(BENCH_BLOB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/blob_test.cc) 2 | 3 | if(BUILD_VINEYARD_BENCHMARKS_ALL) 4 | set(blob_benchmark_options "") 5 | else() 6 | set(blob_benchmark_options "EXCLUDE_FROM_ALL") 7 | endif() 8 | 9 | add_executable(blob_benchmark ${blob_benchmark_options} ${BENCH_BLOB_SRCS}) 10 | 11 | target_include_directories(blob_benchmark PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 12 | 13 | target_link_libraries(blob_benchmark PRIVATE vineyard_client) 14 | 15 | add_dependencies(vineyard_benchmarks blob_benchmark) 16 | -------------------------------------------------------------------------------- /charts/.gitignore: -------------------------------------------------------------------------------- 1 | index.yaml 2 | *.tgz -------------------------------------------------------------------------------- /charts/artifacthub-repo.yml: -------------------------------------------------------------------------------- 1 | # Artifact Hub repository metadata file 2 | # 3 | # Some settings like the verified publisher flag or the ignored packages won't 4 | # be applied until the next time the repository is processed. Please keep in 5 | # mind that the repository won't be processed if it has not changed since the 6 | # last time it was processed. Depending on the repository kind, this is checked 7 | # in a different way. For Helm http based repositories, we consider it has 8 | # changed if the `index.yaml` file changes. For git based repositories, it does 9 | # when the hash of the last commit in the branch you set up changes. This does 10 | # NOT apply to ownership claim operations, which are processed immediately. 11 | # 12 | repositoryID: ee1ccf13-0703-4a39-bf71-129c24da07c0 13 | owners: # (optional, used to claim repository ownership) 14 | - name: Tao He 15 | email: sighingnow@email.com 16 | - name: Ye Cao 17 | email: caoye.cao@alibaba-inc.com -------------------------------------------------------------------------------- /charts/vineyard-operator/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | Thanks for installing {{ .Chart.Name | upper }}, release at namespace: {{ .Release.Namespace }}, name: {{ .Release.Name }}. 2 | 3 | To learn more about the release, try: 4 | 5 | $ helm status {{ .Release.Name }} -n {{ .Release.Namespace }} # get status of running vineyard operator 6 | $ helm get all {{ .Release.Name }} -n {{ .Release.Namespace }} # get all deployment yaml of vineyard operator 7 | 8 | To uninstall the release, try: 9 | 10 | $ helm uninstall {{ .Release.Name }} -n {{ .Release.Namespace }} 11 | 12 | You could get all details about vineyard operator in the doc [https://v6d.io/notes/cloud-native/vineyard-operator.html], just have fun with vineyard operator! -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/kube-scheduler-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-kube-scheduler-rolebinding 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: system:kube-scheduler 11 | subjects: 12 | - kind: ServiceAccount 13 | name: '{{ include "vineyard-operator.fullname" . }}-manager' 14 | namespace: '{{ .Release.Namespace }}' -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/metrics-reader-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-metrics-reader 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | rules: 8 | - nonResourceURLs: 9 | - /metrics 10 | verbs: 11 | - get -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/metrics-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-controller-manager-metrics-service 5 | labels: 6 | control-plane: controller-manager 7 | {{- include "vineyard-operator.labels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.metricsService.type }} 10 | selector: 11 | control-plane: controller-manager 12 | {{- include "vineyard-operator.selectorLabels" . | nindent 4 }} 13 | ports: 14 | {{- .Values.metricsService.ports | toYaml | nindent 2 -}} -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/proxy-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-proxy-role 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | rules: 8 | - apiGroups: 9 | - authentication.k8s.io 10 | resources: 11 | - tokenreviews 12 | verbs: 13 | - create 14 | - apiGroups: 15 | - authorization.k8s.io 16 | resources: 17 | - subjectaccessreviews 18 | verbs: 19 | - create 20 | --- 21 | apiVersion: rbac.authorization.k8s.io/v1 22 | kind: ClusterRoleBinding 23 | metadata: 24 | name: {{ include "vineyard-operator.fullname" . }}-proxy-rolebinding 25 | labels: 26 | {{- include "vineyard-operator.labels" . | nindent 4 }} 27 | roleRef: 28 | apiGroup: rbac.authorization.k8s.io 29 | kind: ClusterRole 30 | name: '{{ include "vineyard-operator.fullname" . }}-proxy-role' 31 | subjects: 32 | - kind: ServiceAccount 33 | name: default 34 | namespace: '{{ .Release.Namespace }}' -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/scheduler-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-scheduler-rolebinding 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: '{{ include "vineyard-operator.fullname" . }}-manager-role' 11 | subjects: 12 | - kind: ServiceAccount 13 | name: '{{ include "vineyard-operator.fullname" . }}-manager' 14 | namespace: '{{ .Release.Namespace }}' -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/volume-scheduler-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-volume-scheduler-rolebinding 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | roleRef: 8 | apiGroup: rbac.authorization.k8s.io 9 | kind: ClusterRole 10 | name: system:volume-scheduler 11 | subjects: 12 | - kind: ServiceAccount 13 | name: '{{ include "vineyard-operator.fullname" . }}-manager' 14 | namespace: '{{ .Release.Namespace }}' -------------------------------------------------------------------------------- /charts/vineyard-operator/templates/webhook-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "vineyard-operator.fullname" . }}-webhook-service 5 | labels: 6 | {{- include "vineyard-operator.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.webhookService.type }} 9 | selector: 10 | control-plane: controller-manager 11 | {{- include "vineyard-operator.selectorLabels" . | nindent 4 }} 12 | ports: 13 | {{- .Values.webhookService.ports | toYaml | nindent 2 -}} -------------------------------------------------------------------------------- /cmake/CheckGCCABI.cmake: -------------------------------------------------------------------------------- 1 | macro(check_gcc_cxx11abi) 2 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 3 | execute_process(COMMAND "${CMAKE_CXX_COMPILER}" -v 4 | OUTPUT_VARIABLE GCC_VERSION_OUT 5 | ERROR_VARIABLE GCC_VERSION_ERR) 6 | if(GCC_VERSION_OUT MATCHES ".*with-default-libstdcxx-abi=new.*" OR GCC_VERSION_ERR MATCHES ".*with-default-libstdcxx-abi=new.*") 7 | set(GCC_USE_CXX11_ABI 1) 8 | else() 9 | if(GCC_VERSION_OUT MATCHES ".*gcc4-compatible.*" OR GCC_VERSION_ERR MATCHES ".*gcc4-compatible.*") 10 | set(GCC_USE_CXX11_ABI 0) 11 | else() 12 | set(GCC_USE_CXX11_ABI 1) 13 | endif() 14 | endif() 15 | else() 16 | set(GCC_USE_CXX11_ABI 1) 17 | endif() 18 | endmacro(check_gcc_cxx11abi) 19 | -------------------------------------------------------------------------------- /docker/Dockerfile.vineyard-python: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.11 2 | 3 | FROM python:$PYTHON_VERSION 4 | 5 | RUN pip3 install --no-cache-dir \ 6 | vineyard \ 7 | vineyard-io \ 8 | ipython 9 | -------------------------------------------------------------------------------- /docker/dev/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | RUN chmod 1777 /tmp 4 | 5 | RUN mkdir -p /workspace/build_scripts 6 | 7 | COPY ./build_scripts/install-deps.sh /workspace/build_scripts 8 | RUN cd /workspace && bash ./build_scripts/install-deps.sh 9 | 10 | COPY ./build_scripts/install-arrow.sh /workspace/build_scripts 11 | RUN cd /workspace && bash ./build_scripts/install-arrow.sh 12 | 13 | COPY ./build_scripts/install-miscs.sh /workspace/build_scripts 14 | COPY ./build_scripts/requirements.txt /workspace/build_scripts 15 | RUN cd /workspace && bash ./build_scripts/install-miscs.sh 16 | -------------------------------------------------------------------------------- /docker/dev/build_scripts/install-arrow.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | set -o pipefail 5 | 6 | export DEBIAN_FRONTEND=noninteractive 7 | export DEBCONF_NONINTERACTIVE_SEEN=true 8 | 9 | # install apache-arrow 10 | wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb 11 | apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb 12 | apt update 13 | apt install -y libarrow-dev=10.0.1-1 libparquet-dev=10.0.1-1 14 | 15 | # install pyarrow from scratch 16 | sudo pip3 install --no-binary pyarrow pyarrow==10.0.1 17 | 18 | # apt-get cleanup 19 | apt-get autoclean 20 | rm -rf ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb 21 | -------------------------------------------------------------------------------- /docker/dev/build_scripts/install-miscs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | set -o pipefail 5 | 6 | # install python packages for codegen, and io adaptors 7 | sudo pip3 install -U "Pygments>=2.4.1" 8 | sudo pip3 install -r build_scripts/requirements.txt 9 | 10 | # install clang-format 11 | sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-1d7ec53d/clang-format-11_linux-amd64 --output /usr/bin/clang-format 12 | sudo chmod +x /usr/bin/clang-format 13 | -------------------------------------------------------------------------------- /docker/dev/build_scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | argcomplete 2 | black>=22.3.0 3 | breathe 4 | click 5 | docutils==0.16 6 | etcd-distro 7 | flake8>=4.0.1 8 | flake8-comprehensions 9 | flake8-logging-format 10 | furo # sphinx theme 11 | isort>=5.10.1 12 | jinja2>=3.0.0 13 | libclang 14 | linkify-it-py 15 | makefun 16 | myst-parser>=0.13.0 17 | nbsphinx 18 | numpy>=1.18.5 19 | pandas<1.0.0; python_version<"3.6" 20 | pandas>=1.0.0; python_version>="3.7" 21 | pandas<1.2.0; python_version<"3.7" 22 | pickle5; python_version<="3.7" 23 | psutil 24 | pyarrow 25 | pygments>=2.4.1 26 | pytest 27 | pytest-benchmark 28 | pytest-datafiles 29 | pytest-timeout 30 | setuptools 31 | shared-memory38; python_version<="3.7" 32 | sortedcontainers 33 | sphinx>=3.0.2,<6 34 | sphinx-copybutton 35 | sphinxemoji 36 | sphinxext-opengraph 37 | sphinx-panels 38 | treelib 39 | -------------------------------------------------------------------------------- /docker/vineyard-fluid-fuse/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PLATFORM=x86_64 2 | FROM ghcr.io/v6d-io/v6d/vineyardd:alpine-latest_x86_64 as base_x86_64 3 | FROM ghcr.io/v6d-io/v6d/vineyardd:alpine-latest_aarch64 as base_aarch64 4 | 5 | FROM base_$PLATFORM 6 | 7 | COPY ./entrypoint.sh /entrypoint.sh 8 | RUN chmod +x /entrypoint.sh 9 | 10 | ENTRYPOINT ["bash", "/entrypoint.sh"] 11 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/.nojekyll -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | v6d.io -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile doxygen 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile doxygen 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | doxygen: 23 | @mkdir -p _build 24 | @doxygen Doxyfile || true 25 | 26 | -------------------------------------------------------------------------------- /docs/_static/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/_static/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /docs/_static/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/_static/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {%- block extrahead %} 4 | 5 | {{ super() }} 6 | 7 | {% endblock %} 8 | -------------------------------------------------------------------------------- /docs/artifacts/vineyard-sigmod-2023.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/artifacts/vineyard-sigmod-2023.pdf -------------------------------------------------------------------------------- /docs/images/airflow_etl.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/airflow_etl.jpg -------------------------------------------------------------------------------- /docs/images/cncf-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/cncf-small.png -------------------------------------------------------------------------------- /docs/images/cncf-tiny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/cncf-tiny.png -------------------------------------------------------------------------------- /docs/images/cncf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/cncf.png -------------------------------------------------------------------------------- /docs/images/dask-tf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/dask-tf.jpg -------------------------------------------------------------------------------- /docs/images/data_sharing_with_deployment.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/data_sharing_with_deployment.jpg -------------------------------------------------------------------------------- /docs/images/data_sharing_with_sidecar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/data_sharing_with_sidecar.jpg -------------------------------------------------------------------------------- /docs/images/fraud-detection-job.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/fraud-detection-job.jpg -------------------------------------------------------------------------------- /docs/images/kubeflow_create_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/kubeflow_create_run.png -------------------------------------------------------------------------------- /docs/images/kubeflow_upload_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/kubeflow_upload_pipeline.png -------------------------------------------------------------------------------- /docs/images/sandbox-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/sandbox-small.png -------------------------------------------------------------------------------- /docs/images/sandbox-tiny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/sandbox-tiny.png -------------------------------------------------------------------------------- /docs/images/sandbox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/sandbox.png -------------------------------------------------------------------------------- /docs/images/vineyard-logo-h.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard-logo-h.png -------------------------------------------------------------------------------- /docs/images/vineyard-logo-notext.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard-logo-notext.png -------------------------------------------------------------------------------- /docs/images/vineyard-logo-rect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard-logo-rect.png -------------------------------------------------------------------------------- /docs/images/vineyard-logo-v6d.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard-logo-v6d.png -------------------------------------------------------------------------------- /docs/images/vineyard-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard-logo.png -------------------------------------------------------------------------------- /docs/images/vineyard.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard.ico -------------------------------------------------------------------------------- /docs/images/vineyard_arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_arch.jpg -------------------------------------------------------------------------------- /docs/images/vineyard_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_compare.png -------------------------------------------------------------------------------- /docs/images/vineyard_composable.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_composable.jpg -------------------------------------------------------------------------------- /docs/images/vineyard_distributed_tensor.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_distributed_tensor.jpg -------------------------------------------------------------------------------- /docs/images/vineyard_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_logo.png -------------------------------------------------------------------------------- /docs/images/vineyard_operator_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/docs/images/vineyard_operator_arch.png -------------------------------------------------------------------------------- /docs/notes/cloud-native/vineyardctl.md: -------------------------------------------------------------------------------- 1 | ../../../k8s/cmd/README.md -------------------------------------------------------------------------------- /docs/notes/developers/contributing.rst: -------------------------------------------------------------------------------- 1 | .. This file is just a placeholder to refer the top-level CONTRIBUTING.rst 2 | to sphinx doc workspace. 3 | 4 | .. include:: ../../../CONTRIBUTING.rst 5 | -------------------------------------------------------------------------------- /docs/notes/integration-bigdata.rst: -------------------------------------------------------------------------------- 1 | Big-data on Vineyard 2 | ==================== 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | :caption: TOC 7 | :hidden: 8 | 9 | integration/dask.rst 10 | integration/ml.rst 11 | 12 | Vineyard serves as a powerful data-sharing engine, seamlessly integrating with 13 | a variety of big-data computing platforms. This includes machine learning 14 | frameworks and the distributed data processing engine, Dask. 15 | 16 | .. panels:: 17 | :header: text-center 18 | :column: col-lg-12 p-2 19 | 20 | .. link-button:: integration/ml 21 | :type: ref 22 | :text: Machine Learning 23 | :classes: btn-block stretched-link 24 | ^^^^^^^^^^^^ 25 | Executing machine learning workflows on top of vineyard. 26 | 27 | --- 28 | 29 | .. link-button:: integration/dask 30 | :type: ref 31 | :text: Dask 32 | :classes: btn-block stretched-link 33 | ^^^^^^^^^^^^ 34 | Using vineyard as the data source / sink of dask computations. 35 | -------------------------------------------------------------------------------- /docs/notes/integration/ray.rst: -------------------------------------------------------------------------------- 1 | Ray on Vineyard 2 | =============== 3 | -------------------------------------------------------------------------------- /docs/notes/references/crds.md: -------------------------------------------------------------------------------- 1 | ../../../k8s/apis/k8s/v1alpha1/README.md -------------------------------------------------------------------------------- /docs/summer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Vineyard • Summer of Code 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /docs/tutorials/tutorials.rst: -------------------------------------------------------------------------------- 1 | .. Gather all tutorials as an index page 2 | 3 | :orphan: 4 | 5 | .. include:: ../tutorials/data-processing.rst 6 | .. include:: ../tutorials/kubernetes.rst 7 | .. include:: ../tutorials/extending.rst 8 | -------------------------------------------------------------------------------- /go.work: -------------------------------------------------------------------------------- 1 | go 1.19 2 | 3 | use ./go/vineyard 4 | 5 | use ./k8s 6 | -------------------------------------------------------------------------------- /go/vineyard/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Kubernetes Generated files - skip generated files, except for vendored files 17 | 18 | !vendor/**/zz_generated.* 19 | 20 | # editor and IDE paraphernalia 21 | .idea 22 | *.swp 23 | *.swo 24 | *~ 25 | 26 | # vendor, used for code-generate only 27 | /vendor/ 28 | -------------------------------------------------------------------------------- /go/vineyard/Makefile: -------------------------------------------------------------------------------- 1 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 2 | ifeq (,$(shell go env GOBIN)) 3 | GOBIN=$(shell go env GOPATH)/bin 4 | else 5 | GOBIN=$(shell go env GOBIN) 6 | endif 7 | 8 | GOLINT := $(shell command -v ${GOBIN}/golangci-lint --version 2>/dev/null) 9 | 10 | ## Tool Versions 11 | GOLANGCI_LINT_VERSION ?= v1.49.0 12 | 13 | all: fmt test 14 | 15 | # Run golangci-lint 16 | lint: 17 | ifeq (${GOLINT},) 18 | @echo "golangci-lint not exist, installing it..." 19 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) 20 | endif 21 | ${GOBIN}/golangci-lint run ./... 22 | .PHONY: golint 23 | 24 | # Run tests 25 | test: 26 | go test ./... 27 | .PHONY: test 28 | 29 | # Run go fmt against code 30 | fmt: 31 | go fmt ./... 32 | .PHONY: fmt 33 | 34 | # Run go vet against code 35 | vet: 36 | go vet ./... 37 | .PHONY: vet 38 | 39 | vendor: 40 | go mod tidy 41 | go mod vendor 42 | .PHONY: vendor 43 | -------------------------------------------------------------------------------- /go/vineyard/README.md: -------------------------------------------------------------------------------- 1 | Vineyard Go SDK 2 | --------------- 3 | 4 | Vineyard SDK is a go bindings for the vineyard client and data structure abstractions. 5 | -------------------------------------------------------------------------------- /go/vineyard/hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ -------------------------------------------------------------------------------- /go/vineyard/pkg/common/memory/slice.go: -------------------------------------------------------------------------------- 1 | package memory 2 | 3 | import ( 4 | "unsafe" 5 | 6 | "github.com/v6d-io/v6d/go/vineyard/pkg/common/types" 7 | ) 8 | 9 | func Slice(s []byte, offset, length uint64) []byte { 10 | return s[offset : offset+length] 11 | } 12 | 13 | func Cast[T types.Number](s []byte, length uint64) []T { 14 | return unsafe.Slice((*T)(unsafe.Pointer(&s[0])), length) 15 | } 16 | 17 | func CastFrom[T types.Number](pointer unsafe.Pointer, length uint64) []T { 18 | return unsafe.Slice((*T)(pointer), length) 19 | } 20 | -------------------------------------------------------------------------------- /go/vineyard/pkg/common/protocol_test.go: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | package common 17 | -------------------------------------------------------------------------------- /go/vineyard/pkg/common/types/traits.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "golang.org/x/exp/constraints" 5 | ) 6 | 7 | type Number interface { 8 | constraints.Integer | constraints.Float 9 | } 10 | -------------------------------------------------------------------------------- /go/vineyard/pkg/common/util.go: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | package common 17 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *.iml 3 | .classpath 4 | .project 5 | .factorypath 6 | .settings/ 7 | hs_err_pid*.log 8 | 9 | -------------------------------------------------------------------------------- /java/.mvn/jvm.config: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/java/.mvn/jvm.config -------------------------------------------------------------------------------- /java/apache-2.0.txt: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | -------------------------------------------------------------------------------- /java/core/.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | cmake-build/ 3 | 4 | -------------------------------------------------------------------------------- /java/core/src/main/java/com/google/common/base/StopwatchContext.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package com.google.common.base; 16 | 17 | public class StopwatchContext { 18 | public static Stopwatch create() { 19 | return new Stopwatch().start(); 20 | } 21 | 22 | public static Stopwatch createUnstarted() { 23 | return new Stopwatch(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /java/core/src/main/java/io/v6d/core/client/RPCClient.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.client; 16 | 17 | /** Vineyard RPC client. */ 18 | public class RPCClient { 19 | public static void main(String[] args) {} 20 | } 21 | -------------------------------------------------------------------------------- /java/core/src/main/java/io/v6d/core/client/ds/Blob.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.client.ds; 16 | 17 | public class Blob {} 18 | -------------------------------------------------------------------------------- /java/core/src/main/java/io/v6d/core/client/ds/ObjectBuilder.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.client.ds; 16 | 17 | public interface ObjectBuilder extends ObjectBase {} 18 | -------------------------------------------------------------------------------- /java/core/src/test/java/io/v6d/core/client/ds/ObjectFactoryTest.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.client.ds; 16 | 17 | import io.v6d.core.common.util.VineyardException; 18 | import org.junit.Test; 19 | 20 | /** Unit test for Object factory. */ 21 | public class ObjectFactoryTest { 22 | @Test 23 | public void connect() throws VineyardException {} 24 | } 25 | -------------------------------------------------------------------------------- /java/core/src/test/java/io/v6d/core/client/ds/ObjectMetaTest.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.client.ds; 16 | 17 | import io.v6d.core.common.util.VineyardException; 18 | import org.junit.Test; 19 | 20 | /** Unit test for Object metadata. */ 21 | public class ObjectMetaTest { 22 | @Test 23 | public void connect() throws VineyardException {} 24 | } 25 | -------------------------------------------------------------------------------- /java/core/src/test/java/io/v6d/core/common/util/ProtocolTest.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.core.common.util; 16 | 17 | /** Unit test for vineyard IPC protocol. */ 18 | public class ProtocolTest {} 19 | -------------------------------------------------------------------------------- /java/core/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{16} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /java/hive/conf/hive-log4j2.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # vineyard logger 18 | logger.Vineyard.name = io.v6d 19 | logger.Vineyard.level = DEBUG 20 | logger.Vineyard.appenderRefs = console,DRFA 21 | -------------------------------------------------------------------------------- /java/hive/docker/.env: -------------------------------------------------------------------------------- 1 | HADOOP_YARN_RM_PORT=8088 2 | HADOOP_YARN_NM_PORT=8042 3 | HADOOP_YARN_PROXYSERVER_PORT=9111 4 | HADOOP_MR_HISTORYSERVER_PORT=19888 5 | HIVE_METASTORE_PORT=9083 6 | HIVE_HIVESERVER2_PORT=10000 7 | HIVE_VERSION=2.3.9 8 | -------------------------------------------------------------------------------- /java/hive/docker/dependency/images/hadoop-config/yarn-hosts-exclude: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/java/hive/docker/dependency/images/hadoop-config/yarn-hosts-exclude -------------------------------------------------------------------------------- /java/hive/docker/dependency/images/hadoop-config/yarn-hosts-include: -------------------------------------------------------------------------------- 1 | hadoop-yarn-nm-0 2 | hadoop-yarn-nm-1 3 | hadoop-yarn-nm-2 4 | -------------------------------------------------------------------------------- /java/hive/docker/dependency/images/mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/java/hive/docker/dependency/images/mysql-connector-java-5.1.49/mysql-connector-java-5.1.49-bin.jar -------------------------------------------------------------------------------- /java/hive/docker/dependency/images/spark-config-distributed/spark-defaults.conf: -------------------------------------------------------------------------------- 1 | spark.yarn.stagingDir=file:///tmp/ 2 | -------------------------------------------------------------------------------- /java/hive/docker/dependency/images/tez-config/tez-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | tez.lib.uris 4 | file:///tez.tar.gz 5 | 6 | -------------------------------------------------------------------------------- /java/hive/docker/dependency/mysql/mysql-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | db: 4 | image: registry.cn-hangzhou.aliyuncs.com/bigdata_cloudnative/mysql:5.7 #mysql版本 5 | container_name: mysql 6 | hostname: mysql 7 | volumes: 8 | - ./data/db:/var/lib/mysql 9 | - ./conf/my.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf 10 | restart: always 11 | ports: 12 | - 13306:3306 13 | networks: 14 | - hadoop-network 15 | environment: 16 | MYSQL_ROOT_PASSWORD: 123456 # for test 17 | secure_file_priv: 18 | healthcheck: 19 | test: ["CMD-SHELL", "curl -I localhost:3306 || exit 1"] 20 | interval: 10s 21 | timeout: 5s 22 | retries: 3 23 | 24 | # 连接外部网络 25 | networks: 26 | hadoop-network: 27 | external: true 28 | -------------------------------------------------------------------------------- /java/hive/src/main/resources/META-INF/services/org.apache.hadoop.hive.ql.io.StorageFormatDescriptor: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Register vineyard storage format descriptor to hive. 16 | 17 | io.v6d.hive.ql.io.VineyardStorageFormatDescriptor 18 | -------------------------------------------------------------------------------- /java/hive/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{16} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /java/hive/test/expected/test_all_primitive_types.q.out: -------------------------------------------------------------------------------- 1 | 1,1,42,1,2.0,1.0,hello world1!,hello world2!,hello world3! ,aGVsbG8gd29ybGQ0IQ==,2023-12-31,true,2023-12-31 23:59:59,1235.00 2 | -------------------------------------------------------------------------------- /java/hive/test/expected/test_hive_dynamic_partition.q.out: -------------------------------------------------------------------------------- 1 | 1,2,1,2018 2 | 2,3,1,2017 3 | 3,4,1,2018 4 | -------------------------------------------------------------------------------- /java/hive/test/expected/test_hive_static_partition.q.out: -------------------------------------------------------------------------------- 1 | 1,2,114514 2 | 1,2,114514 3 | 3,4,666 4 | 3,4,666 5 | 999,2,666 6 | 999,2,666 7 | 999,2,666 8 | 999,2,666 9 | 999,2,666 10 | 999,2,666 11 | -------------------------------------------------------------------------------- /java/hive/test/expected/test_insert.q.out: -------------------------------------------------------------------------------- 1 | a,1 2 | b,2 3 | c,3 4 | -------------------------------------------------------------------------------- /java/hive/test/expected/test_nested_types.q.out: -------------------------------------------------------------------------------- 1 | 421hello2world! 2 | -------------------------------------------------------------------------------- /java/hive/test/query/test_hive_dynamic_partition.q: -------------------------------------------------------------------------------- 1 | drop table if exists hive_dynamic_partition_data; 2 | create table hive_dynamic_partition_data( 3 | src_id int, 4 | dst_id int, 5 | year int) 6 | stored as TEXTFILE 7 | location "file:///tmp/hive_dynamic_partition_data"; 8 | insert into table hive_dynamic_partition_data values (1, 2, 2018),(3, 4, 2018),(2, 3, 2017); 9 | 10 | drop table if exists hive_dynamic_partition_test; 11 | create table hive_dynamic_partition_test 12 | ( 13 | src_id int, 14 | dst_id int 15 | )partitioned by(mounth int, year int); 16 | insert into table hive_dynamic_partition_test partition(mounth=1, year) select src_id,dst_id,year from hive_dynamic_partition_data; 17 | 18 | insert overwrite directory 'file:///tmp/out/test_hive_dynamic_partition/' 19 | ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 20 | select * from hive_dynamic_partition_test 21 | order by src_id asc; 22 | 23 | drop table hive_dynamic_partition_test; 24 | drop table hive_dynamic_partition_data; -------------------------------------------------------------------------------- /java/hive/test/query/test_insert.q: -------------------------------------------------------------------------------- 1 | drop table if exists hive_example; 2 | create table hive_example(field_1 string,field_2 int); 3 | 4 | insert into hive_example values('a', 1), ('b', 2), ('c', 3); 5 | 6 | insert overwrite directory 'file:///tmp/out/test_insert/' 7 | row format delimited fields terminated by ',' 8 | select * from hive_example 9 | order by field_1 asc; 10 | 11 | drop table hive_example; 12 | -------------------------------------------------------------------------------- /java/hive/test/query/test_nested_types.q: -------------------------------------------------------------------------------- 1 | drop table if exists nested_table; 2 | create table nested_table ( 3 | field_1 map>> 6 | ); 7 | 8 | insert into nested_table select 9 | map( 10 | 42, 11 | array(named_struct('field_1', 1, 12 | 'field_2', 'hello'), 13 | named_struct('field_1', 2, 14 | 'field_2', 'world!'))); 15 | 16 | insert overwrite directory 'file:///tmp/out/test_nested_types/' 17 | ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' 18 | select * from nested_table; 19 | drop table nested_table; -------------------------------------------------------------------------------- /java/modules/basic/src/main/java/io/v6d/modules/basic/array/Array.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.modules.basic.array; 16 | 17 | public class Array {} 18 | -------------------------------------------------------------------------------- /java/modules/basic/src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | %d{HH:mm:ss.SSS} [%thread] %-5level %F:%L - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /java/modules/graph/src/main/java/io/v6d/contrib/graph/Graph.java: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | package io.v6d.contrib.graph; 16 | 17 | /** Hello world! */ 18 | public class Graph { 19 | public static void main(String[] args) {} 20 | } 21 | -------------------------------------------------------------------------------- /k8s/.dockerignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /vendor/ -------------------------------------------------------------------------------- /k8s/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | vineyardctl 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Kubernetes Generated files - skip generated files, except for vendored files 18 | 19 | !vendor/**/zz_generated.* 20 | 21 | # editor and IDE paraphernalia 22 | .idea 23 | *.swp 24 | *.swo 25 | *~ 26 | 27 | # vendor, used for code-generate only 28 | /vendor/ 29 | -------------------------------------------------------------------------------- /k8s/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.19 as builder 3 | 4 | ENV GO111MODULE=on 5 | ENV CGO_ENABLED=0 6 | ENV GOOS=linux 7 | 8 | WORKDIR /workspace 9 | 10 | # Copy the vineyard Go SDK 11 | COPY go/ go/ 12 | 13 | # Copy the Go source for vineyardctl 14 | COPY k8s/ k8s/ 15 | 16 | # Update the working directory 17 | WORKDIR /workspace/k8s 18 | 19 | # Build 20 | RUN go mod download 21 | RUN go build -a -o vineyardctl cmd/main.go && \ 22 | strip vineyardctl || true 23 | 24 | 25 | # Use distroless as minimal base image to package the manager binary 26 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 27 | FROM gcr.io/distroless/base:debug 28 | WORKDIR / 29 | 30 | COPY k8s/config/scheduler/config.yaml /etc/kubernetes/scheduler.yaml 31 | 32 | COPY --from=builder /workspace/k8s/vineyardctl /vineyardctl 33 | -------------------------------------------------------------------------------- /k8s/cmd/commands/util/usage/json.go: -------------------------------------------------------------------------------- 1 | package usage 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | func UsageJson(cmd *cobra.Command) error { 10 | usage, err := CobraUsage(cmd) 11 | if err != nil { 12 | return err 13 | } 14 | // note that `goccy/go-json` won't here (hung up) for `MarshalIndent`, 15 | usageJson, err := json.MarshalIndent(usage, "", " ") 16 | if err != nil { 17 | return err 18 | } 19 | cmd.Printf("%+v", string(usageJson)) 20 | return nil 21 | } 22 | -------------------------------------------------------------------------------- /k8s/config.yaml: -------------------------------------------------------------------------------- 1 | # The file is used to generate the CRD reference documentation. 2 | # Copy from https://github.com/elastic/crd-ref-docs/blob/master/config.yaml 3 | 4 | processor: 5 | ignoreFields: 6 | - "status$" 7 | - "TypeMeta$" 8 | 9 | render: 10 | kubernetesVersion: 1.25 11 | knownTypes: 12 | - name: SecretObjectReference 13 | package: sigs.k8s.io/gateway-api/apis/v1beta1 14 | link: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.SecretObjectReference 15 | -------------------------------------------------------------------------------- /k8s/config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /k8s/config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /k8s/config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | group: apiextensions.k8s.io 8 | path: spec/conversion/webhookClientConfig/service/name 9 | 10 | namespace: 11 | - kind: CustomResourceDefinition 12 | group: apiextensions.k8s.io 13 | path: spec/conversion/webhookClientConfig/service/namespace 14 | create: false 15 | 16 | varReference: 17 | - path: metadata/annotations 18 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_backups.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: backups.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_csidrivers.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: csidrivers.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_globalobjects.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: globalobjects.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_localobjects.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: localobjects.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_operations.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: operations.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_recovers.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: recovers.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_sidecars.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: sidecars.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/cainjection_in_vineyardds.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: vineyardds.k8s.v6d.io 8 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_backups.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: backups.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_csidrivers.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: csidrivers.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_globalobjects.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables conversion webhook for CRD 2 | # CRD conversion requires k8s 1.13 or later. 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | name: globalobjects.k8s.v6d.io 7 | spec: 8 | conversion: 9 | strategy: Webhook 10 | webhook: 11 | clientConfig: 12 | service: 13 | namespace: system 14 | name: webhook-service 15 | path: /convert 16 | conversionReviewVersions: 17 | - v1 18 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_localobjects.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: localobjects.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_operations.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: operations.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_recovers.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: recovers.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_sidecars.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: sidecars.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/crd/patches/webhook_in_vineyardds.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: vineyardds.k8s.v6d.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | conversionReviewVersions: 16 | - v1 17 | -------------------------------------------------------------------------------- /k8s/config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | -------------------------------------------------------------------------------- /k8s/config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | name: mutating-webhook-configuration 7 | annotations: 8 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: validating-webhook-configuration 14 | annotations: 15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 16 | -------------------------------------------------------------------------------- /k8s/config/embed.go: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | // Package config contains all template related functions 17 | package config 18 | 19 | import ( 20 | "embed" 21 | ) 22 | 23 | //go:embed * 24 | var Manifests embed.FS 25 | -------------------------------------------------------------------------------- /k8s/config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | apiVersion: kustomize.config.k8s.io/v1beta1 4 | kind: Kustomization 5 | images: 6 | - name: controller 7 | newName: vineyardcloudnative/vineyard-operator 8 | newTag: latest 9 | -------------------------------------------------------------------------------- /k8s/config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /k8s/config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | selector: 15 | matchLabels: 16 | control-plane: controller-manager 17 | -------------------------------------------------------------------------------- /k8s/config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: ["/metrics"] 7 | verbs: ["get"] 8 | -------------------------------------------------------------------------------- /k8s/config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: ["authentication.k8s.io"] 7 | resources: 8 | - tokenreviews 9 | verbs: ["create"] 10 | - apiGroups: ["authorization.k8s.io"] 11 | resources: 12 | - subjectaccessreviews 13 | verbs: ["create"] 14 | -------------------------------------------------------------------------------- /k8s/config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /k8s/config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /k8s/config/rbac/globalobject_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit globalobjects. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: globalobject-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - globalobjects 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - globalobjects/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/globalobject_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view globalobjects. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: globalobject-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - globalobjects 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - globalobjects/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_backup_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit backups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: backup-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - backups 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - backups/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_backup_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view backups. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: backup-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - backups 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - backups/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_csidriver_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit csidrivers. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: csidriver-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - csidrivers 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - csidrivers/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_csidriver_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view csidrivers. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: csidriver-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - csidrivers 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - csidrivers/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_operation_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit operations. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: operation-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - operations 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - operations/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_operation_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view operations. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: operation-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - operations 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - operations/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_recover_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit recovers. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: recover-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - recovers 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - recovers/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_recover_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view recovers. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: recover-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - recovers 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - recovers/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_sidecar_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit sidecars. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: sidecar-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - sidecars 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - sidecars/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/k8s_sidecar_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view sidecars. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: sidecar-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - sidecars 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - sidecars/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - role.yaml 3 | - role_binding.yaml 4 | - leader_election_role.yaml 5 | - leader_election_role_binding.yaml 6 | # Comment the following 4 lines if you want to disable 7 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 8 | # which protects your /metrics endpoint. 9 | - auth_proxy_service.yaml 10 | - auth_proxy_role.yaml 11 | - auth_proxy_role_binding.yaml 12 | - auth_proxy_client_clusterrole.yaml 13 | 14 | # service account for vineyard operator. 15 | - manager_account.yaml 16 | -------------------------------------------------------------------------------- /k8s/config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - "" 21 | resources: 22 | - configmaps/status 23 | verbs: 24 | - get 25 | - update 26 | - patch 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - events 31 | verbs: 32 | - create 33 | - patch 34 | -------------------------------------------------------------------------------- /k8s/config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /k8s/config/rbac/localobject_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit localobjects. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: localobject-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - localobjects 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - localobjects/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/localobject_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view localobjects. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: localobject-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - localobjects 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - localobjects/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: default 12 | namespace: system 13 | -------------------------------------------------------------------------------- /k8s/config/rbac/vineyardd_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit vineyardds. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: vineyardd-editor-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - vineyardds 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - k8s.v6d.io 21 | resources: 22 | - vineyardds/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /k8s/config/rbac/vineyardd_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view vineyardds. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: vineyardd-viewer-role 6 | rules: 7 | - apiGroups: 8 | - k8s.v6d.io 9 | resources: 10 | - vineyardds 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - k8s.v6d.io 17 | resources: 18 | - vineyardds/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_backup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Backup 3 | metadata: 4 | name: backup-sample 5 | spec: 6 | # TODO(user): Add fields here 7 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_csidriver.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: CSIDriver 3 | metadata: 4 | name: csidriver-sample 5 | spec: 6 | clusters: 7 | - namespace: vineyard-system 8 | name: vineyardd-sample 9 | - namespace: default 10 | name: vineyardd-sample -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_globalobject.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: GlobalObject 3 | metadata: 4 | name: globalobject-sample 5 | spec: 6 | # Add fields here 7 | id: o123 8 | signature: s111 9 | typename: vineyard::GlobalTensor 10 | members: 11 | - a1 12 | - a2 13 | - a3 14 | metadata: "yyy" 15 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_localobject.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: LocalObject 3 | metadata: 4 | name: localobject-sample 5 | labels: 6 | k8s.v6d.io/signature: xxxx 7 | spec: 8 | # Add fields here 9 | id: o123 10 | signature: s111 11 | typename: vineyard::Array 12 | instance_id: 10 13 | hostname: h1 14 | metadata: "xxx" 15 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_operation.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Operation 3 | metadata: 4 | name: operation-sample 5 | spec: 6 | name: assembly 7 | type: local 8 | require: job1 9 | target: job2 10 | timeoutSeconds: 300 11 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_recover.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Recover 3 | metadata: 4 | name: recover-sample 5 | spec: 6 | # TODO(user): Add fields here 7 | -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_sidecar.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Sidecar 3 | metadata: 4 | name: sidecar-sample1 5 | namespace: vineyard-job 6 | spec: 7 | replicas: 2 8 | selector: app=sidecar-job-deployment 9 | vineyard: 10 | socket: /var/run/vineyard.sock -------------------------------------------------------------------------------- /k8s/config/samples/k8s_v1alpha1_vineyardd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Vineyardd 3 | metadata: 4 | name: vineyardd-sample 5 | # don't use default namespace 6 | namespace: vineyard-system 7 | spec: 8 | replicas: 2 9 | vineyard: 10 | image: vineyardcloudnative/vineyardd:latest 11 | imagePullPolicy: IfNotPresent 12 | # vineyardd's configuration 13 | size: "" 14 | syncCRDs: true 15 | spill: 16 | name: spill-path 17 | path: /var/checkpoint/spill 18 | persistentVolumeSpec: 19 | storageClassName: manual 20 | capacity: 21 | storage: 1Gi 22 | accessModes: 23 | - ReadWriteOnce 24 | hostPath: 25 | path: /var/vineyard/spill 26 | persistentVolumeClaimSpec: 27 | storageClassName: manual 28 | accessModes: 29 | - ReadWriteOnce 30 | resources: 31 | requests: 32 | storage: 256Mi 33 | service: 34 | type: ClusterIP 35 | port: 9600 36 | -------------------------------------------------------------------------------- /k8s/config/samples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | ## Append samples you want in your CSV to this file as resources ## 2 | resources: 3 | # - k8s_v1alpha1_localobject.yaml 4 | # - k8s_v1alpha1_globalobject.yaml 5 | # +kubebuilder:scaffold:manifestskustomizesamples 6 | -------------------------------------------------------------------------------- /k8s/config/scheduler/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kubescheduler.config.k8s.io/v1beta1 2 | kind: KubeSchedulerConfiguration 3 | leaderElection: 4 | leaderElect: false 5 | leaseDuration: 60s 6 | resourceLock: leases 7 | resourceName: vineyard-scheduler 8 | resourceNamespace: vineyard-system 9 | profiles: 10 | - schedulerName: vineyard-scheduler 11 | plugins: 12 | score: 13 | enabled: 14 | - name: Vineyard 15 | disabled: 16 | - name: "*" 17 | postBind: 18 | enabled: 19 | - name: Vineyard 20 | -------------------------------------------------------------------------------- /k8s/config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | 8 | patchesStrategicMerge: 9 | - operation_namespace_selector_patch.yaml 10 | - sidecar_selector_patch.yaml 11 | - scheduling_selector_patch.yaml -------------------------------------------------------------------------------- /k8s/config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /k8s/config/webhook/operation_namespace_selector_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: mutating-webhook-configuration 5 | webhooks: 6 | - clientConfig: 7 | service: 8 | name: webhook-service 9 | namespace: system 10 | path: /mutate-v1-pod 11 | name: mpod.kb.io 12 | namespaceSelector: 13 | matchLabels: 14 | operation-injection: enabled 15 | -------------------------------------------------------------------------------- /k8s/config/webhook/scheduling_selector_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: mutating-webhook-configuration 5 | webhooks: 6 | - clientConfig: 7 | service: 8 | name: webhook-service 9 | namespace: system 10 | path: /mutate-v1-pod-scheduling 11 | name: mpod.scheduling.kb.io 12 | objectSelector: 13 | matchLabels: 14 | scheduling.v6d.io/enabled: "true" 15 | -------------------------------------------------------------------------------- /k8s/config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | protocol: TCP 11 | targetPort: 9443 12 | selector: 13 | control-plane: controller-manager 14 | -------------------------------------------------------------------------------- /k8s/config/webhook/sidecar_selector_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: admissionregistration.k8s.io/v1 2 | kind: MutatingWebhookConfiguration 3 | metadata: 4 | name: mutating-webhook-configuration 5 | webhooks: 6 | - clientConfig: 7 | service: 8 | name: webhook-service 9 | namespace: system 10 | path: /mutate-v1-pod-sidecar 11 | name: mpod.sidecar.kb.io 12 | namespaceSelector: 13 | matchLabels: 14 | sidecar-injection: enabled 15 | objectSelector: 16 | matchLabels: 17 | sidecar.v6d.io/enabled: "true" 18 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-csidriver/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN pip3 install --no-cache-dir pandas requests scikit-learn numpy vineyard 4 | 5 | WORKDIR / 6 | 7 | ARG APP 8 | ENV APP ${APP} 9 | 10 | COPY ${APP} /${APP} 11 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-csidriver/Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY := "ghcr.io/v6d-io/v6d/csidriver-example" 2 | docker-build: 3 | docker build prepare-data/ -f Dockerfile \ 4 | --build-arg APP=prepare-data.py \ 5 | -t $(REGISTRY)/prepare-data 6 | 7 | docker build preprocess/ -f Dockerfile \ 8 | --build-arg APP=preprocess.py \ 9 | -t $(REGISTRY)/preprocess-data 10 | 11 | docker build train/ -f Dockerfile \ 12 | --build-arg APP=train.py \ 13 | -t $(REGISTRY)/train-data 14 | 15 | docker build test/ -f Dockerfile \ 16 | --build-arg APP=test.py \ 17 | -t $(REGISTRY)/test-data 18 | 19 | push-images: 20 | docker push $(REGISTRY)/prepare-data 21 | docker push $(REGISTRY)/preprocess-data 22 | docker push $(REGISTRY)/train-data 23 | docker push $(REGISTRY)/test-data 24 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-csidriver/rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: pipeline-runner 6 | namespace: kubeflow 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRole 10 | metadata: 11 | name: pipeline-runner-role 12 | rules: 13 | - apiGroups: [""] 14 | resources: ["persistentvolumeclaims"] 15 | verbs: ["get", "create", "update", "list", "delete"] 16 | - apiGroups: [""] 17 | resources: ["pods"] 18 | verbs: ["get", "patch", "create", "update", "list", "delete"] 19 | --- 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | kind: ClusterRoleBinding 22 | metadata: 23 | name: pipeline-runner-rolebinding 24 | roleRef: 25 | apiGroup: rbac.authorization.k8s.io 26 | kind: ClusterRole 27 | name: pipeline-runner-role 28 | subjects: 29 | - kind: ServiceAccount 30 | name: pipeline-runner 31 | namespace: kubeflow -------------------------------------------------------------------------------- /k8s/examples/vineyard-kubeflow/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN pip3 install --no-cache-dir pandas requests scikit-learn numpy vineyard 4 | 5 | WORKDIR / 6 | 7 | ARG APP 8 | ENV APP ${APP} 9 | 10 | COPY ${APP} /${APP} 11 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-kubeflow/Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY := "ghcr.io/v6d-io/v6d/kubeflow-example" 2 | docker-build: 3 | docker build prepare-data/ -f Dockerfile \ 4 | --build-arg APP=prepare-data.py \ 5 | -t $(REGISTRY)/prepare-data 6 | 7 | docker build preprocess/ -f Dockerfile \ 8 | --build-arg APP=preprocess.py \ 9 | -t $(REGISTRY)/preprocess-data 10 | 11 | docker build train/ -f Dockerfile \ 12 | --build-arg APP=train.py \ 13 | -t $(REGISTRY)/train-data 14 | 15 | docker build test/ -f Dockerfile \ 16 | --build-arg APP=test.py \ 17 | -t $(REGISTRY)/test-data 18 | 19 | push-images: 20 | docker push $(REGISTRY)/prepare-data 21 | docker push $(REGISTRY)/preprocess-data 22 | docker push $(REGISTRY)/train-data 23 | docker push $(REGISTRY)/test-data 24 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-kubeflow/rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: pipeline-runner 6 | namespace: kubeflow 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRole 10 | metadata: 11 | name: pipeline-runner-role 12 | rules: 13 | - apiGroups: [""] 14 | resources: ["persistentvolumeclaims"] 15 | verbs: ["get", "create", "update", "list", "delete"] 16 | - apiGroups: [""] 17 | resources: ["pods"] 18 | verbs: ["get", "patch", "create", "update", "list", "delete"] 19 | --- 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | kind: ClusterRoleBinding 22 | metadata: 23 | name: pipeline-runner-rolebinding 24 | roleRef: 25 | apiGroup: rbac.authorization.k8s.io 26 | kind: ClusterRole 27 | name: pipeline-runner-role 28 | subjects: 29 | - kind: ServiceAccount 30 | name: pipeline-runner 31 | namespace: kubeflow -------------------------------------------------------------------------------- /k8s/examples/vineyard-mars-pytorch/Dockerfile.mars-with-vineyard: -------------------------------------------------------------------------------- 1 | # TODO: wait for the pr (https://github.com/mars-project/mars/pull/3324) released 2 | FROM marsproject/mars:v0.10.0(use the next version here) 3 | 4 | WORKDIR /srv 5 | 6 | RUN pip3 install vineyard 7 | 8 | ENTRYPOINT [ "/srv/entrypoint.sh" ] 9 | -------------------------------------------------------------------------------- /k8s/examples/vineyard-mars-pytorch/prepare-data/resources/service-account.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | name: prepare-data 20 | namespace: vineyard-job 21 | labels: 22 | app.kubernetes.io/instance: prepare-data -------------------------------------------------------------------------------- /k8s/examples/vineyard-mars-pytorch/process-data/resources/service-account.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | name: process-data 20 | namespace: vineyard-job 21 | labels: 22 | app.kubernetes.io/instance: process-data -------------------------------------------------------------------------------- /k8s/examples/vineyard-mars-pytorch/train-data/resources/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | apiVersion: v1 17 | kind: Service 18 | metadata: 19 | name: pytorch-job-service 20 | namespace: vineyard-job 21 | spec: 22 | selector: 23 | app: train-data 24 | ports: 25 | - name: pytorch-job-service 26 | protocol: TCP 27 | port: 23456 28 | targetPort: 23456 -------------------------------------------------------------------------------- /k8s/hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ -------------------------------------------------------------------------------- /k8s/pkg/templates/csidriver/storageclass.yaml: -------------------------------------------------------------------------------- 1 | {{- $storage := getStorageConfig }} 2 | kind: StorageClass 3 | apiVersion: storage.k8s.io/v1 4 | metadata: 5 | name: {{ $storage.Namespace -}}.{{- $storage.Name -}}.csi 6 | provisioner: csi.vineyard.v6d.io 7 | parameters: 8 | k8s.v6d.io/vineyard/namespace: {{ $storage.Namespace }} 9 | k8s.v6d.io/vineyard/name: {{ $storage.Name }} 10 | volumeBindingMode: {{ $storage.VolumeBindingMode }} 11 | -------------------------------------------------------------------------------- /k8s/test/csidriver/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.19-buster as builder 3 | 4 | ENV GO111MODULE=on 5 | ENV CGO_ENABLED=0 6 | ENV GOOS=linux 7 | 8 | WORKDIR /workspace 9 | 10 | # Copy the vineyard Go SDK 11 | COPY go/ go/ 12 | 13 | # Copy the Go source for vineyardctl 14 | COPY k8s/ k8s/ 15 | 16 | WORKDIR /workspace/k8s 17 | 18 | RUN go mod download 19 | 20 | # Update the working directory 21 | WORKDIR /workspace/k8s/cmd/commands/csi 22 | 23 | USER root 24 | 25 | CMD ["go", "test", "-run", "./..."] 26 | -------------------------------------------------------------------------------- /k8s/test/e2e/etcd-failover/consumer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: Job 3 | metadata: 4 | name: consumer 5 | spec: 6 | parallelism: 1 7 | template: 8 | metadata: 9 | labels: 10 | app: consumer 11 | spec: 12 | restartPolicy: Never 13 | containers: 14 | - name: consumer 15 | image: localhost:5001/vineyard-python-dev:latest 16 | imagePullPolicy: IfNotPresent 17 | command: 18 | - bash 19 | - -c 20 | - | 21 | cat << EOF >> consumer.py 22 | import vineyard 23 | client = vineyard.connect(host="vineyardd-svc.default.svc.cluster.local",port=9600) 24 | obj_id = client.get_name("test_data") 25 | print(obj_id) 26 | client.close() 27 | EOF 28 | python consumer.py; -------------------------------------------------------------------------------- /k8s/test/e2e/failover-demo/backup.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Backup 3 | metadata: 4 | name: backup-sample 5 | namespace: vineyard-system 6 | spec: 7 | vineyarddName: vineyardd-sample 8 | vineyarddNamespace: vineyard-system 9 | backupPath: /var/vineyard/dump 10 | persistentVolumeSpec: 11 | storageClassName: manual 12 | capacity: 13 | storage: 1Gi 14 | accessModes: 15 | - ReadWriteOnce 16 | hostPath: 17 | path: /var/vineyard/dump 18 | persistentVolumeClaimSpec: 19 | storageClassName: manual 20 | accessModes: 21 | - ReadWriteOnce 22 | resources: 23 | requests: 24 | storage: 1Gi -------------------------------------------------------------------------------- /k8s/test/e2e/failover-demo/recover.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: k8s.v6d.io/v1alpha1 2 | kind: Recover 3 | metadata: 4 | name: recover-sample 5 | namespace: vineyard-system 6 | spec: 7 | backupName: backup-sample 8 | backupNamespace: vineyard-system -------------------------------------------------------------------------------- /k8s/test/e2e/kind-with-local-registry.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | containerdConfigPatches: 4 | - |- 5 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5001"] 6 | endpoint = ["http://kind-registry:5000"] 7 | nodes: 8 | - role: control-plane 9 | image: kindest/node:v1.19.3 10 | - role: worker 11 | image: kindest/node:v1.19.3 12 | - role: worker 13 | image: kindest/node:v1.19.3 14 | - role: worker 15 | image: kindest/node:v1.19.3 -------------------------------------------------------------------------------- /k8s/test/e2e/kind.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | kind: Cluster 17 | apiVersion: kind.x-k8s.io/v1alpha4 18 | nodes: 19 | - role: control-plane 20 | image: kindest/node:v1.19.3 21 | - role: worker 22 | image: kindest/node:v1.19.3 23 | - role: worker 24 | image: kindest/node:v1.19.3 25 | - role: worker 26 | image: kindest/node:v1.19.3 27 | -------------------------------------------------------------------------------- /k8s/test/e2e/repartition-demo/Dockerfile.dask-worker-with-vineyard: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/dask/dask:2022.8.1-py3.10 2 | 3 | RUN pip3 install --no-cache-dir vineyard vineyard-dask 4 | -------------------------------------------------------------------------------- /k8s/test/e2e/verify/basic-types-unit.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | (1, 1.2345000505447388, 'xxxxabcd') -------------------------------------------------------------------------------- /k8s/test/e2e/verify/dag-success.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | DAG is successful -------------------------------------------------------------------------------- /k8s/test/e2e/verify/distributed-assembly-values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: sum 18 | value: 360 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/distributed-object-values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: sum 18 | value: 152 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/local-assembly-values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: sum 18 | value: 180 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/nodes.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: AllJobInVineyarddNodes 18 | value: true 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/object-id.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | ObjectID -------------------------------------------------------------------------------- /k8s/test/e2e/verify/partition.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: partition 18 | value: 1 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/serialize.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: result 18 | value: test passed 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/serializedfile.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: data 18 | value: __elements_-0 __elements_-1 __elements_-2 __elements_-3 metadata.json 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/spill.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: result 18 | value: Passed 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/spilledfile.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: data 18 | value: {{ gt .value 0 }} 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/e2e/verify/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2023 Alibaba Group Holding Limited. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | {{- contains . }} 17 | - key: sum 18 | value: 45 19 | {{- end }} -------------------------------------------------------------------------------- /k8s/test/hack/delete-kind-cluster-with-local-registry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -o errexit 3 | 4 | kind_name=kind 5 | 6 | if [ ! -z "$1" ] ; then 7 | kind_name=$1 8 | fi 9 | # delete the local registry container if it's running 10 | reg_name='kind-registry' 11 | reg_port='5001' 12 | if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" == 'true' ]; then 13 | docker stop ${reg_name} 14 | docker rm ${reg_name} 15 | fi 16 | 17 | # delete the kind cluster 18 | kind delete cluster --name=${kind_name} 19 | 20 | -------------------------------------------------------------------------------- /misc/release-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -o pipefail 5 | 6 | base=`pwd` 7 | version=$1 8 | 9 | if [ "$version" = "" ]; then 10 | echo "./prepare-wheels.sh: requires a specific version" 11 | exit 1 12 | fi 13 | 14 | mkdir -p wheels/$version 15 | 16 | # download wheels from github artifacts 17 | pushd wheels/$version 18 | curl -s https://api.github.com/repos/v6d-io/v6d/releases/tags/$version | grep "browser_download_url.*whl" | cut -d : -f 2,3 | tr -d \" | wget -qi - 19 | popd 20 | 21 | twine check ./wheels/$version/*.whl 22 | for wheel in $(ls ./wheels/$version/*.whl); do 23 | twine upload $wheel -u __token__ -p "$PYPI_TOKEN" || true 24 | done 25 | -------------------------------------------------------------------------------- /modules/fuse/adaptors/orc/orc.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include "modules/fuse/adaptors/orc/orc.h" 17 | 18 | #if defined(WITH_ORC) 19 | 20 | #include "parquet/file_writer.h" 21 | 22 | namespace vineyard { 23 | namespace fuse { 24 | 25 | void orc_view(std::shared_ptr& df) {} 26 | 27 | } // namespace fuse 28 | } // namespace vineyard 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /modules/fuse/test/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /modules/graph/fragment/arrow_fragment_modifier.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef MODULES_GRAPH_FRAGMENT_ARROW_FRAGMENT_MODIFIER_H_ 17 | #define MODULES_GRAPH_FRAGMENT_ARROW_FRAGMENT_MODIFIER_H_ 18 | 19 | namespace vineyard { 20 | 21 | // keep for backwards compatibility. 22 | 23 | } // namespace vineyard 24 | 25 | #endif // MODULES_GRAPH_FRAGMENT_ARROW_FRAGMENT_MODIFIER_H_ 26 | -------------------------------------------------------------------------------- /modules/graph/loader/arrow_fragment_loader_int32.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include "graph/loader/arrow_fragment_loader_impl.h" 17 | 18 | namespace vineyard { 19 | 20 | template class ArrowFragmentLoader; 21 | 22 | template class ArrowFragmentLoader; 23 | 24 | } // namespace vineyard 25 | -------------------------------------------------------------------------------- /modules/graph/loader/arrow_fragment_loader_int64.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include "graph/loader/arrow_fragment_loader_impl.h" 17 | 18 | namespace vineyard { 19 | 20 | template class ArrowFragmentLoader; 21 | 22 | template class ArrowFragmentLoader; 23 | 24 | } // namespace vineyard 25 | -------------------------------------------------------------------------------- /modules/graph/loader/arrow_fragment_loader_string.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include "graph/loader/arrow_fragment_loader_impl.h" 17 | 18 | namespace vineyard { 19 | 20 | template class ArrowFragmentLoader; 21 | 22 | template class ArrowFragmentLoader; 23 | 24 | } // namespace vineyard 25 | -------------------------------------------------------------------------------- /modules/graph/loader/gar_fragment_loader_int32.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifdef ENABLE_GAR 17 | 18 | #include "graph/loader/gar_fragment_loader_impl.h" 19 | 20 | namespace vineyard { 21 | 22 | template class GARFragmentLoader; 23 | 24 | template class GARFragmentLoader; 25 | } // namespace vineyard 26 | 27 | #endif // ENABLE_GAR 28 | -------------------------------------------------------------------------------- /modules/graph/loader/gar_fragment_loader_int64.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifdef ENABLE_GAR 17 | 18 | #include "graph/loader/gar_fragment_loader_impl.h" 19 | 20 | namespace vineyard { 21 | 22 | template class GARFragmentLoader; 23 | 24 | template class GARFragmentLoader; 25 | 26 | } // namespace vineyard 27 | 28 | #endif // ENABLE_GAR 29 | -------------------------------------------------------------------------------- /modules/graph/vertex_map/arrow_vertex_map_builder.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef MODULES_GRAPH_VERTEX_MAP_ARROW_VERTEX_MAP_BUILDER_H_ 17 | #define MODULES_GRAPH_VERTEX_MAP_ARROW_VERTEX_MAP_BUILDER_H_ 18 | 19 | namespace vineyard { 20 | 21 | // keep for backwards compatibility. 22 | 23 | } // namespace vineyard 24 | 25 | #endif // MODULES_GRAPH_VERTEX_MAP_ARROW_VERTEX_MAP_BUILDER_H_ 26 | -------------------------------------------------------------------------------- /modules/graph/writer/arrow_fragment_writer_int32.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifdef ENABLE_GAR 17 | 18 | #include "graph/writer/arrow_fragment_writer_impl.h" 19 | 20 | namespace vineyard { 21 | 22 | template class ArrowFragmentWriter>; 23 | 24 | } // namespace vineyard 25 | 26 | #endif // ENABLE_GA 27 | -------------------------------------------------------------------------------- /modules/graph/writer/arrow_fragment_writer_int64.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifdef ENABLE_GAR 17 | 18 | #include "graph/writer/arrow_fragment_writer_impl.h" 19 | 20 | namespace vineyard { 21 | 22 | template class ArrowFragmentWriter>; 23 | 24 | } // namespace vineyard 25 | 26 | #endif // ENABLE_GA 27 | -------------------------------------------------------------------------------- /modules/graph/writer/arrow_fragment_writer_string.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifdef ENABLE_GAR 17 | 18 | #include "graph/writer/arrow_fragment_writer_impl.h" 19 | 20 | namespace vineyard { 21 | 22 | template class ArrowFragmentWriter>; 23 | 24 | } // namespace vineyard 25 | 26 | #endif // ENABLE_GA 27 | -------------------------------------------------------------------------------- /modules/io/.gitignore: -------------------------------------------------------------------------------- 1 | setup.cfg 2 | build/ 3 | dist/ 4 | *.egg-info/ 5 | -------------------------------------------------------------------------------- /modules/llm-cache/tests/k8s-test/Dockerfile.master: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | WORKDIR / 4 | 5 | COPY master.py /master.py 6 | 7 | RUN pip3 install kubernetes 8 | 9 | CMD ["python3", "master.py"] -------------------------------------------------------------------------------- /modules/llm-cache/tests/k8s-test/Dockerfile.worker: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/v6d-io/v6d/vineyard-python-dev:latest_x86_64 as builder 2 | 3 | FROM python:3.10 4 | 5 | WORKDIR / 6 | 7 | COPY worker.py /worker.py 8 | COPY --from=builder /tmp/vineyard_llm-0.22.1-py3-none-any.whl vineyard_llm-0.22.1-py3-none-any.whl 9 | 10 | RUN apt update && \ 11 | apt install fio -y 12 | 13 | RUN pip3 install vineyard /vineyard_llm-0.22.1-py3-none-any.whl && \ 14 | pip3 install networkx==3.1 && \ 15 | pip3 install numpy 16 | 17 | CMD ["python3", "worker.py"] 18 | -------------------------------------------------------------------------------- /modules/llm-cache/tests/k8s-test/Makefile: -------------------------------------------------------------------------------- 1 | registry = registry.cn-wulanchabu.aliyuncs.com/vineyard 2 | build-images: 3 | docker build -t ${registry}/fs-llm-master:latest -f ./Dockerfile.master . 4 | docker build -t ${registry}/fs-llm-worker:latest -f ./Dockerfile.worker . 5 | push-images: 6 | docker push ${registry}/fs-llm-master:latest 7 | docker push ${registry}/fs-llm-worker:latest -------------------------------------------------------------------------------- /modules/malloc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # build vineyard-allocator 2 | 3 | file(GLOB_RECURSE BASIC_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}" "*.cc") 4 | 5 | add_library(vineyard_malloc ${BASIC_SRC_FILES} 6 | # requires mimalloc 7 | $) 8 | target_add_debuginfo(vineyard_malloc) 9 | target_link_libraries(vineyard_malloc PUBLIC vineyard_client) 10 | target_link_libraries(vineyard_malloc PRIVATE $) 11 | 12 | if(BUILD_VINEYARD_MALLOC_OVERRIDE) 13 | # TODO(not implemented yet) 14 | target_compile_options(vineyard_malloc PRIVATE -DVINEYARD_MALLOC_OVERRIDE) 15 | endif() 16 | 17 | install_export_vineyard_target(vineyard_malloc) 18 | install_vineyard_headers("${CMAKE_CURRENT_SOURCE_DIR}") 19 | -------------------------------------------------------------------------------- /modules/msgpack/packed_object.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "msgpack.hpp" 22 | 23 | #include "client/client.h" 24 | #include "msgpack/packed_object.h" 25 | 26 | namespace vineyard {} // namespace vineyard 27 | -------------------------------------------------------------------------------- /modules/msgpack/pickle.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "msgpack.hpp" 22 | 23 | #include "client/client.h" 24 | #include "msgpack/pickle.h" 25 | 26 | namespace vineyard {} // namespace vineyard 27 | -------------------------------------------------------------------------------- /modules/msgpack/vbuffer.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include "msgpack/vbuffer.h" 17 | -------------------------------------------------------------------------------- /packages-java/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | *.iml 3 | .classpath 4 | .project 5 | .factorypath 6 | .settings/ 7 | hs_err_pid*.log 8 | 9 | -------------------------------------------------------------------------------- /packages-java/excludes.txt: -------------------------------------------------------------------------------- 1 | nonstd.sv_lite.basic_string_view 2 | std.map.const_reference 3 | std.map.reference 4 | std.set.const_reference 5 | std.set.reference 6 | std.allocator.reference 7 | std.allocator.const_reference 8 | std.unordered_map.const_reference 9 | std.unordered_map.reference 10 | std.vector.const_reference 11 | std.vector.reference 12 | arrow.ArrayBuilder::Finish 13 | -------------------------------------------------------------------------------- /packages-java/src/main/java/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore generated java sources 2 | 3 | *.java 4 | 5 | -------------------------------------------------------------------------------- /packages-java/type-alias-oracles.txt: -------------------------------------------------------------------------------- 1 | std.vector.size_type=std.size_t 2 | -------------------------------------------------------------------------------- /prometheus/Dockerfile.grok-exporter: -------------------------------------------------------------------------------- 1 | # target: vineyardcloudnative/vineyardd-grok-exporter:latest 2 | # 3 | FROM frolvlad/alpine-glibc:alpine-3.12 4 | 5 | RUN apk add --no-cache wget unzip \ 6 | && wget -q https://github.com/fstab/grok_exporter/releases/download/v1.0.0.RC5/grok_exporter-1.0.0.RC5.linux-amd64.zip \ 7 | -O /tmp/grok_exporter-1.0.0.RC5.linux-amd64.zip \ 8 | && unzip /tmp/grok_exporter-1.0.0.RC5.linux-amd64.zip -d /opt \ 9 | && rm /tmp/grok_exporter-1.0.0.RC5.linux-amd64.zip \ 10 | && mkdir -p /var/log/vineyard/ 11 | 12 | COPY grok_exporter.yml /etc/grok_exporter.yml 13 | 14 | WORKDIR /opt/grok_exporter-1.0.0.RC5.linux-amd64/ 15 | ENTRYPOINT ["./grok_exporter"] 16 | 17 | CMD ["-config", "/etc/grok_exporter.yml", "-disable-exporter-metrics"] 18 | 19 | EXPOSE 9144/tcp 20 | 21 | -------------------------------------------------------------------------------- /prometheus/prometheus.rules.yml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: request 3 | rules: 4 | - record: operation:data_request_latency:avg 5 | expr: avg by (operation) (data_request_latency_sum)/(data_request_latency_count) 6 | - alert: EtcdRequestsSlow 7 | expr: etcd_request_duration_microseconds{quantile="0.5"} > 3000 8 | for: 2m 9 | labels: 10 | severity: warning 11 | annotations: 12 | summary: "Etcd requests slow (operation {{ $labels.operation }})" 13 | description: "Etcd requests slowing down, 50th percentile is over 3000 microseconds, \n value = {{ $value }}, \n user is {{ $labels.user }}" -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | evaluation_interval: 5s 4 | external_labels: 5 | monitor: 'vineyardd-monitor' 6 | 7 | rule_files: 8 | - './prometheus.rules.yml' 9 | 10 | scrape_configs: 11 | - job_name: "vineyardd" 12 | static_configs: 13 | - targets: ["localhost:9144"] -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.black] 6 | line-length = 88 7 | exclude = ''' 8 | ( 9 | .*\/\.git\/.* 10 | | .*\/benchmark\/.* 11 | | .*\/build\/.* 12 | | .*\/dist\/.* 13 | | .*\/misc\/.* 14 | | .*\/python\-libs\/.* 15 | | .*\/kedro\-starters\/.* 16 | | .*\/thirdparty\/.* 17 | | .*\/python/vineyard/contrib/kedro/benchmark\/.* 18 | ) 19 | ''' 20 | skip-string-normalization = true 21 | -------------------------------------------------------------------------------- /python/vineyard/.gitignore: -------------------------------------------------------------------------------- 1 | /*.so 2 | /*.dylib 3 | /vineyardd 4 | /vineyardctl 5 | -------------------------------------------------------------------------------- /python/vineyard/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from vineyard.deploy.__main__ import main 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /python/vineyard/bdist/.gitignore: -------------------------------------------------------------------------------- 1 | vineyardctl 2 | vineyardd 3 | -------------------------------------------------------------------------------- /python/vineyard/bdist/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/airflow/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/airflow/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # docker build . -f Dockerfile -t vineyardcloudnative/vineyard-airflow:2.6.3 2 | 3 | FROM apache/airflow:2.6.3-python3.10 4 | 5 | USER airflow 6 | 7 | RUN pip install --no-cache-dir \ 8 | numpy \ 9 | pandas \ 10 | vineyard \ 11 | airflow-provider-vineyard 12 | ENV AIRFLOW__CORE__XCOM_BACKEND=vineyard.contrib.airflow.xcom.VineyardXCom 13 | -------------------------------------------------------------------------------- /python/vineyard/contrib/airflow/example_dags/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/airflow/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/airflow/xcom/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .backend import VineyardXCom # pylint: disable=wrong-import-position 20 | -------------------------------------------------------------------------------- /python/vineyard/contrib/dask/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/dask/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/dask/tests/conftest.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from vineyard.conftest import vineyard_ipc_sockets # pylint: disable=unused-import 20 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .io import VineyardDataSet 20 | from .plugins import VineyardHook 21 | from .plugins import hooks 22 | from .runner import ParallelRunner 23 | from .runner import SequentialRunner 24 | from .runner import ThreadRunner 25 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=python:3.9-slim 2 | FROM $BASE_IMAGE as runtime-environment 3 | 4 | # install project requirements 5 | COPY src/requirements.txt /tmp/requirements.txt 6 | 7 | RUN pip install --no-cache-dir -r /tmp/requirements.txt 8 | RUN pip install --no-cache-dir semver 9 | 10 | # install vineyard dependencies 11 | RUN pip install --no-cache-dir vineyard-kedro 12 | 13 | # add kedro user 14 | ARG KEDRO_UID=999 15 | ARG KEDRO_GID=0 16 | RUN groupadd -f -g ${KEDRO_GID} kedro_group && \ 17 | useradd -m -d /home/kedro_docker -s /bin/bash -g ${KEDRO_GID} -u ${KEDRO_UID} kedro_docker 18 | 19 | WORKDIR /home/kedro_docker 20 | USER kedro_docker 21 | 22 | FROM runtime-environment 23 | 24 | ARG KEDRO_UID=999 25 | ARG KEDRO_GID=0 26 | COPY --chown=${KEDRO_UID}:${KEDRO_GID} . . 27 | 28 | CMD ["kedro", "run"] 29 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/Makefile: -------------------------------------------------------------------------------- 1 | all: docker-build 2 | 3 | docker-build: 4 | docker build . -t mlops-benchmark:latest -f Dockerfile 5 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/conf/aws-s3/credentials.yml: -------------------------------------------------------------------------------- 1 | # Here you can define credentials for different data sets and environment. 2 | # 3 | # 4 | # Example: 5 | # 6 | # dev_s3: 7 | # aws_access_key_id: token 8 | # aws_secret_access_key: key 9 | # 10 | # prod_s3: 11 | # aws_access_key_id: token 12 | # aws_secret_access_key: key 13 | # 14 | # dev_sql: 15 | # username: admin 16 | # password: admin 17 | 18 | benchmark_aws_s3: 19 | client_kwargs: 20 | aws_access_key_id: Your AWS Access Key ID 21 | aws_secret_access_key: Your AWS Secret Access Key 22 | region_name: Your AWS Region Name 23 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/conf/base/catalog.yml: -------------------------------------------------------------------------------- 1 | # Here you can define all your data sets by using simple YAML syntax. 2 | # 3 | # Documentation for this file format can be found in "The Data Catalog" 4 | # Link: https://docs.kedro.org/en/stable/data/data_catalog.html 5 | 6 | house_prices: 7 | type: pandas.CSVDataSet 8 | filepath: data/01_raw/train.csv 9 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/conf/local/catalog.yml: -------------------------------------------------------------------------------- 1 | # Here you can define all your data sets by using simple YAML syntax. 2 | # 3 | # Documentation for this file format can be found in "The Data Catalog" 4 | # Link: https://docs.kedro.org/en/stable/data/data_catalog.html 5 | 6 | house_prices: 7 | type: pandas.CSVDataSet 8 | filepath: data/01_raw/train.csv 9 | 10 | house_prices_clean: 11 | type: pandas.CSVDataSet 12 | filepath: data/tmp/house_prices_clean1.csv 13 | house_prices_drop: 14 | type: pandas.CSVDataSet 15 | filepath: data/tmp/house_prices_drop1.csv 16 | house_prices_no_na: 17 | type: pandas.CSVDataSet 18 | filepath: data/tmp/house_prices_no_na1.csv 19 | house_prices_no_outliers: 20 | type: pandas.CSVDataSet 21 | filepath: data/tmp/house_prices_no_outliers1.csv 22 | y_train: 23 | type: pandas.CSVDataSet 24 | filepath: data/tmp/y_train1.csv 25 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/conf/minio-s3/credentials.yml: -------------------------------------------------------------------------------- 1 | # Here you can define credentials for different data sets and environment. 2 | # 3 | # 4 | # Example: 5 | # 6 | # dev_s3: 7 | # aws_access_key_id: token 8 | # aws_secret_access_key: key 9 | # 10 | # prod_s3: 11 | # aws_access_key_id: token 12 | # aws_secret_access_key: key 13 | # 14 | # dev_sql: 15 | # username: admin 16 | # password: admin 17 | 18 | benchmark_minio_s3: 19 | key: minioadmin 20 | secret: minioadmin 21 | client_kwargs: 22 | endpoint_url : 'http://minio.minio-dev:9000' 23 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/conf/vineyard/catalog.yml: -------------------------------------------------------------------------------- 1 | # Here you can define all your data sets by using simple YAML syntax. 2 | # 3 | # Documentation for this file format can be found in "The Data Catalog" 4 | # Link: https://docs.kedro.org/en/stable/data/data_catalog.html 5 | house_prices: 6 | type: pandas.CSVDataSet 7 | filepath: data/01_raw/train.csv 8 | 9 | house_prices_clean: 10 | ds_name: house_prices_clean 11 | type: vineyard.contrib.kedro.io.dataset.VineyardDataSet 12 | house_prices_drop: 13 | ds_name: house_prices_drop 14 | type: vineyard.contrib.kedro.io.dataset.VineyardDataSet 15 | house_prices_no_na: 16 | ds_name: house_prices_no_na 17 | type: vineyard.contrib.kedro.io.dataset.VineyardDataSet 18 | house_prices_no_outliers: 19 | ds_name: house_prices_no_outliers 20 | type: vineyard.contrib.kedro.io.dataset.VineyardDataSet 21 | y_train: 22 | ds_name: y_train 23 | type: vineyard.contrib.kedro.io.dataset.VineyardDataSet 24 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/data/tmp/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/contrib/kedro/benchmark/mlops/data/tmp/.gitkeep -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. mlops documentation master file, created by sphinx-quickstart. 2 | You can adapt this file completely to your liking, but it should at least 3 | contain the root `toctree` directive. 4 | 5 | Welcome to project mlops's API docs! 6 | ============================================= 7 | 8 | .. toctree:: 9 | :maxdepth: 4 10 | 11 | modules 12 | 13 | 14 | Indices and tables 15 | ================== 16 | 17 | * :ref:`genindex` 18 | * :ref:`modindex` 19 | * :ref:`search` 20 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/notebooks/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/contrib/kedro/benchmark/mlops/notebooks/.gitkeep -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.kedro] 2 | package_name = "mlops" 3 | project_name = "mlops" 4 | kedro_init_version = "0.18.10" 5 | 6 | [tool.isort] 7 | profile = "black" 8 | 9 | [tool.pytest.ini_options] 10 | addopts = """ 11 | --cov-report term-missing \ 12 | --cov src/mlops -ra""" 13 | 14 | [tool.coverage.report] 15 | fail_under = 0 16 | show_missing = true 17 | exclude_lines = ["pragma: no cover", "raise NotImplementedError"] 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=88 3 | extend-ignore=E203 4 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/mlops/__init__.py: -------------------------------------------------------------------------------- 1 | """mlops 2 | """ 3 | 4 | __version__ = "0.1" 5 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/mlops/pipeline_registry.py: -------------------------------------------------------------------------------- 1 | """Project pipelines.""" 2 | from __future__ import annotations 3 | 4 | from kedro.framework.project import find_pipelines 5 | from kedro.pipeline import Pipeline 6 | 7 | 8 | def register_pipelines() -> dict[str, Pipeline]: 9 | """Register the project's pipelines. 10 | 11 | Returns: 12 | A mapping from pipeline names to ``Pipeline`` objects. 13 | """ 14 | pipelines = find_pipelines() 15 | pipelines["__default__"] = sum(pipelines.values()) 16 | return pipelines 17 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/mlops/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/contrib/kedro/benchmark/mlops/src/mlops/pipelines/__init__.py -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/requirements.txt: -------------------------------------------------------------------------------- 1 | black~=22.0 2 | flake8>=3.7.9, <5.0 3 | ipython>=8.10; python_version >= '3.8' 4 | isort~=5.0 5 | jupyter~=1.0 6 | jupyterlab_server>=2.11.1, <2.16.0 7 | jupyterlab~=3.0, <3.6.0 8 | kedro~=0.18.10 9 | kedro-telemetry~=0.2.0 10 | nbstripout~=0.4 11 | pytest-cov~=3.0 12 | pytest-mock>=1.7.1, <2.0 13 | pytest~=7.2 14 | s3fs 15 | numpy 16 | pandas 17 | scikit-learn 18 | kedro-sagemaker==0.3.0 19 | vineyard-kedro -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/contrib/kedro/benchmark/mlops/src/tests/__init__.py -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/benchmark/mlops/src/tests/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/contrib/kedro/benchmark/mlops/src/tests/pipelines/__init__.py -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/io/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .dataset import VineyardDataSet 20 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from . import argo 20 | from . import catalog 21 | from . import cli 22 | from . import docker 23 | from .hook import VineyardHook 24 | from .hook import hooks 25 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/plugins/starter.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from kedro.framework.cli.starters import KedroStarterSpec 20 | 21 | starters = [] 22 | -------------------------------------------------------------------------------- /python/vineyard/contrib/kedro/runner/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .parallel_runner import ParallelRunner 20 | from .sequential_runner import SequentialRunner 21 | from .thread_runner import ThreadRunner 22 | -------------------------------------------------------------------------------- /python/vineyard/contrib/ml/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/ml/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/pyspark/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/pyspark/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/contrib/pyspark/tests/conftest.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from vineyard.conftest import vineyard_ipc_socket # pylint: disable=unused-import 20 | from vineyard.conftest import vineyard_spark_jar_path # pylint: disable=unused-import 21 | -------------------------------------------------------------------------------- /python/vineyard/contrib/ray/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/core/codegen/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .main import main 20 | -------------------------------------------------------------------------------- /python/vineyard/core/codegen/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .main import main 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /python/vineyard/core/codegen/python.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | 20 | def codegen(): 21 | raise NotImplementedError 22 | -------------------------------------------------------------------------------- /python/vineyard/core/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/data/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/data/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/deploy/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/drivers/io/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from . import stream 20 | -------------------------------------------------------------------------------- /python/vineyard/drivers/io/adaptors/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/drivers/io/kube_ssh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | POD_NAME=$1 4 | shift 5 | 6 | KUBE_NS="" 7 | KUBE_POD_NAME=${POD_NAME} 8 | KUBE_CONTAINER_NAME="" 9 | if [[ "$POD_NAME" =~ ":" ]]; then 10 | KUBE_NS="-n "${POD_NAME%%:*} 11 | KUBE_POD_CONTAINER_NAME=${POD_NAME#*:} 12 | if [[ "$KUBE_POD_CONTAINER_NAME" =~ ":" ]]; then 13 | KUBE_POD_NAME=${KUBE_POD_CONTAINER_NAME%%:*} 14 | KUBE_CONTAINER_NAME="-c "${KUBE_POD_CONTAINER_NAME#*:} 15 | else 16 | KUBE_POD_NAME=${KUBE_POD_CONTAINER_NAME} 17 | fi 18 | fi 19 | 20 | kubectl ${KUBE_NS} exec ${KUBE_POD_NAME} ${KUBE_CONTAINER_NAME} -- /bin/bash -lc " 21 | cat /etc/hosts > /dev/null || true && \ 22 | source /etc/profile || true && \ 23 | source /etc/bash.bashrc || true && \ 24 | source ~/.profile || true && \ 25 | source ~/.bashrc || true && \ 26 | shopt -s huponexit 2>/dev/null || true && \ 27 | $*" 28 | -------------------------------------------------------------------------------- /python/vineyard/drivers/io/ssh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOST_NAME=$1 4 | shift 5 | 6 | if [ "${HOST_NAME}" = "localhost" ] || [ "${HOST_NAME}" = "$(hostname)" ] || [ "${HOST_NAME}" = "$(hostname -i)" ]; 7 | then 8 | shopt -s huponexit 2>/dev/null || true; 9 | bash -c "$*" 10 | else 11 | ssh ${HOST_NAME} -- "/bin/bash -lc '\ 12 | cat /etc/hosts > /dev/null || true && \ 13 | source /etc/profile || true && \ 14 | source /etc/bash.bashrc || true && \ 15 | source ~/.profile || true && \ 16 | source ~/.bashrc || true && \ 17 | source ~/.bash_profile || true && \ 18 | shopt -s huponexit 2>/dev/null || true && \ 19 | $*'" 20 | fi 21 | -------------------------------------------------------------------------------- /python/vineyard/drivers/io/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/io/core.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/io/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /python/vineyard/io/scripts/stream_bash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -xe 2 | 3 | $@ 4 | -------------------------------------------------------------------------------- /python/vineyard/io/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/launcher/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .launcher import Launcher 20 | from .script import ScriptLauncher 21 | -------------------------------------------------------------------------------- /python/vineyard/llm/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from vineyard.llm.cache import FileCacheConfig 20 | from vineyard.llm.cache import KVCache 21 | from vineyard.llm.cache import KVTensor 22 | from vineyard.llm.cache import VineyardCacheConfig 23 | -------------------------------------------------------------------------------- /python/vineyard/llm/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v6d-io/v6d/f57ca13939827f0e1078476005e15608a16c0971/python/vineyard/py.typed -------------------------------------------------------------------------------- /python/vineyard/shared_memory/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | from .shared_memory import ShareableList 20 | from .shared_memory import SharedMemory 21 | -------------------------------------------------------------------------------- /python/vineyard/shared_memory/tests/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/systemd/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | -------------------------------------------------------------------------------- /python/vineyard/version.py.in: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020-2023 Alibaba Group Holding Limited. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | __version__ = '@VINEYARD_VERSION@' 20 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | black>=22.3.0 2 | breathe 3 | docutils==0.20.1 4 | flake8>=4.0.1 5 | flake8-comprehensions 6 | flake8-logging-format 7 | furo # sphinx theme 8 | isort>=5.10.1 9 | jinja2>=3.1.0 10 | libclang 11 | linkify-it-py 12 | myst-parser>=0.13.0 13 | nbsphinx 14 | pygments>=2.4.1 15 | pytest<8.0.0 16 | pytest-benchmark 17 | pytest-cases 18 | pytest-datafiles 19 | pytest-timeout 20 | sphinx>=7.1.2 21 | sphinx-copybutton 22 | sphinx-panels 23 | sphinxemoji 24 | sphinxext-opengraph 25 | google-re2==1.1 26 | -------------------------------------------------------------------------------- /requirements-extra.txt: -------------------------------------------------------------------------------- 1 | scipy 2 | polars 3 | -------------------------------------------------------------------------------- /requirements-kubernetes.txt: -------------------------------------------------------------------------------- 1 | kubernetes -------------------------------------------------------------------------------- /requirements-setup.txt: -------------------------------------------------------------------------------- 1 | libclang 2 | parsec 3 | setuptools 4 | wheel 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argcomplete 2 | click 3 | etcd-distro 4 | lazy-import 5 | makefun 6 | numpy>=1.18.5 7 | pandas<1.0.0; python_version<"3.6" 8 | pandas<1.2.0; python_version<"3.7" 9 | pandas>=1.0.0; python_version>="3.7" 10 | pickle5; python_version<="3.7" 11 | psutil 12 | pyarrow 13 | pyyaml 14 | setuptools 15 | shared-memory38; python_version<="3.7" 16 | sortedcontainers 17 | treelib 18 | -------------------------------------------------------------------------------- /rust/.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | target 3 | -------------------------------------------------------------------------------- /rust/vineyard-datafusion/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "vineyard-datafusion" 3 | version = { workspace = true } 4 | description = "Vineyard Rust SDK: arrow datafusion integration for DataFrame" 5 | homepage = { workspace = true } 6 | repository = { workspace = true } 7 | license = { workspace = true } 8 | keywords = { workspace = true } 9 | include = { workspace = true } 10 | edition = { workspace = true } 11 | readme = { workspace = true } 12 | 13 | [features] 14 | default = ["nightly"] 15 | nightly = [] 16 | 17 | [lib] 18 | name = "vineyard_datafusion" 19 | path = "src/lib.rs" 20 | 21 | [dependencies] 22 | arrow-array = { workspace = true } 23 | arrow-schema = { workspace = true } 24 | datafusion = { workspace = true } 25 | itertools = { workspace = true } 26 | serde_json = { workspace = true } 27 | vineyard = { workspace = true } 28 | 29 | [dev-dependencies] 30 | spectral = { workspace = true } 31 | futures = { workspace = true } 32 | -------------------------------------------------------------------------------- /rust/vineyard-datafusion/src/ds/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod dataframe; 16 | pub mod dataframe_test; 17 | -------------------------------------------------------------------------------- /rust/vineyard-datafusion/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #![allow(clippy::needless_return)] 16 | 17 | pub mod ds; 18 | -------------------------------------------------------------------------------- /rust/vineyard-integration-testing/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "vineyard-integration-testing" 3 | version = { workspace = true } 4 | description = "Vineyard Rust SDK: integration testing" 5 | homepage = { workspace = true } 6 | repository = { workspace = true } 7 | license = { workspace = true } 8 | keywords = { workspace = true } 9 | include = { workspace = true } 10 | edition = { workspace = true } 11 | readme = { workspace = true } 12 | 13 | [features] 14 | default = ["nightly"] 15 | nightly = [] 16 | 17 | [lib] 18 | name = "vineyard_integration_testing" 19 | path = "src/lib.rs" 20 | 21 | [dependencies] 22 | 23 | [dev-dependencies] 24 | arrow-array = { workspace = true } 25 | inline-python = { workspace = true } 26 | polars-core = { workspace = true } 27 | spectral = { workspace = true } 28 | vineyard = { workspace = true } 29 | vineyard-polars = { workspace = true } 30 | -------------------------------------------------------------------------------- /rust/vineyard-integration-testing/src/ds/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod arrow_test; 16 | pub mod numpy_test; 17 | pub mod pandas_test; 18 | pub mod polars_test; 19 | -------------------------------------------------------------------------------- /rust/vineyard-integration-testing/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #![allow(clippy::needless_return)] 16 | 17 | pub mod ds; 18 | -------------------------------------------------------------------------------- /rust/vineyard-polars/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "vineyard-polars" 3 | version = { workspace = true } 4 | description = "Vineyard Rust SDK: polars integration for DataFrame" 5 | homepage = { workspace = true } 6 | repository = { workspace = true } 7 | license = { workspace = true } 8 | keywords = { workspace = true } 9 | include = { workspace = true } 10 | edition = { workspace = true } 11 | readme = { workspace = true } 12 | 13 | [features] 14 | default = ["nightly"] 15 | nightly = [] 16 | 17 | [lib] 18 | name = "vineyard_polars" 19 | path = "src/lib.rs" 20 | 21 | [dependencies] 22 | arrow-array = { workspace = true } 23 | arrow-schema = { workspace = true } 24 | arrow2 = { workspace = true } 25 | itertools = { workspace = true } 26 | polars-arrow = { workspace = true } 27 | polars-core = { workspace = true } 28 | serde_json = { workspace = true } 29 | vineyard = { workspace = true } 30 | 31 | [dev-dependencies] 32 | spectral = { workspace = true } 33 | -------------------------------------------------------------------------------- /rust/vineyard-polars/src/ds/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod dataframe; 16 | pub mod dataframe_test; 17 | -------------------------------------------------------------------------------- /rust/vineyard-polars/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #![allow(clippy::needless_return)] 16 | 17 | pub mod ds; 18 | -------------------------------------------------------------------------------- /rust/vineyard/src/client/ds/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod blob; 16 | pub mod blob_test; 17 | pub mod object; 18 | pub mod object_factory; 19 | pub mod object_meta; 20 | 21 | pub use blob::*; 22 | pub use object::*; 23 | pub use object_factory::*; 24 | pub use object_meta::*; 25 | -------------------------------------------------------------------------------- /rust/vineyard/src/client/rpc_client_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(test)] 16 | mod tests { 17 | use super::super::*; 18 | 19 | #[test] 20 | fn test_rpc_connect() -> Result<()> { 21 | let mut client = RPCClient::default()?; 22 | assert!(client.connected()); 23 | 24 | return Ok(()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /rust/vineyard/src/common/memory/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod fling; 16 | pub mod payload; 17 | 18 | pub use fling::{recv_fd, send_fd}; 19 | pub use payload::Payload; 20 | -------------------------------------------------------------------------------- /rust/vineyard/src/common/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod memory; 16 | pub mod util; 17 | -------------------------------------------------------------------------------- /rust/vineyard/src/common/util/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod arrow; 16 | pub mod json; 17 | pub mod protocol; 18 | pub mod status; 19 | pub mod typename; 20 | pub mod uuid; 21 | -------------------------------------------------------------------------------- /rust/vineyard/src/ds/dataframe_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[allow(unused_imports)] 16 | use std::any::Any; 17 | -------------------------------------------------------------------------------- /rust/vineyard/src/ds/hashmap.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[allow(unused_imports)] 16 | use std::any::Any; 17 | -------------------------------------------------------------------------------- /rust/vineyard/src/ds/hashmap_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[allow(unused_imports)] 16 | use std::any::Any; 17 | -------------------------------------------------------------------------------- /rust/vineyard/src/ds/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod array; 16 | pub mod array_test; 17 | pub mod arrow; 18 | pub mod arrow_test; 19 | pub mod arrow_utils; 20 | pub mod dataframe; 21 | pub mod dataframe_test; 22 | pub mod hashmap; 23 | pub mod hashmap_test; 24 | pub mod tensor; 25 | pub mod tensor_test; 26 | -------------------------------------------------------------------------------- /rust/vineyard/src/ds/tensor_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2020-2023 Alibaba Group Holding Limited. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[allow(unused_imports)] 16 | use std::any::Any; 17 | -------------------------------------------------------------------------------- /src/client/ds/core_types.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef SRC_CLIENT_DS_CORE_TYPES_H_ 17 | #define SRC_CLIENT_DS_CORE_TYPES_H_ 18 | 19 | #include "client/ds/core_types.h" 20 | 21 | namespace vineyard {} // namespace vineyard 22 | 23 | #endif // SRC_CLIENT_DS_CORE_TYPES_H_ 24 | -------------------------------------------------------------------------------- /src/common/rdma/libfabric.map: -------------------------------------------------------------------------------- 1 | FABRIC_1.0 { 2 | global: 3 | fi_getinfo; 4 | fi_freeinfo; 5 | fi_dupinfo; 6 | fi_fabric; 7 | fi_version; 8 | fi_strerror; 9 | fi_tostr; 10 | fi_log_enabled; 11 | fi_log; 12 | fi_param_define; 13 | fi_param_get; 14 | fi_getparams; 15 | fi_freeparams; 16 | local: fi_*; 17 | }; 18 | 19 | FABRIC_1.1 { 20 | global: 21 | fi_getinfo; 22 | fi_freeinfo; 23 | fi_dupinfo; 24 | fi_fabric; 25 | } FABRIC_1.0; 26 | 27 | FABRIC_1.2 { 28 | global: 29 | fi_getinfo; 30 | fi_freeinfo; 31 | fi_dupinfo; 32 | } FABRIC_1.1; 33 | 34 | FABRIC_1.3 { 35 | global: 36 | fi_getinfo; 37 | fi_freeinfo; 38 | fi_dupinfo; 39 | } FABRIC_1.2; 40 | 41 | FABRIC_1.4 { 42 | global: 43 | fi_tostr_r; 44 | } FABRIC_1.3; 45 | 46 | FABRIC_1.5 { 47 | global: 48 | fi_open; 49 | } FABRIC_1.4; 50 | 51 | FABRIC_1.6 { 52 | global: 53 | fi_log_ready; 54 | } FABRIC_1.5; 55 | 56 | FABRIC_1.7 { 57 | global: 58 | fi_getinfo; 59 | fi_freeinfo; 60 | fi_dupinfo; 61 | } FABRIC_1.6; 62 | -------------------------------------------------------------------------------- /src/common/util/base64.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef SRC_COMMON_UTIL_BASE64_H_ 17 | #define SRC_COMMON_UTIL_BASE64_H_ 18 | 19 | #include 20 | 21 | namespace vineyard { 22 | 23 | std::string base64_encode(const std::string& buffer); 24 | std::string base64_decode(const std::string& buffer); 25 | } // namespace vineyard 26 | #endif // SRC_COMMON_UTIL_BASE64_H_ 27 | -------------------------------------------------------------------------------- /src/common/util/flags.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef SRC_COMMON_UTIL_FLAGS_H_ 17 | #define SRC_COMMON_UTIL_FLAGS_H_ 18 | 19 | #include 20 | 21 | namespace vineyard { 22 | 23 | #ifdef GFLAGS_NAMESPACE 24 | namespace flags = GFLAGS_NAMESPACE; // NOLINT 25 | #else 26 | namespace flags = gflags; 27 | #endif 28 | 29 | } // namespace vineyard 30 | 31 | #endif // SRC_COMMON_UTIL_FLAGS_H_ 32 | -------------------------------------------------------------------------------- /src/common/util/logging.h: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #ifndef SRC_COMMON_UTIL_LOGGING_H_ 17 | #define SRC_COMMON_UTIL_LOGGING_H_ 18 | 19 | #include 20 | 21 | namespace vineyard { 22 | namespace logging = google; // NOLINT 23 | } // namespace vineyard 24 | 25 | #endif // SRC_COMMON_UTIL_LOGGING_H_ 26 | -------------------------------------------------------------------------------- /test/vineyard-cmake-example/example.cc: -------------------------------------------------------------------------------- 1 | /** Copyright 2020-2023 Alibaba Group Holding Limited. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #include "vineyard/client/client.h" 19 | 20 | int main() { 21 | vineyard::Client& client = vineyard::Client::Default(); 22 | std::cout << "Connect to vineyard at: " << client.IPCSocket() << std::endl; 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /vineyard-config-version.in.cmake: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@VINEYARD_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() 12 | --------------------------------------------------------------------------------