├── .env ├── .env.secrets.component ├── .github ├── .codecov.yml ├── CODEOWNERS ├── CONTRIBUTING.md └── workflows │ ├── codeql.yml │ ├── coverage.yml │ ├── golangci-lint.yml │ ├── images.yml │ ├── integration-test.yml │ ├── releases.yml │ ├── semantic.yml │ └── test-compogen.yml ├── .gitignore ├── .golangci.yaml ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.dev ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── init │ ├── definitionupdater │ │ ├── updater.go │ │ └── updater_test.go │ └── main.go ├── main │ └── main.go ├── migration │ └── main.go └── worker │ └── main.go ├── config ├── config.go └── config.yaml ├── github.json ├── go.mod ├── go.sum ├── integration-test ├── data │ ├── bear.jpg │ ├── cat.jpg │ ├── dog-rgba.png │ ├── dog.jpg │ └── dummy-det-model.zip ├── pipeline │ ├── const.js │ ├── grpc-pipeline-private.js │ ├── grpc-pipeline-public-with-jwt.js │ ├── grpc-pipeline-public.js │ ├── grpc-trigger-async.js │ ├── grpc-trigger.js │ ├── grpc.js │ ├── helper.js │ ├── rest-component-definition.js │ ├── rest-integration.js │ ├── rest-pipeline-private.js │ ├── rest-pipeline-public-with-jwt.js │ ├── rest-pipeline-public.js │ ├── rest-trigger-async.js │ ├── rest-trigger.js │ └── rest.js └── proto │ ├── common │ ├── healthcheck │ │ └── v1beta │ │ │ └── healthcheck.proto │ ├── run │ │ └── v1alpha │ │ │ └── run.proto │ └── task │ │ └── v1alpha │ │ └── task.proto │ ├── core │ └── mgmt │ │ └── v1beta │ │ ├── google │ │ ├── api │ │ │ ├── annotations.proto │ │ │ ├── client.proto │ │ │ ├── field_behavior.proto │ │ │ ├── http.proto │ │ │ ├── resource.proto │ │ │ └── visibility.proto │ │ └── type │ │ │ └── date.proto │ │ ├── metric.proto │ │ ├── mgmt.proto │ │ ├── mgmt_private_service.proto │ │ ├── mgmt_public_service.proto │ │ └── protoc-gen-openapiv2 │ │ └── options │ │ ├── annotations.proto │ │ └── openapiv2.proto │ └── pipeline │ └── pipeline │ └── v1beta │ ├── common.proto │ ├── component_definition.proto │ ├── google │ ├── api │ │ ├── annotations.proto │ │ ├── client.proto │ │ ├── field_behavior.proto │ │ ├── http.proto │ │ ├── resource.proto │ │ └── visibility.proto │ ├── longrunning │ │ └── operations.proto │ ├── rpc │ │ └── status.proto │ └── type │ │ └── date.proto │ ├── integration.proto │ ├── pipeline.proto │ ├── pipeline_private_service.proto │ ├── pipeline_public_service.proto │ ├── protoc-gen-openapiv2 │ └── options │ │ ├── annotations.proto │ │ └── openapiv2.proto │ ├── secret.proto │ └── validate │ └── validate.proto ├── pkg ├── acl │ ├── acl.go │ ├── errors.go │ └── model.go ├── component │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── ai │ │ ├── anthropic │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── anthropic.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ └── main.go │ │ ├── cohere │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── cohere.svg │ │ │ │ ├── client.go │ │ │ │ ├── client_test.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── embedding.go │ │ │ │ ├── main.go │ │ │ │ ├── rerank.go │ │ │ │ └── text_generation.go │ │ ├── embedding.go │ │ ├── fireworksai │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── fireworks-ai.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── fireworks_client_interface_mock_test.go │ │ │ │ ├── main.go │ │ │ │ ├── tasks.go │ │ │ │ └── tasks_test.go │ │ ├── groq │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── groq.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── groq_client_interface_mock_test.go │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ ├── huggingface │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── hugging-face.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── structs.go │ │ ├── instill │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── instill-model.svg │ │ │ │ ├── client.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── embedding.go │ │ │ │ ├── main.go │ │ │ │ ├── struct.go │ │ │ │ ├── text_generation.go │ │ │ │ ├── text_generation_chat.go │ │ │ │ ├── text_to_image.go │ │ │ │ └── vision.go │ │ ├── integration_test.go │ │ ├── mistralai │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── mistral-ai.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ ├── ollama │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── setup-hosting.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── ollama.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ ├── ollama_client_interface_mock.gen.go │ │ │ │ └── tasks.go │ │ ├── openai │ │ │ ├── v0 │ │ │ │ ├── .compogen │ │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ │ └── openai.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ │ ├── definition.yaml │ │ │ │ │ ├── setup.yaml │ │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── list_models.go │ │ │ │ ├── main.go │ │ │ │ ├── task_audio_transcriptions.go │ │ │ │ ├── task_text_embeddings.go │ │ │ │ ├── task_text_generation.go │ │ │ │ ├── task_text_to_image.go │ │ │ │ ├── task_text_to_speech.go │ │ │ │ └── testdata │ │ │ │ │ └── sample1.wav │ │ │ └── v1 │ │ │ │ ├── client.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ ├── text_chat_task.go │ │ │ │ └── text_chat_task_test.go │ │ ├── perplexity │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── perplexity.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_chat.go │ │ │ │ └── task_chat_test.go │ │ ├── stabilityai │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── stability-ai.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── list_engines.go │ │ │ │ ├── main.go │ │ │ │ ├── task_image_to_image.go │ │ │ │ ├── task_text_to_image.go │ │ │ │ └── testdata │ │ │ │ └── dog.png │ │ ├── text_chat.go │ │ └── universalai │ │ │ └── v0 │ │ │ ├── .compogen │ │ │ └── bottom.mdx │ │ │ ├── README.mdx │ │ │ ├── assets │ │ │ └── universal-ai.svg │ │ │ ├── client.go │ │ │ ├── config │ │ │ ├── definition.yaml │ │ │ ├── setup.yaml │ │ │ └── tasks.yaml │ │ │ ├── main.go │ │ │ ├── text_chat_task.go │ │ │ └── text_chat_task_test.go │ ├── application │ │ ├── asana │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── asana_task.go │ │ │ │ ├── assets │ │ │ │ └── asana.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── data_type.go │ │ │ │ ├── goal.go │ │ │ │ ├── goal_test.go │ │ │ │ ├── job.go │ │ │ │ ├── main.go │ │ │ │ ├── mockasana │ │ │ │ ├── mock_database.go │ │ │ │ ├── mock_goal.go │ │ │ │ ├── mock_portfolio.go │ │ │ │ ├── mock_project.go │ │ │ │ ├── mock_server.go │ │ │ │ └── mock_task.go │ │ │ │ ├── portfolio.go │ │ │ │ ├── portfolio_test.go │ │ │ │ ├── project.go │ │ │ │ ├── project_test.go │ │ │ │ ├── task.go │ │ │ │ ├── task_test.go │ │ │ │ └── utils.go │ │ ├── email │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── extra-intro.mdx │ │ │ │ ├── extra-read-mails.mdx │ │ │ │ └── extra-setup.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── email.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ ├── read_emails.go │ │ │ │ ├── read_emails_test.go │ │ │ │ ├── send_email.go │ │ │ │ └── send_email_test.go │ │ ├── freshdesk │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── agent.go │ │ │ │ ├── agent_test.go │ │ │ │ ├── assets │ │ │ │ └── freshdesk.svg │ │ │ │ ├── client.go │ │ │ │ ├── company.go │ │ │ │ ├── company_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── contact.go │ │ │ │ ├── contact_test.go │ │ │ │ ├── freshdesk_interface_mock_test.go │ │ │ │ ├── get_all.go │ │ │ │ ├── get_all_test.go │ │ │ │ ├── main.go │ │ │ │ ├── product.go │ │ │ │ ├── product_test.go │ │ │ │ ├── ticket.go │ │ │ │ └── ticket_test.go │ │ ├── github │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── github.svg │ │ │ │ ├── client.go │ │ │ │ ├── commits.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── events.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── event_star_created.go │ │ │ │ ├── event_structs.go │ │ │ │ ├── event_utils.go │ │ │ │ ├── io.go │ │ │ │ ├── issues.go │ │ │ │ ├── main.go │ │ │ │ ├── mock.go │ │ │ │ ├── mock_issues_service.go │ │ │ │ ├── mock_orgs_service.go │ │ │ │ ├── mock_pull_requests_service.go │ │ │ │ ├── mock_repositories_service.go │ │ │ │ ├── mock_users_service.go │ │ │ │ ├── orgs.go │ │ │ │ ├── pull_request.go │ │ │ │ ├── repositories.go │ │ │ │ ├── response.go │ │ │ │ ├── review_comment.go │ │ │ │ ├── task_create_issue.go │ │ │ │ ├── task_create_issue_test.go │ │ │ │ ├── task_create_review_comment.go │ │ │ │ ├── task_create_review_comment_test.go │ │ │ │ ├── task_create_webhook.go │ │ │ │ ├── task_create_webhook_test.go │ │ │ │ ├── task_get_commit.go │ │ │ │ ├── task_get_commit_test.go │ │ │ │ ├── task_get_issue.go │ │ │ │ ├── task_get_issue_test.go │ │ │ │ ├── task_get_organization.go │ │ │ │ ├── task_get_organization_test.go │ │ │ │ ├── task_get_pull_request.go │ │ │ │ ├── task_get_pull_request_test.go │ │ │ │ ├── task_get_user.go │ │ │ │ ├── task_get_user_test.go │ │ │ │ ├── task_list_issues.go │ │ │ │ ├── task_list_issues_test.go │ │ │ │ ├── task_list_pull_requests.go │ │ │ │ ├── task_list_pull_requests_test.go │ │ │ │ ├── task_list_review_comments.go │ │ │ │ ├── task_list_review_comments_test.go │ │ │ │ ├── users.go │ │ │ │ ├── utils.go │ │ │ │ └── webhook.go │ │ ├── googlesearch │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── google-search.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── search.go │ │ ├── hubspot │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── hubspot.svg │ │ │ │ ├── association.go │ │ │ │ ├── association_test.go │ │ │ │ ├── company.go │ │ │ │ ├── company_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── contact.go │ │ │ │ ├── contact_test.go │ │ │ │ ├── custom_client.go │ │ │ │ ├── deal.go │ │ │ │ ├── deal_test.go │ │ │ │ ├── get_all.go │ │ │ │ ├── get_all_test.go │ │ │ │ ├── main.go │ │ │ │ ├── owner.go │ │ │ │ ├── owner_test.go │ │ │ │ ├── thread.go │ │ │ │ ├── thread_test.go │ │ │ │ ├── ticket.go │ │ │ │ └── ticket_test.go │ │ ├── jira │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── jira.svg │ │ │ │ ├── boards.go │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── issues.go │ │ │ │ ├── main.go │ │ │ │ ├── mock_create_issue.go │ │ │ │ ├── mock_create_sprint.go │ │ │ │ ├── mock_database.go │ │ │ │ ├── mock_server.go │ │ │ │ ├── mock_update_issue.go │ │ │ │ ├── mock_update_sprint.go │ │ │ │ ├── sprint.go │ │ │ │ ├── task_create_issue.go │ │ │ │ ├── task_create_issue_test.go │ │ │ │ ├── task_create_sprint.go │ │ │ │ ├── task_create_sprint_test.go │ │ │ │ ├── task_get_issue.go │ │ │ │ ├── task_get_issue_test.go │ │ │ │ ├── task_get_sprint.go │ │ │ │ ├── task_get_sprint_test.go │ │ │ │ ├── task_list_boards.go │ │ │ │ ├── task_list_boards_test.go │ │ │ │ ├── task_list_issues.go │ │ │ │ ├── task_list_issues_test.go │ │ │ │ ├── task_list_sprints.go │ │ │ │ ├── task_list_sprints_test.go │ │ │ │ ├── task_update_issue.go │ │ │ │ ├── task_update_issue_test.go │ │ │ │ ├── task_update_sprint.go │ │ │ │ └── task_update_sprint_test.go │ │ ├── leadiq │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── bottom.mdx │ │ │ │ └── find_prospects.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── leadiq.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── queries │ │ │ │ ├── flat_advanced_search.txt │ │ │ │ └── search_people.txt │ │ │ │ ├── task_find_prospects.go │ │ │ │ └── task_find_prospects_test.go │ │ ├── numbers │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── numbers.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ └── main.go │ │ ├── slack │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── bottom.mdx │ │ │ │ └── extra-setup.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── api_functions.go │ │ │ │ ├── assets │ │ │ │ └── slack.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── events.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── event_new_message.go │ │ │ │ ├── event_structs.go │ │ │ │ ├── main.go │ │ │ │ ├── setup_struct.go │ │ │ │ └── task_functions.go │ │ ├── smartlead │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── bottom.mdx │ │ │ │ └── intro.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── smartlead.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_add_leads.go │ │ │ │ ├── task_add_leads_test.go │ │ │ │ ├── task_add_sender_email.go │ │ │ │ ├── task_add_sender_email_test.go │ │ │ │ ├── task_create_campaign.go │ │ │ │ ├── task_create_campaign_test.go │ │ │ │ ├── task_get_campaign_mertic.go │ │ │ │ ├── task_get_campaign_mertic_test.go │ │ │ │ ├── task_get_sequences.go │ │ │ │ ├── task_get_sequences_test.go │ │ │ │ ├── task_list_leads_status.go │ │ │ │ ├── task_list_leads_status_test.go │ │ │ │ ├── task_save_sequences.go │ │ │ │ ├── task_save_sequences_test.go │ │ │ │ ├── task_setup_campaign.go │ │ │ │ ├── task_setup_campaign_test.go │ │ │ │ ├── task_update_campaign_status.go │ │ │ │ ├── task_update_campaign_status_test.go │ │ │ │ └── utils.go │ │ └── whatsapp │ │ │ └── v0 │ │ │ ├── README.mdx │ │ │ ├── assets │ │ │ └── whatsapp.svg │ │ │ ├── client.go │ │ │ ├── config │ │ │ ├── definition.yaml │ │ │ ├── setup.yaml │ │ │ └── tasks.yaml │ │ │ ├── main.go │ │ │ ├── send_message.go │ │ │ ├── send_message_test.go │ │ │ ├── send_template_message.go │ │ │ └── send_template_message_test.go │ ├── base │ │ ├── component.go │ │ ├── component_test.go │ │ ├── execution.go │ │ ├── executionwrapper.go │ │ ├── executionwrapper_test.go │ │ ├── migrate.go │ │ ├── oauth.go │ │ ├── testdata │ │ │ ├── componentAdditional.yaml │ │ │ ├── componentConfig.yaml │ │ │ ├── componentDef.yaml │ │ │ ├── componentTasks.yaml │ │ │ ├── operatorDef.yaml │ │ │ ├── operatorTasks.yaml │ │ │ ├── test_image.png │ │ │ ├── wantComponentDefinition.yaml │ │ │ └── wantOperatorDefinition.yaml │ │ └── usage.go │ ├── data │ │ ├── bigquery │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── bigquery.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── insert.go │ │ │ │ ├── main.go │ │ │ │ ├── main_test.go │ │ │ │ └── read.go │ │ ├── chroma │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── chroma.svg │ │ │ │ ├── batch_upsert.go │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── create_collection.go │ │ │ │ ├── delete.go │ │ │ │ ├── delete_collection.go │ │ │ │ ├── get_collection.go │ │ │ │ ├── main.go │ │ │ │ ├── query.go │ │ │ │ └── upsert.go │ │ ├── elasticsearch │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── elasticsearch.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ ├── googlecloudstorage │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── gcs.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── create_bucket.go │ │ │ │ ├── main.go │ │ │ │ ├── main_test.go │ │ │ │ ├── read_objects.go │ │ │ │ └── upload.go │ │ ├── googledrive │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── intro.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── google-drive.svg │ │ │ │ ├── client │ │ │ │ └── client.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── main_test.go │ │ │ │ ├── task_read_file.go │ │ │ │ └── task_read_folder.go │ │ ├── googlesheets │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── google-sheets.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_add_sheet.go │ │ │ │ ├── task_add_sheet_test.go │ │ │ │ ├── task_create_spreadsheet.go │ │ │ │ ├── task_create_spreadsheet_column.go │ │ │ │ ├── task_create_spreadsheet_column_test.go │ │ │ │ ├── task_create_spreadsheet_test.go │ │ │ │ ├── task_delete_multiple_rows.go │ │ │ │ ├── task_delete_multiple_rows_test.go │ │ │ │ ├── task_delete_row.go │ │ │ │ ├── task_delete_row_test.go │ │ │ │ ├── task_delete_sheet.go │ │ │ │ ├── task_delete_sheet_test.go │ │ │ │ ├── task_delete_spreadsheet.go │ │ │ │ ├── task_delete_spreadsheet_column.go │ │ │ │ ├── task_delete_spreadsheet_column_test.go │ │ │ │ ├── task_delete_spreadsheet_test.go │ │ │ │ ├── task_get_multiple_rows.go │ │ │ │ ├── task_get_multiple_rows_test.go │ │ │ │ ├── task_get_row.go │ │ │ │ ├── task_get_row_test.go │ │ │ │ ├── task_insert_multiple_rows.go │ │ │ │ ├── task_insert_multiple_rows_test.go │ │ │ │ ├── task_insert_row.go │ │ │ │ ├── task_insert_row_test.go │ │ │ │ ├── task_list_rows.go │ │ │ │ ├── task_list_rows_test.go │ │ │ │ ├── task_lookup_rows.go │ │ │ │ ├── task_lookup_rows_test.go │ │ │ │ ├── task_update_multiple_rows.go │ │ │ │ ├── task_update_multiple_rows_test.go │ │ │ │ ├── task_update_row.go │ │ │ │ ├── task_update_row_test.go │ │ │ │ └── utils.go │ │ ├── instillartifact │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── bottom.mdx │ │ │ │ └── extra-intro.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── instill-artifact.svg │ │ │ │ ├── client.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── main_test.go │ │ │ │ ├── task_get_chunks_metadata.go │ │ │ │ ├── task_get_file_in_markdown.go │ │ │ │ ├── task_get_files_metadata.go │ │ │ │ ├── task_match_files_status.go │ │ │ │ ├── task_query.go │ │ │ │ ├── task_search_chunks.go │ │ │ │ ├── task_sync_files.go │ │ │ │ ├── task_sync_files_test.go │ │ │ │ ├── task_upload_file.go │ │ │ │ ├── task_upload_files.go │ │ │ │ ├── testdata │ │ │ │ ├── test.pdf │ │ │ │ ├── test_2.pdf │ │ │ │ └── test_3.pdf │ │ │ │ └── upload_test.go │ │ ├── milvus │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── milvus.svg │ │ │ │ ├── batch_upsert.go │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── create_collection.go │ │ │ │ ├── create_index.go │ │ │ │ ├── create_partition.go │ │ │ │ ├── delete.go │ │ │ │ ├── drop_collection.go │ │ │ │ ├── drop_index.go │ │ │ │ ├── drop_partition.go │ │ │ │ ├── main.go │ │ │ │ ├── release_collection.go │ │ │ │ ├── search.go │ │ │ │ └── upsert.go │ │ ├── mongodb │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── mongodb.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ ├── pinecone │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── pinecone.svg │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── structs.go │ │ │ │ ├── task_batch_upsert.go │ │ │ │ ├── task_query.go │ │ │ │ ├── task_rerank.go │ │ │ │ └── task_upsert.go │ │ ├── qdrant │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── qdrant.svg │ │ │ │ ├── batch_upsert.go │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── create_collection.go │ │ │ │ ├── delete.go │ │ │ │ ├── delete_collection.go │ │ │ │ ├── main.go │ │ │ │ ├── upsert.go │ │ │ │ └── vector_search.go │ │ ├── redis │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── redis.svg │ │ │ │ ├── chat_history.go │ │ │ │ ├── client.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ └── main.go │ │ ├── sql │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── sql.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ ├── weaviate │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── weaviate.svg │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── main.go │ │ │ │ └── tasks.go │ │ └── zilliz │ │ │ └── v0 │ │ │ ├── README.mdx │ │ │ ├── assets │ │ │ └── zilliz.svg │ │ │ ├── batch_upsert.go │ │ │ ├── client.go │ │ │ ├── component_test.go │ │ │ ├── config │ │ │ ├── definition.yaml │ │ │ ├── setup.yaml │ │ │ └── tasks.yaml │ │ │ ├── create_collection.go │ │ │ ├── create_partition.go │ │ │ ├── delete.go │ │ │ ├── drop_collection.go │ │ │ ├── drop_partition.go │ │ │ ├── main.go │ │ │ ├── search.go │ │ │ └── upsert.go │ ├── generic │ │ ├── collection │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── collection.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_append.go │ │ │ │ ├── task_append_test.go │ │ │ │ ├── task_assign.go │ │ │ │ ├── task_assign_test.go │ │ │ │ ├── task_concat.go │ │ │ │ ├── task_concat_test.go │ │ │ │ ├── task_difference.go │ │ │ │ ├── task_difference_test.go │ │ │ │ ├── task_intersection.go │ │ │ │ ├── task_intersection_test.go │ │ │ │ ├── task_split.go │ │ │ │ ├── task_split_test.go │ │ │ │ ├── task_symmetric_difference.go │ │ │ │ ├── task_symmetric_difference_test.go │ │ │ │ ├── task_union.go │ │ │ │ └── task_union_test.go │ │ ├── http │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── http.svg │ │ │ │ ├── auth.go │ │ │ │ ├── client.go │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── setup.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ └── main.go │ │ └── scheduler │ │ │ └── v0 │ │ │ ├── .compogen │ │ │ └── bottom.mdx │ │ │ ├── README.mdx │ │ │ ├── assets │ │ │ └── scheduler.svg │ │ │ ├── config │ │ │ ├── definition.yaml │ │ │ ├── events.yaml │ │ │ └── tasks.yaml │ │ │ ├── event_cron_job_triggered.go │ │ │ ├── event_structs.go │ │ │ └── main.go │ ├── internal │ │ ├── jsonref │ │ │ ├── interface.go │ │ │ ├── options.go │ │ │ └── ref.go │ │ ├── mock │ │ │ ├── README.md │ │ │ ├── agent_public_service_server_mock.gen.go │ │ │ ├── artifact_public_service_client_mock.gen.go │ │ │ ├── artifact_public_service_server_mock.gen.go │ │ │ ├── command_runner_mock.gen.go │ │ │ ├── error_handler_mock.gen.go │ │ │ ├── generator.go │ │ │ ├── helper.go │ │ │ ├── i_drive_service_mock.gen.go │ │ │ ├── input_reader_mock.gen.go │ │ │ ├── output_writer_mock.gen.go │ │ │ ├── usage_handler_mock.gen.go │ │ │ └── write_closer_mock.gen.go │ │ └── util │ │ │ ├── helper.go │ │ │ ├── helper_test.go │ │ │ └── httpclient │ │ │ ├── httpclient.go │ │ │ └── httpclient_test.go │ ├── operator │ │ ├── audio │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── audio.svg │ │ │ │ ├── audio.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_detect_activity.go │ │ │ │ ├── task_detect_activity_nontag.go │ │ │ │ ├── task_detect_activity_test.go │ │ │ │ ├── task_segment.go │ │ │ │ ├── task_segment_test.go │ │ │ │ └── testdata │ │ │ │ ├── voice1-activity-segments.json │ │ │ │ ├── voice1.wav │ │ │ │ ├── voice2-activity-segments.json │ │ │ │ └── voice2.wav │ │ ├── base64 │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── base64.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── encoding_test.go │ │ │ │ └── main.go │ │ ├── document │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ └── bottom.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── document.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── convert_document_to_markdown.go │ │ │ │ ├── convert_document_to_markdown_test.go │ │ │ │ ├── convert_test.go │ │ │ │ ├── convert_to_images.go │ │ │ │ ├── convert_to_images_test.go │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── testdata │ │ │ │ ├── test.csv │ │ │ │ ├── test.docx │ │ │ │ ├── test.html │ │ │ │ ├── test.jpg │ │ │ │ ├── test.md │ │ │ │ ├── test.odt │ │ │ │ ├── test.pdf │ │ │ │ ├── test.png │ │ │ │ ├── test.pptx │ │ │ │ ├── test.rtf │ │ │ │ ├── test.tif │ │ │ │ ├── test.txt │ │ │ │ ├── test.xls │ │ │ │ └── test.xlsx │ │ │ │ └── transformer │ │ │ │ ├── const.go │ │ │ │ ├── execution │ │ │ │ ├── docling_pdf_to_md_converter.py │ │ │ │ ├── get_page_numbers.py │ │ │ │ ├── image_converter.py │ │ │ │ ├── pdf_checker.py │ │ │ │ └── pdfplumber_pdf_to_md_converter.py │ │ │ │ ├── helper.go │ │ │ │ ├── images.go │ │ │ │ ├── markdown.go │ │ │ │ ├── markdowntransformer.go │ │ │ │ ├── pdf_to_markdown │ │ │ │ ├── __init__.py │ │ │ │ ├── page_image_processor.py │ │ │ │ └── pdf_transformer.py │ │ │ │ └── text.go │ │ ├── image │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── image.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ ├── schema.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── draw.go │ │ │ │ ├── draw_test.go │ │ │ │ ├── font.go │ │ │ │ ├── helper_test.go │ │ │ │ ├── image.go │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_concat.go │ │ │ │ ├── task_concat_test.go │ │ │ │ ├── task_crop.go │ │ │ │ ├── task_crop_test.go │ │ │ │ ├── task_draw_classification.go │ │ │ │ ├── task_draw_classification_test.go │ │ │ │ ├── task_draw_detection.go │ │ │ │ ├── task_draw_detection_test.go │ │ │ │ ├── task_draw_instance_segmentation.go │ │ │ │ ├── task_draw_instance_segmentation_test.go │ │ │ │ ├── task_draw_keypoint.go │ │ │ │ ├── task_draw_keypoint_test.go │ │ │ │ ├── task_draw_ocr.go │ │ │ │ ├── task_draw_ocr_test.go │ │ │ │ ├── task_draw_semantic_segmentation.go │ │ │ │ ├── task_draw_semantic_segmentation_test.go │ │ │ │ ├── task_resize.go │ │ │ │ ├── task_resize_test.go │ │ │ │ └── testdata │ │ │ │ ├── cls-dog.jpeg │ │ │ │ ├── cls-dog.json │ │ │ │ ├── det-coco-1.jpeg │ │ │ │ ├── det-coco-1.json │ │ │ │ ├── det-coco-2.jpeg │ │ │ │ ├── det-coco-2.json │ │ │ │ ├── inst-seg-coco-1.jpeg │ │ │ │ ├── inst-seg-coco-1.json │ │ │ │ ├── inst-seg-coco-2.jpeg │ │ │ │ ├── inst-seg-coco-2.json │ │ │ │ ├── inst-seg-stomata.jpeg │ │ │ │ ├── inst-seg-stomata.json │ │ │ │ ├── kp-coco-1.jpeg │ │ │ │ ├── kp-coco-1.json │ │ │ │ ├── kp-coco-2.jpeg │ │ │ │ ├── kp-coco-2.json │ │ │ │ ├── ocr-mm.jpeg │ │ │ │ ├── ocr-mm.json │ │ │ │ ├── sem-seg-cityscape.jpeg │ │ │ │ ├── sem-seg-cityscape.json │ │ │ │ ├── test_output_task_concat_testconcat_1x4_grid_without_padding.jpeg │ │ │ │ ├── test_output_task_concat_testconcat_2x2_grid_with_padding.jpeg │ │ │ │ ├── test_output_task_concat_testconcat_default_square_grid.jpeg │ │ │ │ ├── test_output_task_crop_testcrop_circular_crop.jpeg │ │ │ │ ├── test_output_task_crop_testcrop_corner_radius_crop.jpeg │ │ │ │ ├── test_output_task_crop_testcrop_no_crop_(all_offsets_0).jpeg │ │ │ │ ├── test_output_task_crop_testcrop_rectangular_crop.jpeg │ │ │ │ ├── test_output_task_draw_classification_testdrawclassification_classification_dog.jpeg │ │ │ │ ├── test_output_task_draw_detection_testdrawdetection_detection_coco_1.jpeg │ │ │ │ ├── test_output_task_draw_detection_testdrawdetection_detection_coco_2.jpeg │ │ │ │ ├── test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_1.jpeg │ │ │ │ ├── test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_2.jpeg │ │ │ │ ├── test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_stomata.jpeg │ │ │ │ ├── test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_1.jpeg │ │ │ │ ├── test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_2.jpeg │ │ │ │ ├── test_output_task_draw_ocr_testdrawocr_ocr_mm.jpeg │ │ │ │ ├── test_output_task_draw_semantic_segmentation_testdrawsemanticsegmentation_semantic_segmentation_cityscape.jpeg │ │ │ │ ├── test_output_task_resize_testresize_resize_by_ratio.jpeg │ │ │ │ ├── test_output_task_resize_testresize_resize_by_ratio_0.jpeg │ │ │ │ ├── test_output_task_resize_testresize_resize_by_width_0.jpeg │ │ │ │ ├── test_output_task_resize_testresize_resize_by_width_and_height.jpeg │ │ │ │ └── test_output_task_resize_testresize_resize_by_width_and_height_0.jpeg │ │ ├── json │ │ │ └── v0 │ │ │ │ ├── .compogen │ │ │ │ ├── bottom.mdx │ │ │ │ └── extra-jq.mdx │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── json.svg │ │ │ │ ├── component_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ └── main.go │ │ ├── text │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── text.svg │ │ │ │ ├── chunk_text.go │ │ │ │ ├── chunk_text_test.go │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── helper.go │ │ │ │ ├── main.go │ │ │ │ ├── main_test.go │ │ │ │ ├── markdown_document.go │ │ │ │ ├── markdown_splitter.go │ │ │ │ ├── markdown_splitter_test.go │ │ │ │ └── testdata │ │ │ │ ├── chinese │ │ │ │ ├── chunk1_1.txt │ │ │ │ ├── chunk1_2.txt │ │ │ │ ├── chunk1_3.txt │ │ │ │ └── text1.txt │ │ │ │ ├── list_position_test.md │ │ │ │ ├── test.docx │ │ │ │ ├── test.html │ │ │ │ ├── test.pdf │ │ │ │ └── test.txt │ │ ├── video │ │ │ └── v0 │ │ │ │ ├── README.mdx │ │ │ │ ├── assets │ │ │ │ └── video.svg │ │ │ │ ├── config │ │ │ │ ├── definition.yaml │ │ │ │ └── tasks.yaml │ │ │ │ ├── helper_test.go │ │ │ │ ├── io.go │ │ │ │ ├── main.go │ │ │ │ ├── task_embed_audio.go │ │ │ │ ├── task_embed_audio_test.go │ │ │ │ ├── task_extract_audio.go │ │ │ │ ├── task_extract_audio_test.go │ │ │ │ ├── task_extract_frames.go │ │ │ │ ├── task_extract_frames_test.go │ │ │ │ ├── task_segment.go │ │ │ │ ├── task_segment_test.go │ │ │ │ ├── task_subsample.go │ │ │ │ ├── task_subsample_test.go │ │ │ │ └── testdata │ │ │ │ ├── audio.ogg │ │ │ │ ├── embed-video.mp4 │ │ │ │ ├── frame-0.png │ │ │ │ ├── frame-1.png │ │ │ │ ├── frame-2.png │ │ │ │ ├── frame-3.png │ │ │ │ ├── frame-4.png │ │ │ │ ├── video-sample-bunny.mp4 │ │ │ │ ├── video-segment-0.mp4 │ │ │ │ ├── video-segment-1.mp4 │ │ │ │ ├── video-segment-2.mp4 │ │ │ │ ├── video-segment-3.mp4 │ │ │ │ ├── video-segment-4.mp4 │ │ │ │ ├── video-segment-5.mp4 │ │ │ │ ├── video-segments.json │ │ │ │ ├── video-subsample-280x0.mp4 │ │ │ │ ├── video-subsample-abr-128.mp4 │ │ │ │ ├── video-subsample-fps-15.mp4 │ │ │ │ ├── video-subsample-vbr-1000.mp4 │ │ │ │ └── video.mp4 │ │ └── web │ │ │ └── v0 │ │ │ ├── .compogen │ │ │ ├── bottom.mdx │ │ │ ├── crawl_site.mdx │ │ │ └── scrape_page.mdx │ │ │ ├── README.mdx │ │ │ ├── assets │ │ │ └── web.svg │ │ │ ├── config │ │ │ ├── definition.yaml │ │ │ └── tasks.yaml │ │ │ ├── crawl_website.go │ │ │ ├── helper.go │ │ │ ├── helper_test.go │ │ │ ├── main.go │ │ │ ├── main_test.go │ │ │ ├── scrape_sitemap.go │ │ │ └── scrape_webpages.go │ ├── resources │ │ ├── onnx │ │ │ └── silero_vad.onnx │ │ └── schemas │ │ │ ├── schema.go │ │ │ └── schema.yaml │ ├── store │ │ └── store.go │ └── tools │ │ └── compogen │ │ ├── Makefile │ │ ├── README.md │ │ ├── cmd │ │ ├── execute_test.go │ │ ├── readme.go │ │ ├── root.go │ │ └── testdata │ │ │ ├── readme-component1.txt │ │ │ ├── readme-component2.txt │ │ │ ├── readme-usage.txt │ │ │ └── usage.txt │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ └── pkg │ │ └── gen │ │ ├── case.go │ │ ├── component_type.go │ │ ├── definition.go │ │ ├── definition_test.go │ │ ├── readme.go │ │ ├── readme_test.go │ │ ├── resources │ │ └── templates │ │ │ └── readme.mdx.tmpl │ │ ├── schema.go │ │ ├── schema_test.go │ │ ├── task.go │ │ ├── utils.go │ │ └── validation.go ├── constant │ └── constant.go ├── data │ ├── array.go │ ├── audio.go │ ├── audio_test.go │ ├── boolean.go │ ├── bytearray.go │ ├── convert.go │ ├── data.go │ ├── document.go │ ├── document_test.go │ ├── encode.go │ ├── encode_test.go │ ├── file.go │ ├── format │ │ ├── format.go │ │ └── value.go │ ├── image.go │ ├── image_test.go │ ├── map.go │ ├── null.go │ ├── number.go │ ├── number_test.go │ ├── path │ │ ├── path.go │ │ └── path_test.go │ ├── string.go │ ├── struct.go │ ├── struct_test.go │ ├── testdata │ │ ├── original_links.txt │ │ ├── readme.md │ │ ├── sample1.docx │ │ ├── sample1.mp3 │ │ ├── sample1.ogg │ │ ├── sample1.wav │ │ ├── sample2.pdf │ │ ├── sample2.txt │ │ ├── sample_640_360.mov │ │ ├── sample_640_360.mp4 │ │ ├── sample_640_360.wmv │ │ ├── sample_640_426.jpeg │ │ ├── sample_640_426.png │ │ └── sample_640_426.tiff │ ├── utils.go │ ├── video.go │ └── video_test.go ├── datamodel │ ├── datamodel.go │ ├── datamodel_test.go │ └── runlogging.go ├── db │ ├── db.go │ └── migration │ │ ├── 000001_init.down.sql │ │ ├── 000001_init.up.sql │ │ ├── 000002_init.down.sql │ │ ├── 000002_init.up.sql │ │ ├── 000003_init.down.sql │ │ ├── 000003_init.up.sql │ │ ├── 000004_init.down.sql │ │ ├── 000004_init.up.sql │ │ ├── 000005_init.down.sql │ │ ├── 000005_init.up.sql │ │ ├── 000006_init.down.sql │ │ ├── 000006_init.up.sql │ │ ├── 000007_init.down.sql │ │ ├── 000007_init.up.sql │ │ ├── 000008_init.down.sql │ │ ├── 000008_init.up.sql │ │ ├── 000009_init.down.sql │ │ ├── 000009_init.up.sql │ │ ├── 000010_init.down.sql │ │ ├── 000010_init.up.sql │ │ ├── 000011_init.down.sql │ │ ├── 000011_init.up.sql │ │ ├── 000012_init.down.sql │ │ ├── 000012_init.up.sql │ │ ├── 000013_init.down.sql │ │ ├── 000013_init.up.sql │ │ ├── 000014_init.down.sql │ │ ├── 000014_init.up.sql │ │ ├── 000015_init.down.sql │ │ ├── 000015_init.up.sql │ │ ├── 000016_init.down.sql │ │ ├── 000016_init.up.sql │ │ ├── 000017_init.down.sql │ │ ├── 000017_init.up.sql │ │ ├── 000018_init.down.sql │ │ ├── 000018_init.up.sql │ │ ├── 000019_init.down.sql │ │ ├── 000019_init.up.sql │ │ ├── 000020_init.down.sql │ │ ├── 000020_init.up.sql │ │ ├── 000021_init.down.sql │ │ ├── 000021_init.up.sql │ │ ├── 000022_init.down.sql │ │ ├── 000022_init.up.sql │ │ ├── 000023_run_logging.down.sql │ │ ├── 000023_run_logging.up.sql │ │ ├── 000024_init.down.sql │ │ ├── 000024_init.up.sql │ │ ├── 000025_add_integration_column.down.sql │ │ ├── 000025_add_integration_column.up.sql │ │ ├── 000026_add_connections.down.sql │ │ ├── 000026_add_connections.up.sql │ │ ├── 000027_run_logging_namespace.down.sql │ │ ├── 000027_run_logging_namespace.up.sql │ │ ├── 000028_oauth_integration.down.sql │ │ ├── 000028_oauth_integration.up.sql │ │ ├── 000029_init.down.sql │ │ ├── 000029_init.up.sql │ │ ├── 000030_connection_identity.down.sql │ │ ├── 000030_connection_identity.up.sql │ │ ├── 000031_migrate_slack_setup.down.sql │ │ ├── 000031_migrate_slack_setup.up.sql │ │ ├── 000032_migrate_web_recipes.down.sql │ │ ├── 000032_migrate_web_recipes.down.up.sql │ │ ├── 000033_migrate_web_recipes.down.sql │ │ ├── 000033_migrate_web_recipes.up.sql │ │ ├── 000034_rename_http_component.down.sql │ │ ├── 000034_rename_http_component.up.sql │ │ ├── 000035_run_requester_uid.down.sql │ │ ├── 000035_run_requester_uid.up.sql │ │ ├── 000036_rename_instill_format.down.sql │ │ ├── 000036_rename_instill_format.up.sql │ │ ├── 000037_create_pipeline_run_on_table.down.sql │ │ ├── 000037_create_pipeline_run_on_table.up.sql │ │ ├── 000038_add_pipeline_run_on_table_column.down.sql │ │ ├── 000038_add_pipeline_run_on_table_column.up.sql │ │ ├── 000039_add_pipeline_run_expiration_time.down.sql │ │ ├── 000039_add_pipeline_run_expiration_time.up.sql │ │ ├── 000040_rename_format_to_type.down.sql │ │ ├── 000040_rename_format_to_type.up.sql │ │ ├── convert │ │ ├── convert000013 │ │ │ └── main.go │ │ ├── convert000015 │ │ │ └── main.go │ │ ├── convert000016 │ │ │ └── main.go │ │ ├── convert000019 │ │ │ └── convert.go │ │ ├── convert000020 │ │ │ └── convert.go │ │ ├── convert000021 │ │ │ └── convert.go │ │ ├── convert000022 │ │ │ └── convert.go │ │ ├── convert000024 │ │ │ └── convert.go │ │ ├── convert000029 │ │ │ └── convert.go │ │ ├── convert000031 │ │ │ └── convert.go │ │ ├── convert000032 │ │ │ └── convert.go │ │ ├── convert000033 │ │ │ └── convert.go │ │ ├── convert000034 │ │ │ └── convert.go │ │ ├── convert000036 │ │ │ └── convert.go │ │ ├── convert000039 │ │ │ └── convert.go │ │ ├── convert000040 │ │ │ └── convert.go │ │ ├── converter.go │ │ └── legacy │ │ │ ├── 000006_migrate.go │ │ │ ├── 000007_migrate.go │ │ │ └── 000012_migrate.go │ │ └── migration.go ├── errors │ └── errors.go ├── external │ └── external.go ├── handler │ ├── component_definition.go │ ├── errors.go │ ├── fieldannotation.go │ ├── integration.go │ ├── main.go │ ├── pipeline.go │ ├── secret.go │ ├── triggerhandler.go │ ├── utils.go │ └── webhook.go ├── logger │ ├── logger.go │ └── otel │ │ ├── const.go │ │ ├── metrics.go │ │ └── tracing.go ├── memory │ ├── store.go │ └── workflow.go ├── middleware │ ├── interceptor.go │ ├── interceptor_test.go │ ├── middleware.go │ └── misc.go ├── mock │ ├── acl_client_interface_mock.gen.go │ ├── converter_mock.gen.go │ ├── generator.go │ ├── mgmt_private_service_client_mock.gen.go │ └── repository_mock.gen.go ├── pubsub │ ├── pubsub.go │ └── redis.go ├── recipe │ ├── connection.go │ ├── dag.go │ ├── eval.go │ ├── eval_test.go │ ├── schema.go │ ├── schema.json │ └── variable.go ├── repository │ ├── errors.go │ ├── influx.go │ ├── repository.go │ ├── repository_mocksql_test.go │ ├── repository_test.go │ ├── transpiler.go │ └── utils.go ├── resource │ └── resource.go ├── service │ ├── blobstorage.go │ ├── component_definition.go │ ├── convert.go │ ├── errors.go │ ├── integration.go │ ├── main.go │ ├── metadataretention.go │ ├── pipeline.go │ ├── pipeline_test.go │ ├── pipelinerun.go │ ├── pipelinerun_test.go │ ├── secret.go │ ├── utils.go │ ├── validator.go │ └── webhook.go ├── usage │ └── usage.go ├── utils │ ├── async.go │ ├── blobstorage.go │ └── utils.go └── worker │ ├── blobstorage.go │ ├── io.go │ ├── main.go │ ├── minioactivity.go │ ├── utils.go │ └── workflow.go ├── release-please ├── config.json └── manifest.json └── resp.json /.github/.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | informational: true 6 | patch: 7 | default: 8 | informational: true 9 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @donch1989 @jvallesm 2 | *.md @GeorgeWilliamStrong 3 | *.mdx @GeorgeWilliamStrong 4 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: Code Scanning 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | paths-ignore: 9 | - "**/*.md" 10 | - "**/*.mdx" 11 | schedule: 12 | - cron: "0 0 * * 0" 13 | 14 | jobs: 15 | CodeQL-Build: 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - name: Check out code 20 | uses: actions/checkout@v3 21 | 22 | - name: Load .env file 23 | uses: cardinalby/export-env-action@v2 24 | with: 25 | envFile: .env 26 | 27 | - uses: actions/setup-go@v5 28 | with: 29 | go-version: ${{ env.GOLANG_VERSION }} 30 | 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@v1 33 | with: 34 | languages: go 35 | queries: security-and-quality 36 | 37 | - name: Perform CodeQL Analysis 38 | uses: github/codeql-action/analyze@v1 39 | -------------------------------------------------------------------------------- /.github/workflows/semantic.yml: -------------------------------------------------------------------------------- 1 | name: Lint PR 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | name: Validate PR title 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: amannn/action-semantic-pull-request@v4 16 | with: 17 | requireScope: true 18 | env: 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/test-compogen.yml: -------------------------------------------------------------------------------- 1 | name: Compogen test 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "pkg/component/tools/compogen/**" 7 | push: 8 | branches: 9 | - main 10 | paths: 11 | - "pkg/component/tools/compogen/**" 12 | 13 | jobs: 14 | test-compogen: 15 | name: Test Compogen 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | token: ${{ secrets.GITHUB_TOKEN }} 21 | 22 | - name: Load .env file 23 | uses: cardinalby/export-env-action@v2 24 | with: 25 | envFile: .env 26 | expand: true 27 | 28 | - uses: actions/setup-go@v5 29 | with: 30 | go-version: ${{ env.GOLANG_VERSION }} 31 | 32 | - name: Test compogen 33 | run: | 34 | cd pkg/component/tools/compogen 35 | go mod tidy 36 | go test -mod=mod -race -cover ./... 37 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | enable: 4 | - staticcheck 5 | settings: 6 | staticcheck: 7 | checks: 8 | - all 9 | - -SA1019 10 | exclusions: 11 | generated: lax 12 | presets: 13 | - comments 14 | - common-false-positives 15 | - legacy 16 | - std-error-handling 17 | paths: 18 | - third_party$ 19 | - builtin$ 20 | - examples$ 21 | formatters: 22 | exclusions: 23 | generated: lax 24 | paths: 25 | - third_party$ 26 | - builtin$ 27 | - examples$ 28 | -------------------------------------------------------------------------------- /integration-test/data/bear.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/integration-test/data/bear.jpg -------------------------------------------------------------------------------- /integration-test/data/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/integration-test/data/cat.jpg -------------------------------------------------------------------------------- /integration-test/data/dog-rgba.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/integration-test/data/dog-rgba.png -------------------------------------------------------------------------------- /integration-test/data/dog.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/integration-test/data/dog.jpg -------------------------------------------------------------------------------- /integration-test/data/dummy-det-model.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/integration-test/data/dummy-det-model.zip -------------------------------------------------------------------------------- /integration-test/proto/common/healthcheck/v1beta/healthcheck.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package common.healthcheck.v1beta; 4 | 5 | // Google API 6 | import "google/api/field_behavior.proto"; 7 | 8 | // HealthCheckRequest represents a request to health check a service 9 | message HealthCheckRequest { 10 | // Service name to check for its readiness status 11 | optional string service = 1 [(google.api.field_behavior) = OPTIONAL]; 12 | } 13 | 14 | // HealthCheckResponse represents a response for a service heath status 15 | message HealthCheckResponse { 16 | // ServingStatus enumerates the status of a queried service 17 | enum ServingStatus { 18 | // Serving status: UNSPECIFIED 19 | SERVING_STATUS_UNSPECIFIED = 0; 20 | // Serving status: SERVING 21 | SERVING_STATUS_SERVING = 1; 22 | // Serving status: NOT SERVING 23 | SERVING_STATUS_NOT_SERVING = 2; 24 | } 25 | 26 | // Status is the instance of the enum type ServingStatus 27 | ServingStatus status = 1; 28 | } 29 | -------------------------------------------------------------------------------- /integration-test/proto/common/run/v1alpha/run.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package common.run.v1alpha; 4 | 5 | // RunStatus defines the status of a pipeline or model run. 6 | enum RunStatus { 7 | // Unspecified. 8 | RUN_STATUS_UNSPECIFIED = 0; 9 | // Run in progress. 10 | RUN_STATUS_PROCESSING = 1; 11 | // Run succeeded. 12 | RUN_STATUS_COMPLETED = 2; 13 | // Run failed. 14 | RUN_STATUS_FAILED = 3; 15 | // Run is waiting to be executed. 16 | RUN_STATUS_QUEUED = 4; 17 | } 18 | 19 | // RunSource defines the source of a pipeline or model run. 20 | enum RunSource { 21 | // Unspecified. 22 | RUN_SOURCE_UNSPECIFIED = 0; 23 | // Run from frontend UI. 24 | RUN_SOURCE_CONSOLE = 1; 25 | // Run from API or SDK. 26 | RUN_SOURCE_API = 2; 27 | } 28 | -------------------------------------------------------------------------------- /pkg/acl/errors.go: -------------------------------------------------------------------------------- 1 | package acl 2 | 3 | import "errors" 4 | 5 | var ErrMembershipNotFound = errors.New("membership not found") 6 | -------------------------------------------------------------------------------- /pkg/component/ai/anthropic/v0/assets/anthropic.svg: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /pkg/component/ai/anthropic/v0/client.go: -------------------------------------------------------------------------------- 1 | package anthropic 2 | 3 | import ( 4 | "google.golang.org/protobuf/types/known/structpb" 5 | 6 | anthropicsdk "github.com/anthropics/anthropic-sdk-go" 7 | anthropicsdkoption "github.com/anthropics/anthropic-sdk-go/option" 8 | ) 9 | 10 | func newClient(apiKey string) *anthropicsdk.Client { 11 | client := anthropicsdk.NewClient( 12 | anthropicsdkoption.WithAPIKey(apiKey), // defaults to os.LookupEnv("ANTHROPIC_API_KEY") 13 | ) 14 | return client 15 | 16 | } 17 | 18 | func getAPIKey(setup *structpb.Struct) string { 19 | return setup.GetFields()[cfgAPIKey].GetStringValue() 20 | } 21 | -------------------------------------------------------------------------------- /pkg/component/ai/anthropic/v0/component_test.go: -------------------------------------------------------------------------------- 1 | package anthropic 2 | 3 | // TODO: Add tests 4 | // We change the implementation to use the Anthropic SDK. 5 | // So, we need to test the new implementation. 6 | -------------------------------------------------------------------------------- /pkg/component/ai/anthropic/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | custom: false 4 | documentationUrl: https://instill-ai.dev/docs/component/ai/anthropic 5 | icon: assets/anthropic.svg 6 | id: anthropic 7 | public: true 8 | title: Anthropic 9 | description: Connect the AI models served on the Anthropic Platform. 10 | type: COMPONENT_TYPE_AI 11 | uid: 42bdb620-74ad-486a-82da-25e45431b42c 12 | vendor: Anthropic 13 | vendorAttributes: {} 14 | version: 0.1.1 15 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/anthropic/v0 16 | releaseStage: RELEASE_STAGE_ALPHA 17 | -------------------------------------------------------------------------------- /pkg/component/ai/anthropic/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Anthropic API key. To find your keys, visit the Anthropic console page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Anthropic Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/cohere/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | - TASK_TEXT_EMBEDDINGS 4 | - TASK_TEXT_RERANKING 5 | documentationUrl: https://instill-ai.dev/docs/component/ai/cohere 6 | icon: assets/cohere.svg 7 | id: cohere 8 | public: true 9 | title: Cohere 10 | description: Connect the AI models served on the Cohere Platform. 11 | type: COMPONENT_TYPE_AI 12 | uid: 11550338-de54-4338-a4ca-4a21c4757817 13 | vendor: Cohere 14 | vendorAttributes: {} 15 | version: 0.1.0 16 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/cohere/v0 17 | releaseStage: RELEASE_STAGE_ALPHA 18 | -------------------------------------------------------------------------------- /pkg/component/ai/cohere/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Cohere API key. To find your keys, visit the Cohere dashboard page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Cohere Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/fireworksai/v0/.compogen/bottom.mdx: -------------------------------------------------------------------------------- 1 | ## Example Recipes 2 | 3 | ```yaml 4 | version: v1beta 5 | component: 6 | fireworks-0: 7 | type: fireworks-ai 8 | task: TASK_TEXT_GENERATION_CHAT 9 | input: 10 | max-new-tokens: 200 11 | model: qwen2-72b-instruct 12 | prompt: ${variable.prompt} 13 | system-message: You are a expert social media content writing assistant. Output the result in chinese for 小紅書. 14 | temperature: 0.05 15 | top-k: 10 16 | top-p: 0.5 17 | setup: 18 | api-key: ${secret.INSTILL_SECRET} 19 | variable: 20 | prompt: 21 | title: prompt 22 | description: input prompt i.e. "寫一份生椰拿鐵文案", "write about horses" 23 | type: string 24 | output: 25 | output: 26 | title: output 27 | value: ${fireworks-0.output.text} 28 | ``` 29 | -------------------------------------------------------------------------------- /pkg/component/ai/fireworksai/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | - TASK_TEXT_EMBEDDINGS 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/ai/fireworks-ai 6 | icon: assets/fireworks-ai.svg 7 | id: fireworks-ai 8 | public: true 9 | title: Fireworks AI 10 | description: Connect the AI models served on the Fireworks AI Platform. 11 | type: COMPONENT_TYPE_AI 12 | uid: 09258316-dad7-4b84-be50-41eb76ba9cf0 13 | vendor: Fireworks AI 14 | vendorAttributes: {} 15 | version: 0.1.0 16 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/fireworksai/v0 17 | releaseStage: RELEASE_STAGE_ALPHA 18 | -------------------------------------------------------------------------------- /pkg/component/ai/fireworksai/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Fireworks AI API key. To find your keys, visit the Fireworks AI API Keys page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Fireworks AI Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/groq/v0/.compogen/bottom.mdx: -------------------------------------------------------------------------------- 1 | ## Example Recipes 2 | 3 | ```yaml 4 | version: v1beta 5 | component: 6 | groq-0: 7 | type: groq 8 | task: TASK_TEXT_GENERATION_CHAT 9 | input: 10 | max-new-tokens: 300 11 | model: llama3-groq-70b-8192-tool-use-preview 12 | prompt: |- 13 | Rewrite this experience using the STAR (Situation, Task, Action, Result) method for a resume or CV: 14 | 15 | ${variable.experience} 16 | system-message: You are a helpful resume assistant. 17 | temperature: 0.05 18 | top-k: 10 19 | top-p: 0.5 20 | user: instill-ai 21 | setup: 22 | api-key: ${secret.INSTILL_SECRET} 23 | variable: 24 | experience: 25 | title: experience 26 | description: describe your work experience 27 | type: string 28 | instill-ui-multiline: true 29 | output: 30 | resume_format: 31 | title: resume_format 32 | value: ${groq-0.output.text} 33 | ``` 34 | -------------------------------------------------------------------------------- /pkg/component/ai/groq/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | documentationUrl: https://instill-ai.dev/docs/component/ai/groq 4 | icon: assets/groq.svg 5 | id: groq 6 | public: true 7 | title: Groq 8 | description: Connect the AI models served on GroqCloud. 9 | type: COMPONENT_TYPE_AI 10 | uid: d5e64e5c-2dd2-4358-82dd-0e3a035c2157 11 | vendor: Groq 12 | vendorAttributes: {} 13 | version: 0.1.0 14 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/groq/v0 15 | releaseStage: RELEASE_STAGE_ALPHA 16 | -------------------------------------------------------------------------------- /pkg/component/ai/groq/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your GroqCloud API key. To find your keys, visit the GroqCloud API Keys page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: GroqCloud Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/instill/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CLASSIFICATION 3 | - TASK_INSTANCE_SEGMENTATION 4 | - TASK_KEYPOINT 5 | - TASK_DETECTION 6 | - TASK_OCR 7 | - TASK_SEMANTIC_SEGMENTATION 8 | - TASK_TEXT_GENERATION 9 | - TASK_TEXT_GENERATION_CHAT 10 | - TASK_TEXT_TO_IMAGE 11 | - TASK_VISUAL_QUESTION_ANSWERING 12 | - TASK_CHAT 13 | - TASK_EMBEDDING 14 | custom: false 15 | documentationUrl: https://instill-ai.dev/docs/component/ai/instill-model 16 | icon: assets/instill-model.svg 17 | iconUrl: '' 18 | id: instill-model 19 | public: true 20 | title: Instill Model 21 | description: Connect the AI models served on the Instill Model Platform. 22 | tombstone: false 23 | type: COMPONENT_TYPE_AI 24 | uid: ddcf42c3-4c30-4c65-9585-25f1c89b2b48 25 | vendor: Instill 26 | vendorAttributes: {} 27 | version: 0.1.0 28 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/instill/v0 29 | releaseStage: RELEASE_STAGE_ALPHA 30 | -------------------------------------------------------------------------------- /pkg/component/ai/instill/v0/struct.go: -------------------------------------------------------------------------------- 1 | package instill 2 | 3 | type RequestWrapper struct { 4 | Data any `json:"data,omitempty"` 5 | Parameter any `json:"parameter,omitempty"` 6 | } 7 | -------------------------------------------------------------------------------- /pkg/component/ai/mistralai/v0/component_test.go: -------------------------------------------------------------------------------- 1 | package mistralai 2 | 3 | // TODO: Add tests 4 | // We change the implementation to use the stream API. 5 | // So, we need to test the new implementation. 6 | -------------------------------------------------------------------------------- /pkg/component/ai/mistralai/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | - TASK_TEXT_EMBEDDINGS 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/ai/mistral-ai 6 | icon: assets/mistral-ai.svg 7 | id: mistral-ai 8 | public: true 9 | title: Mistral AI 10 | description: Connect the AI models served on the Mistral AI Platform. 11 | type: COMPONENT_TYPE_AI 12 | uid: 5e349d27-b00d-4961-86a3-249c30c06073 13 | vendor: Mistral AI 14 | vendorAttributes: {} 15 | version: 0.1.0 16 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/mistralai/v0 17 | releaseStage: RELEASE_STAGE_ALPHA 18 | -------------------------------------------------------------------------------- /pkg/component/ai/mistralai/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Mistral API key. To find your keys, visit the Mistral AI platform page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Mistral AI Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/ollama/v0/.compogen/setup-hosting.mdx: -------------------------------------------------------------------------------- 1 | #### Local Ollama Instance 2 | 3 | To set up an Ollama instance on your local machine, follow the instructions below: 4 | 5 | > Note: These instructions only work for Instill Core CE 6 | 7 | 1. Follow the tutorial on the official [GitHub repository](https://github.com/ollama/ollama) to install Ollama on your machine. 8 | 2. Follow the instructions in the [FAQ section](https://github.com/ollama/ollama/blob/main/docs/faq.md) to modify the variable `OLLAMA_HOST` to `0.0.0.0`, then restart Ollama. 9 | 3. Get the IP address of your machine on the local network. 10 | - On Linux and macOS, open the terminal and type `ifconfig`. 11 | - On Windows, open the command prompt and type `ipconfig`. 12 | 4. Suppose the IP address is `192.168.178.88`, then the Ollama hosting endpoint would be `192.168.178.88:11434`. 13 | 5. Enjoy fast LLM inference on your local machine and integration with 💧 Instill Pipeline. 14 | -------------------------------------------------------------------------------- /pkg/component/ai/ollama/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION_CHAT 3 | - TASK_TEXT_EMBEDDINGS 4 | documentationUrl: https://instill-ai.dev/docs/component/ai/ollama 5 | icon: assets/ollama.svg 6 | id: ollama 7 | public: true 8 | title: Ollama 9 | description: Connect the AI models served with the Ollama library. 10 | type: COMPONENT_TYPE_AI 11 | uid: 5f6dcfc4-efd0-45a1-aae9-c9b4beb68a32 12 | vendor: Ollama 13 | vendorAttributes: {} 14 | version: 0.1.0 15 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/ollama/v0 16 | releaseStage: RELEASE_STAGE_ALPHA 17 | -------------------------------------------------------------------------------- /pkg/component/ai/ollama/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | endpoint: 4 | description: 'Fill in your Ollama hosting endpoint. ### WARNING ###: As of 2024-07-26, the Ollama component does not support authentication methods. 5 | To prevent unauthorized access to your Ollama serving resources, please implement additional security measures such as IP whitelisting.' 6 | type: string 7 | default: http://localhost:11434 8 | uiOrder: 0 9 | title: Endpoint 10 | auto-pull: 11 | description: Automatically pull the requested models from the Ollama server if the model is not found in the local cache. 12 | type: boolean 13 | uiOrder: 1 14 | title: Model Auto-Pull 15 | required: 16 | - endpoint 17 | - auto-pull 18 | title: Ollama Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_GENERATION 3 | - TASK_TEXT_EMBEDDINGS 4 | - TASK_SPEECH_RECOGNITION 5 | - TASK_TEXT_TO_SPEECH 6 | - TASK_TEXT_TO_IMAGE 7 | custom: false 8 | icon: assets/openai.svg 9 | iconUrl: '' 10 | id: openai 11 | public: true 12 | title: OpenAI 13 | description: Connect the AI models served on the OpenAI Platform. 14 | type: COMPONENT_TYPE_AI 15 | uid: 9fb6a2cb-bff5-4c69-bc6d-4538dd8e3362 16 | vendor: OpenAI 17 | vendorAttributes: {} 18 | version: 0.1.1 19 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/openai/v0 20 | releaseStage: RELEASE_STAGE_ALPHA 21 | documentationUrl: https://instill-ai.dev/docs/component/ai/openai 22 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your OpenAI API key. To find your keys, visit your OpenAI's API Keys page. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | organization: 11 | description: Specify which organization is used for the requests. Usage will count against the specified organization's subscription quota. 12 | type: string 13 | uiOrder: 1 14 | title: Organization ID 15 | required: [] 16 | title: OpenAI Connection 17 | type: object 18 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/task_text_embeddings.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | const ( 4 | embeddingsPath = "/v1/embeddings" 5 | ) 6 | 7 | type TextEmbeddingsReq struct { 8 | Model string `json:"model"` 9 | Dimensions int `json:"dimensions,omitempty"` 10 | Input []string `json:"input"` 11 | } 12 | 13 | type TextEmbeddingsResp struct { 14 | Object string `json:"object"` 15 | Data []Data `json:"data"` 16 | Model string `json:"model"` 17 | Usage usageOpenAI `json:"usage"` 18 | } 19 | 20 | type Data struct { 21 | Object string `json:"object"` 22 | Embedding []float32 `json:"embedding"` 23 | Index int `json:"index"` 24 | } 25 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/task_text_to_image.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | const ( 4 | imgGenerationPath = "/v1/images/generations" 5 | ) 6 | 7 | type ImageGenerationsReq struct { 8 | Prompt string `json:"prompt"` 9 | Model string `json:"model"` 10 | N *int `json:"n,omitempty"` 11 | Quality *string `json:"quality,omitempty"` 12 | Size *string `json:"size,omitempty"` 13 | Style *string `json:"style,omitempty"` 14 | ResponseFormat string `json:"response_format"` 15 | } 16 | 17 | type ImageGenerationsRespData struct { 18 | Image string `json:"b64_json"` 19 | RevisedPrompt string `json:"revised_prompt"` 20 | } 21 | type ImageGenerationsResp struct { 22 | Data []ImageGenerationsRespData `json:"data"` 23 | } 24 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/task_text_to_speech.go: -------------------------------------------------------------------------------- 1 | package openai 2 | 3 | const ( 4 | createSpeechPath = "/v1/audio/speech" 5 | ) 6 | 7 | type TextToSpeechReq struct { 8 | Input string `json:"input"` 9 | Model string `json:"model"` 10 | Voice string `json:"voice"` 11 | ResponseFormat *string `json:"response_format,omitempty"` 12 | Speed *float32 `json:"speed,omitempty"` 13 | } 14 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v0/testdata/sample1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/ai/openai/v0/testdata/sample1.wav -------------------------------------------------------------------------------- /pkg/component/ai/openai/v1/config/definition.yaml: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v1/config/setup.yaml: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v1/config/tasks.yaml: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /pkg/component/ai/openai/v1/text_chat_task_test.go: -------------------------------------------------------------------------------- 1 | // TODO: chuang8511 2 | package openaiv1 3 | -------------------------------------------------------------------------------- /pkg/component/ai/perplexity/v0/assets/perplexity.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/ai/perplexity/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CHAT 3 | custom: false 4 | documentationUrl: https://instill-ai.dev/docs/component/ai/perplexity 5 | icon: assets/perplexity.svg 6 | id: perplexity 7 | public: true 8 | title: Perplexity 9 | description: Connect the AI models served on the Perplexity Platform. 10 | type: COMPONENT_TYPE_AI 11 | uid: 61339791-dfe0-4ba1-9efc-7d4452c03e53 12 | vendor: Perplexity 13 | vendorAttributes: {} 14 | version: 0.1.0 15 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/perplexity/v0 16 | releaseStage: RELEASE_STAGE_ALPHA 17 | -------------------------------------------------------------------------------- /pkg/component/ai/perplexity/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your API key from the vendor's platform. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Perplexity Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/perplexity/v0/task_chat_test.go: -------------------------------------------------------------------------------- 1 | package perplexity 2 | -------------------------------------------------------------------------------- /pkg/component/ai/stabilityai/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_TEXT_TO_IMAGE 3 | - TASK_IMAGE_TO_IMAGE 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/ai/stability-ai 6 | icon: assets/stability-ai.svg 7 | iconUrl: '' 8 | id: stability-ai 9 | public: true 10 | title: Stability AI 11 | description: Connect the AI models served on the Stability AI Platform. 12 | tombstone: false 13 | type: COMPONENT_TYPE_AI 14 | uid: c86a95cc-7d32-4e22-a290-8c699f6705a4 15 | vendor: Stability AI 16 | vendorAttributes: {} 17 | version: 0.1.0 18 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/stabilityai/v0 19 | releaseStage: RELEASE_STAGE_ALPHA 20 | -------------------------------------------------------------------------------- /pkg/component/ai/stabilityai/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Stability AI API key. To find your keys, visit here. 5 | type: string 6 | instillSecret: true 7 | instillCredential: true 8 | uiOrder: 0 9 | title: API Key 10 | required: [] 11 | title: Stability AI Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/ai/stabilityai/v0/list_engines.go: -------------------------------------------------------------------------------- 1 | package stabilityai 2 | 3 | const ( 4 | listEnginesPath = "/v1/engines/list" 5 | ) 6 | 7 | // Engine represents a Stability AI Engine. 8 | type Engine struct { 9 | Description string `json:"description"` 10 | ID string `json:"id"` 11 | Name string `json:"name"` 12 | Type string `json:"type"` 13 | } 14 | -------------------------------------------------------------------------------- /pkg/component/ai/stabilityai/v0/testdata/dog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/ai/stabilityai/v0/testdata/dog.png -------------------------------------------------------------------------------- /pkg/component/ai/universalai/v0/.compogen/bottom.mdx: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Example Recipes 4 | 5 | Please refer to the part of `type: universal-ai` 6 | 7 | ```yaml 8 | version: v1beta 9 | variable: 10 | prompt: 11 | title: Prompt 12 | description: "User input message to be sent to the AI model." 13 | type: string 14 | 15 | output: 16 | ai-output: 17 | title: AI Output 18 | value: "${ai.output.data.choices[0]}" 19 | 20 | component: 21 | ai: 22 | type: universal-ai 23 | input: 24 | parameter: 25 | n: 1 26 | stream: true 27 | data: 28 | messages: 29 | - content: 30 | - type: text 31 | text: ${variable.prompt} 32 | role: user 33 | setup: 34 | model: gpt-4 35 | task: TASK_CHAT 36 | ``` 37 | -------------------------------------------------------------------------------- /pkg/component/ai/universalai/v0/client.go: -------------------------------------------------------------------------------- 1 | package universalai 2 | 3 | import ( 4 | "fmt" 5 | 6 | "go.uber.org/zap" 7 | "google.golang.org/protobuf/types/known/structpb" 8 | 9 | openaiv1 "github.com/instill-ai/pipeline-backend/pkg/component/ai/openai/v1" 10 | ) 11 | 12 | func newClient(setup *structpb.Struct, logger *zap.Logger, vendor string) (interface{}, error) { 13 | switch vendor { 14 | case "openai": 15 | return openaiv1.NewClient(setup, logger), nil 16 | default: 17 | return nil, fmt.Errorf("unsupported vendor: %s", vendor) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/component/ai/universalai/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CHAT 3 | custom: false 4 | icon: assets/universal-ai.svg 5 | id: universal-ai 6 | public: true 7 | title: Universal AI 8 | description: Connect the AI models served on the different platforms with standardized input and output formats. 9 | type: COMPONENT_TYPE_AI 10 | uid: 7656cb11-d504-4ca0-b481-6ef80964f2c9 11 | vendorAttributes: {} 12 | version: 0.2.0 13 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/ai/universalai/v0 14 | releaseStage: RELEASE_STAGE_ALPHA 15 | documentationUrl: https://instill-ai.dev/docs/component/ai/universal-ai 16 | -------------------------------------------------------------------------------- /pkg/component/application/asana/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CRUD_GOAL 3 | - TASK_CRUD_TASK 4 | - TASK_CRUD_PROJECT 5 | - TASK_CRUD_PORTFOLIO 6 | documentationUrl: https://instill-ai.dev/docs/component/application/asana 7 | icon: assets/asana.svg 8 | id: asana 9 | public: true 10 | title: Asana 11 | vendor: Asana 12 | description: Do anything available on Asana. 13 | tombstone: false 14 | type: COMPONENT_TYPE_APPLICATION 15 | uid: 7c83cda3-50a9-46fc-992f-c4e8b949bbec 16 | version: 0.1.0 17 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/asana/v0 18 | releaseStage: RELEASE_STAGE_ALPHA 19 | -------------------------------------------------------------------------------- /pkg/component/application/asana/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | token: 4 | description: Fill in your Asana Personal Access Token (PAT). You can generate one from developer console. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: Token 9 | required: 10 | - token 11 | title: Asana Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/application/email/v0/.compogen/extra-intro.mdx: -------------------------------------------------------------------------------- 1 | You can connect to different email servers through the Email component. 2 | Emails are fetched and sent using the IMAP and SMTP protocols, respectively. You can set the server address and port for each protocol in the component configuration. 3 | -------------------------------------------------------------------------------- /pkg/component/application/email/v0/.compogen/extra-read-mails.mdx: -------------------------------------------------------------------------------- 1 | #### Mailbox 2 | 3 | You have to confirm what exactly the mailbox name is. 4 | Take Gmail as an example, the mailbox names are following. 5 | 6 | | Mailbox | Mailbox Name to input | 7 | | :--- | :--- | 8 | | Inbox | `INBOX` | 9 | | Sent | `[Gmail]/Sent Mail` | 10 | | Drafts | `[Gmail]/Drafts` | 11 | 12 | 13 | #### Search From and Search To 14 | You need to input the exact same email address with `<` as a prefix and `>` as a suffix as the email you want to search for. 15 | For example, if you want to search for the email from `email@example.com`, you need to input ``. 16 | -------------------------------------------------------------------------------- /pkg/component/application/email/v0/assets/email.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/application/email/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_SEND_EMAIL 3 | - TASK_READ_EMAILS 4 | documentationUrl: https://instill-ai.dev/docs/component/application/email 5 | icon: assets/email.svg 6 | id: email 7 | public: true 8 | title: Email 9 | description: Get and send email from Mail Protocol. 10 | tombstone: false 11 | type: COMPONENT_TYPE_APPLICATION 12 | uid: ee8edff7-443f-459d-8db1-a99ea71db233 13 | version: 0.1.0 14 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/email/v0 15 | releaseStage: RELEASE_STAGE_ALPHA 16 | -------------------------------------------------------------------------------- /pkg/component/application/email/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | email-address: 4 | description: The email address of the user. 5 | type: string 6 | instillSecret: false 7 | uiOrder: 2 8 | title: Email Address 9 | password: 10 | description: The password of the App passwords in Gmail settings. 11 | type: string 12 | instillSecret: true 13 | uiOrder: 3 14 | title: App Password 15 | server-address: 16 | description: The address of the email server. 17 | type: string 18 | uiOrder: 0 19 | title: Server Address 20 | server-port: 21 | description: The port of the email server. 22 | type: integer 23 | uiOrder: 1 24 | title: Server Port 25 | required: 26 | - email-address 27 | - password 28 | - server-address 29 | - server-port 30 | title: Email 31 | type: object 32 | -------------------------------------------------------------------------------- /pkg/component/application/freshdesk/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_GET_TICKET 3 | - TASK_CREATE_TICKET 4 | - TASK_REPLY_TO_TICKET 5 | - TASK_CREATE_TICKET_NOTE 6 | - TASK_GET_ALL_CONVERSATIONS 7 | - TASK_GET_CONTACT 8 | - TASK_CREATE_CONTACT 9 | - TASK_GET_COMPANY 10 | - TASK_CREATE_COMPANY 11 | - TASK_GET_ALL 12 | - TASK_GET_PRODUCT 13 | - TASK_GET_AGENT 14 | - TASK_GET_ROLE 15 | - TASK_GET_GROUP 16 | - TASK_GET_SKILL 17 | documentationUrl: https://instill-ai.dev/docs/component/application/freshdesk 18 | icon: assets/freshdesk.svg 19 | id: freshdesk 20 | public: true 21 | title: Freshdesk 22 | description: Use Freshdesk API to manage tickets, contacts and more. 23 | tombstone: false 24 | type: COMPONENT_TYPE_APPLICATION 25 | uid: e89b5830-bc28-4886-9e04-e3db6e0efe9f 26 | vendor: Freshdesk 27 | vendorAttributes: {} 28 | version: 0.1.0 29 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/freshdesk/v0 30 | releaseStage: RELEASE_STAGE_ALPHA 31 | -------------------------------------------------------------------------------- /pkg/component/application/freshdesk/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Freshdesk API key. To find your key, go to profile settigs and on the right pane, you can get your key once you have completed 5 | the captcha verification. 6 | type: string 7 | instillSecret: true 8 | uiOrder: 0 9 | title: API key 10 | domain: 11 | description: Your Freshdesk domain. You can find this by going to Admin -> Search for "Portals" -> Portal URL. Your portal URL will look like "yourdomain.freshdesk.com". 12 | Please only input your domain and not the whole URL. 13 | type: string 14 | instillSecret: true 15 | uiOrder: 1 16 | title: Domain 17 | required: 18 | - api-key 19 | - domain 20 | title: Freshdesk Connection 21 | type: object 22 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_LIST_PULL_REQUESTS 3 | - TASK_GET_PULL_REQUEST 4 | - TASK_GET_COMMIT 5 | - TASK_LIST_REVIEW_COMMENTS 6 | - TASK_CREATE_REVIEW_COMMENT 7 | - TASK_LIST_ISSUES 8 | - TASK_GET_ISSUE 9 | - TASK_GET_USER 10 | - TASK_GET_ORGANIZATION 11 | - TASK_CREATE_ISSUE 12 | - TASK_CREATE_WEBHOOK 13 | availableEvents: 14 | - EVENT_STAR_CREATED 15 | documentationUrl: https://instill-ai.dev/docs/component/application/github 16 | icon: assets/github.svg 17 | id: github 18 | public: true 19 | title: GitHub 20 | vendor: GitHub 21 | description: Do anything available on GitHub. 22 | tombstone: false 23 | type: COMPONENT_TYPE_APPLICATION 24 | uid: 9c14438b-90fa-41fc-83bb-4a3d9b8cbba6 25 | version: 0.1.0 26 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/github/v0 27 | releaseStage: RELEASE_STAGE_ALPHA 28 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | token: 4 | description: Fill in your GitHub access token for advanced usages. For more information about how to create tokens, please refer to the github 5 | settings. 6 | type: string 7 | instillSecret: true 8 | uiOrder: 0 9 | title: Token 10 | required: 11 | - token 12 | instillOAuthConfig: 13 | authUrl: https://github.com/login/oauth/authorize 14 | accessUrl: https://github.com/login/oauth/access_token 15 | scopes: 16 | - repo 17 | - admin:repo_hook 18 | title: GitHub Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/event_star_created.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | "github.com/instill-ai/pipeline-backend/pkg/data" 8 | ) 9 | 10 | func (c *component) handleStarCreated(ctx context.Context, rawEvent *base.RawEvent) (parsedEvent *base.ParsedEvent, err error) { 11 | unmarshaler := data.NewUnmarshaler(c.BinaryFetcher) 12 | 13 | r := rawGithubStarCreated{} 14 | err = unmarshaler.Unmarshal(ctx, rawEvent.Message, &r) 15 | if err != nil { 16 | return nil, err 17 | } 18 | 19 | githubEvent := githubStarCreated{ 20 | Action: r.Action, 21 | StarredAt: r.StarredAt, 22 | Repository: convertRawRepository(r.Repository), 23 | Sender: convertRawUser(r.Sender), 24 | } 25 | marshaler := data.NewMarshaler() 26 | m, err := marshaler.Marshal(githubEvent) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return &base.ParsedEvent{ 32 | ParsedMessage: m, 33 | Response: data.Map{}, 34 | }, nil 35 | } 36 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/mock.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | const ( 4 | fakeHost = "https://fake-github.com" 5 | ) 6 | 7 | func middleWare(req string) int { 8 | if req == "rate_limit" { 9 | return 403 10 | } 11 | if req == "not_found" { 12 | return 404 13 | } 14 | if req == "unprocessable_entity" { 15 | return 422 16 | } 17 | if req == "no_pr" { 18 | return 201 19 | } 20 | return 200 21 | } 22 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/task_get_commit.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (client *Client) getCommit(ctx context.Context, job *base.Job) error { 11 | var input getCommitInput 12 | if err := job.Input.ReadData(ctx, &input); err != nil { 13 | return fmt.Errorf("reading input data: %w", err) 14 | } 15 | owner, repository, err := parseTargetRepo(input) 16 | if err != nil { 17 | return err 18 | } 19 | sha := input.SHA 20 | commit, _, err := client.Repositories.GetCommit(ctx, owner, repository, sha, nil) 21 | if err != nil { 22 | return addErrMsgToClientError(err) 23 | } 24 | var output getCommitOutput 25 | output.Commit, err = client.extractCommitInformation(ctx, owner, repository, commit, true) 26 | if err != nil { 27 | return err 28 | } 29 | if err := job.Output.WriteData(ctx, output); err != nil { 30 | return err 31 | } 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/task_get_issue.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (client *Client) getIssue(ctx context.Context, job *base.Job) error { 11 | var input getIssueInput 12 | if err := job.Input.ReadData(ctx, &input); err != nil { 13 | return fmt.Errorf("reading input data: %w", err) 14 | } 15 | owner, repository, err := parseTargetRepo(input) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | issueNumber := input.IssueNumber 21 | issue, _, err := client.Issues.Get(ctx, owner, repository, issueNumber) 22 | if err != nil { 23 | return addErrMsgToClientError(err) 24 | } 25 | 26 | var output getIssueOutput 27 | output.Issue = client.extractIssue(issue) 28 | if err := job.Output.WriteData(ctx, output); err != nil { 29 | return err 30 | } 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/task_get_organization.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/google/go-github/v62/github" 8 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 9 | ) 10 | 11 | func (client *Client) getOrganization(ctx context.Context, job *base.Job) error { 12 | var input getOrganizationInput 13 | if err := job.Input.ReadData(ctx, &input); err != nil { 14 | return fmt.Errorf("reading input data: %w", err) 15 | } 16 | 17 | var org *github.Organization 18 | var err error 19 | 20 | if input.OrgID != 0 { 21 | org, _, err = client.Organizations.GetByID(ctx, input.OrgID) 22 | } else { 23 | org, _, err = client.Organizations.Get(ctx, input.OrgName) 24 | } 25 | if err != nil { 26 | return addErrMsgToClientError(err) 27 | } 28 | 29 | var output getOrganizationOutput 30 | output.Organization = client.extractOrganization(org) 31 | if err := job.Output.WriteData(ctx, output); err != nil { 32 | return err 33 | } 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/task_get_user.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/google/go-github/v62/github" 8 | 9 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 10 | ) 11 | 12 | func (client *Client) getUser(ctx context.Context, job *base.Job) error { 13 | var input getUserInput 14 | if err := job.Input.ReadData(ctx, &input); err != nil { 15 | return fmt.Errorf("reading input data: %w", err) 16 | } 17 | 18 | var user *github.User 19 | var err error 20 | 21 | if input.UserID != 0 { 22 | user, _, err = client.Users.GetByID(ctx, input.UserID) 23 | } else { 24 | user, _, err = client.Users.Get(ctx, input.Username) 25 | } 26 | if err != nil { 27 | return addErrMsgToClientError(err) 28 | } 29 | 30 | var output getUserOutput 31 | output.User = client.extractUser(user) 32 | if err := job.Output.WriteData(ctx, output); err != nil { 33 | return err 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/component/application/github/v0/utils.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | // PageOptions represents the pagination options for a request 4 | type PageOptions struct { 5 | Page int `instill:"page"` 6 | PerPage int `instill:"per-page"` 7 | } 8 | 9 | // getString safely dereferences a string pointer and returns an empty string if nil 10 | func getString(s *string) string { 11 | if s != nil { 12 | return *s 13 | } 14 | return "" 15 | } 16 | 17 | // getInt safely dereferences an int pointer and returns 0 if nil 18 | func getInt(i *int) int { 19 | if i != nil { 20 | return *i 21 | } 22 | return 0 23 | } 24 | 25 | // getInt64 safely dereferences an int64 pointer and returns 0 if nil 26 | func getInt64(i *int64) int64 { 27 | if i != nil { 28 | return *i 29 | } 30 | return 0 31 | } 32 | 33 | // getBool safely dereferences a bool pointer and returns false if nil 34 | func getBool(b *bool) bool { 35 | if b != nil { 36 | return *b 37 | } 38 | return false 39 | } 40 | -------------------------------------------------------------------------------- /pkg/component/application/googlesearch/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_SEARCH 3 | custom: false 4 | documentationUrl: https://instill-ai.dev/docs/component/application/google-search 5 | icon: assets/google-search.svg 6 | iconUrl: '' 7 | id: google-search 8 | public: true 9 | title: Google Search 10 | description: Leverage the Google Search engine. 11 | tombstone: false 12 | type: COMPONENT_TYPE_APPLICATION 13 | uid: 2b1da686-878a-462c-b2c6-a9690199939c 14 | vendor: Google 15 | vendorAttributes: {} 16 | version: 0.1.1 17 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/googlesearch/v0 18 | releaseStage: RELEASE_STAGE_ALPHA 19 | -------------------------------------------------------------------------------- /pkg/component/application/hubspot/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_GET_CONTACT 3 | - TASK_CREATE_CONTACT 4 | - TASK_GET_DEAL 5 | - TASK_CREATE_DEAL 6 | - TASK_UPDATE_DEAL 7 | - TASK_GET_COMPANY 8 | - TASK_CREATE_COMPANY 9 | - TASK_GET_TICKET 10 | - TASK_CREATE_TICKET 11 | - TASK_UPDATE_TICKET 12 | - TASK_GET_THREAD 13 | - TASK_INSERT_MESSAGE 14 | - TASK_RETRIEVE_ASSOCIATION 15 | - TASK_GET_OWNER 16 | - TASK_GET_ALL 17 | documentationUrl: https://instill-ai.dev/docs/component/application/hubspot 18 | icon: assets/hubspot.svg 19 | id: hubspot 20 | public: true 21 | title: HubSpot 22 | description: Use HubSpot application to do various tasks. 23 | tombstone: false 24 | type: COMPONENT_TYPE_APPLICATION 25 | uid: 0cd80b30-29bc-4f19-91ca-5911de3a3aae 26 | vendor: HubSpot 27 | vendorAttributes: {} 28 | version: 0.1.0 29 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/hubspot/v0 30 | releaseStage: RELEASE_STAGE_ALPHA 31 | -------------------------------------------------------------------------------- /pkg/component/application/hubspot/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | token: 4 | description: Fill in your HubSpot private app access token. Go here for [more information](https://developers.hubspot.com/docs/api/private-apps). 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: Token 9 | required: 10 | - token 11 | title: HubSpot Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/application/jira/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CREATE_ISSUE 3 | - TASK_CREATE_SPRINT 4 | - TASK_GET_ISSUE 5 | - TASK_GET_SPRINT 6 | - TASK_LIST_BOARDS 7 | - TASK_LIST_ISSUES 8 | - TASK_LIST_SPRINTS 9 | - TASK_UPDATE_ISSUE 10 | - TASK_UPDATE_SPRINT 11 | documentationUrl: https://instill-ai.dev/docs/component/application/jira 12 | icon: assets/jira.svg 13 | id: jira 14 | public: true 15 | title: Jira 16 | vendor: Atlassian 17 | description: Do anything available on Jira. 18 | tombstone: false 19 | type: COMPONENT_TYPE_APPLICATION 20 | uid: 3b27f50d-a754-4b9d-8141-95aaec647cc5 21 | version: 0.1.0 22 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/jira/v0 23 | releaseStage: RELEASE_STAGE_ALPHA 24 | -------------------------------------------------------------------------------- /pkg/component/application/jira/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | token: 4 | description: Fill in your Jira API token. You can generate one from your Jira account "settings > security > API tokens". 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: Token 9 | email: 10 | description: Fill in your Jira email address. 11 | type: string 12 | uiOrder: 1 13 | title: Email 14 | base-url: 15 | description: Fill in your Jira base URL. For example, if your Jira URL is "https://mycompany.atlassian.net/...", then your base URL is https://mycompany.atlassian.net. 16 | type: string 17 | uiOrder: 1 18 | title: Base URL 19 | required: 20 | - token 21 | - email 22 | - base-url 23 | title: Jira Connection 24 | type: object 25 | -------------------------------------------------------------------------------- /pkg/component/application/jira/v0/task_get_issue.go: -------------------------------------------------------------------------------- 1 | package jira 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (c *client) getIssue(ctx context.Context, job *base.Job) error { 11 | var input getIssueInput 12 | if err := job.Input.ReadData(ctx, &input); err != nil { 13 | return fmt.Errorf("reading input data: %w", err) 14 | } 15 | 16 | issue, err := getIssue(c.Client, input.IssueKey, input.UpdateHistory) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | output := getIssueOutput{Issue: *issue} 22 | return job.Output.WriteData(ctx, output) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/component/application/jira/v0/task_get_sprint.go: -------------------------------------------------------------------------------- 1 | package jira 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | "github.com/instill-ai/x/errmsg" 9 | ) 10 | 11 | func (c *client) getSprint(ctx context.Context, job *base.Job) error { 12 | var input getSprintInput 13 | if err := job.Input.ReadData(ctx, &input); err != nil { 14 | return fmt.Errorf("reading input data: %w", err) 15 | } 16 | 17 | apiEndpoint := fmt.Sprintf("rest/agile/1.0/sprint/%v", input.SprintID) 18 | req := c.R().SetResult(&Sprint{}) 19 | resp, err := req.Get(apiEndpoint) 20 | if err != nil { 21 | return fmt.Errorf("getting sprint: %w", err) 22 | } 23 | 24 | issue, ok := resp.Result().(*Sprint) 25 | if !ok { 26 | return errmsg.AddMessage( 27 | fmt.Errorf("failed to convert response to `Get Sprint` Output"), 28 | fmt.Sprintf("failed to convert %v to `Get Sprint` Output", resp.Result()), 29 | ) 30 | } 31 | output := *extractSprintOutput(issue) 32 | return job.Output.WriteData(ctx, output) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/component/application/jira/v0/task_list_boards.go: -------------------------------------------------------------------------------- 1 | package jira 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (c *client) listBoards(ctx context.Context, job *base.Job) error { 11 | var input listBoardsInput 12 | if err := job.Input.ReadData(ctx, &input); err != nil { 13 | return fmt.Errorf("reading input data: %w", err) 14 | } 15 | 16 | resp, err := listBoards(c, &input) 17 | if err != nil { 18 | return fmt.Errorf("listing boards: %w", err) 19 | } 20 | var output listBoardsOutput 21 | output.Boards = append(output.Boards, resp.Boards...) 22 | if output.Boards == nil { 23 | output.Boards = []Board{} 24 | } 25 | output.StartAt = resp.StartAt 26 | output.MaxResults = resp.MaxResults 27 | output.IsLast = resp.IsLast 28 | output.Total = resp.Total 29 | return job.Output.WriteData(ctx, output) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/component/application/leadiq/v0/.compogen/find_prospects.mdx: -------------------------------------------------------------------------------- 1 | #### Regex Match Sample 2 | 3 | - Match Sales or Founder with case-insensitive 4 | ```regex 5 | (?i)\b(sales|founder)\b 6 | ``` 7 | 8 | 9 | - Match titles containing "Manager", "Director", or "VP" (case-insensitive) 10 | ```regex 11 | (?i)\b(manager|director|vp)\b 12 | ``` 13 | 14 | - Match titles starting with "Chief" (e.g., "Chief Executive Officer", "Chief Marketing Officer") 15 | ```regex 16 | (?i)\bchief\s\w+\b 17 | ``` 18 | 19 | 20 | - Match "Engineer", "Developer", or "Programmer" (case-insensitive) 21 | ```regex 22 | (?i)\b(engineer|developer|programmer)\b 23 | ``` 24 | -------------------------------------------------------------------------------- /pkg/component/application/leadiq/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_FIND_PROSPECTS 3 | documentationUrl: https://instill-ai.dev/docs/component/application/leadiq 4 | icon: assets/leadiq.svg 5 | id: leadiq 6 | public: true 7 | title: LeadIQ 8 | vendor: LeadIQ 9 | description: Search for prospects and enrich your leads. 10 | tombstone: false 11 | type: COMPONENT_TYPE_APPLICATION 12 | uid: 9fbda0b2-e8a6-497f-bd26-7e3f0c40ea44 13 | version: 0.1.0 14 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/leadiq/v0 15 | releaseStage: RELEASE_STAGE_ALPHA 16 | -------------------------------------------------------------------------------- /pkg/component/application/leadiq/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your LeadIQ Secret Base64 API key, you can find it in [API Key](https://account.leadiq.com/app/settings/api-keys). 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | required: 10 | - api-key 11 | title: LeadIQ Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/application/leadiq/v0/queries/search_people.txt: -------------------------------------------------------------------------------- 1 | query SearchPeople($input: SearchPeopleInput!) { 2 | searchPeople(input: $input) { 3 | totalResults 4 | results { 5 | name { 6 | first 7 | fullName 8 | last 9 | } 10 | linkedin { 11 | linkedinUrl 12 | } 13 | personalPhones { 14 | type 15 | status 16 | value 17 | } 18 | currentPositions { 19 | title 20 | seniority 21 | function 22 | emails { 23 | type 24 | status 25 | value 26 | } 27 | phones { 28 | type 29 | status 30 | value 31 | } 32 | companyInfo { 33 | name 34 | } 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pkg/component/application/leadiq/v0/task_find_prospects_test.go: -------------------------------------------------------------------------------- 1 | // will be added in ins-6946 2 | package leadiq 3 | -------------------------------------------------------------------------------- /pkg/component/application/numbers/v0/assets/numbers.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/application/numbers/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_REGISTER 3 | custom: false 4 | documentationUrl: https://instill-ai.dev/docs/component/application/numbers 5 | icon: assets/numbers.svg 6 | iconUrl: '' 7 | id: numbers 8 | public: true 9 | title: Numbers Protocol 10 | description: Seamlessly integrate third-party blockchain services through the Numbers Protocol, providing security, verifiability and traceability to data 11 | management. 12 | tombstone: false 13 | type: COMPONENT_TYPE_APPLICATION 14 | uid: 70d8664a-d512-4517-a5e8-5d4da81756a7 15 | vendor: Numbers Protocol 16 | vendorAttributes: {} 17 | version: 0.1.0 18 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/numbers/v0 19 | releaseStage: RELEASE_STAGE_ALPHA 20 | -------------------------------------------------------------------------------- /pkg/component/application/numbers/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | capture-token: 4 | description: Fill in your Capture token in the Capture App. To access your tokens, you need a Capture App account and you can sign in with email or 5 | wallet to acquire the Capture Token. 6 | type: string 7 | instillSecret: true 8 | uiOrder: 0 9 | title: Capture token 10 | required: 11 | - capture-token 12 | title: Numbers Protocol Connection 13 | type: object 14 | -------------------------------------------------------------------------------- /pkg/component/application/slack/v0/client.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "github.com/slack-go/slack" 5 | ) 6 | 7 | func newClient(token string) *slack.Client { 8 | return slack.New(token) 9 | } 10 | -------------------------------------------------------------------------------- /pkg/component/application/slack/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_READ_MESSAGE 3 | - TASK_WRITE_MESSAGE 4 | availableEvents: 5 | - EVENT_NEW_MESSAGE 6 | custom: false 7 | documentationUrl: https://instill-ai.dev/docs/component/application/slack 8 | icon: assets/slack.svg 9 | iconUrl: '' 10 | id: slack 11 | public: true 12 | title: Slack 13 | description: Get and send message on Slack. 14 | tombstone: false 15 | type: COMPONENT_TYPE_APPLICATION 16 | uid: 1e9f469e-da5e-46eb-8a89-23466627e3b5 17 | vendor: Slack 18 | vendorAttributes: {} 19 | version: 0.2.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/slack/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/application/slack/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | bot-token: 4 | description: Token associated to the application bot. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: Bot OAuth Token 9 | user-token: 10 | description: Token to act on behalf of a Slack user. 11 | type: string 12 | instillSecret: true 13 | uiOrder: 1 14 | title: User OAuth Token 15 | required: 16 | - bot-token 17 | instillOAuthConfig: 18 | authUrl: https://slack.com/oauth/v2/authorize 19 | accessUrl: https://slack.com/api/oauth.v2.access 20 | scopes: 21 | - channels:history 22 | - channels:read 23 | - groups:history 24 | - groups:read 25 | - chat:write 26 | - users:read 27 | - users:read.email 28 | - users.profile:read 29 | title: Slack Connection 30 | type: object 31 | -------------------------------------------------------------------------------- /pkg/component/application/slack/v0/setup_struct.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | type slackComponentSetup struct { 4 | BotToken string `instill:"bot-token"` 5 | UserToken *string `instill:"user-token"` 6 | } 7 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/.compogen/intro.mdx: -------------------------------------------------------------------------------- 1 | ## Use Cases 2 | 3 | To achieve the basic use case that creating a campaign from scratch, you need to follow the steps below: 4 | 5 | 1. Create campaign by `TASK_CREATE_CAMPAIGN`. 6 | 2. Update campaign schedule & update campaign general settings by `TASK_SETUP_CAMPAIGN`. 7 | 3. Save sequences by `TASK_SAVE_SEQUENCES`. 8 | 4. Add leads to the campaign by `TASK_ADD_LEADS`. 9 | 5. Add sender email by `TASK_ADD_SENDER_EMAIL`. Please notice that you need to create a sender email and configure in Smartlead console before adding it to the campaign. 10 | 6. Start campaign by `TASK_UPDATE_CAMPAIGN_STATUS`. -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CREATE_CAMPAIGN 3 | - TASK_SETUP_CAMPAIGN 4 | - TASK_SAVE_SEQUENCES 5 | - TASK_GET_SEQUENCES 6 | - TASK_ADD_LEADS 7 | - TASK_ADD_SENDER_EMAIL 8 | - TASK_UPDATE_CAMPAIGN_STATUS 9 | - TASK_GET_CAMPAIGN_METRIC 10 | - TASK_LIST_LEADS_STATUS 11 | documentationUrl: https://instill-ai.dev/docs/component/application/smartlead 12 | icon: assets/smartlead.svg 13 | id: smartlead 14 | public: true 15 | title: Smartlead 16 | vendor: Smartlead 17 | description: Organize your leads and helps you close more deals. 18 | tombstone: false 19 | type: COMPONENT_TYPE_APPLICATION 20 | uid: 4a736615-025b-4edf-a736-2a3905b00704 21 | version: 0.1.0 22 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/smartlead/v0 23 | releaseStage: RELEASE_STAGE_ALPHA 24 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Smartlead API key. You can generate one from your [Smartlead account](https://app.smartlead.ai/app/settings/profile). 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | required: 10 | - api-key 11 | title: Smartlead Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_add_leads_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_add_sender_email_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_create_campaign_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_get_campaign_mertic_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_get_sequences_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_list_leads_status_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_save_sequences_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_setup_campaign_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/smartlead/v0/task_update_campaign_status_test.go: -------------------------------------------------------------------------------- 1 | // Will do it in ins-6969 2 | package smartlead 3 | -------------------------------------------------------------------------------- /pkg/component/application/whatsapp/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_SEND_TEXT_BASED_TEMPLATE_MESSAGE 3 | - TASK_SEND_MEDIA_BASED_TEMPLATE_MESSAGE 4 | - TASK_SEND_LOCATION_BASED_TEMPLATE_MESSAGE 5 | - TASK_SEND_AUTHENTICATION_TEMPLATE_MESSAGE 6 | - TASK_SEND_TEXT_MESSAGE 7 | - TASK_SEND_MEDIA_MESSAGE 8 | - TASK_SEND_LOCATION_MESSAGE 9 | - TASK_SEND_CONTACT_MESSAGE 10 | - TASK_SEND_INTERACTIVE_CALL_TO_ACTION_URL_BUTTON_MESSAGE 11 | documentationUrl: https://instill-ai.dev/docs/component/application/whatsapp 12 | icon: assets/whatsapp.svg 13 | id: whatsapp 14 | public: true 15 | title: WhatsApp 16 | description: Use WhatsApp Business Platform API to send template and messages. 17 | tombstone: false 18 | type: COMPONENT_TYPE_APPLICATION 19 | uid: 028c0ce7-94ff-4ac2-9324-d9a362724eed 20 | vendor: WhatsApp 21 | vendorAttributes: {} 22 | version: 0.1.0 23 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/whatsapp/v0 24 | releaseStage: RELEASE_STAGE_ALPHA 25 | -------------------------------------------------------------------------------- /pkg/component/application/whatsapp/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | token: 4 | description: Fill in your WhatsApp access token. Go [here](https://developers.facebook.com/docs/whatsapp/cloud-api/get-started) for more information. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: Token 9 | required: 10 | - token 11 | title: WhatsApp Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/componentAdditional.yaml: -------------------------------------------------------------------------------- 1 | components: 2 | schemas: 3 | CreateEmbeddingRequest: 4 | additionalProperties: false 5 | properties: 6 | model: 7 | description: ID of the model to use 8 | enum: 9 | - text-embedding-ada-002 10 | type: string 11 | required: 12 | - model 13 | type: object 14 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/componentConfig.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: true 2 | properties: 3 | api-key: 4 | description: Fill in your OpenAI API key. To find your keys, visit your OpenAI's API Keys page. 5 | instillSecret: true 6 | uiOrder: 0 7 | title: API Key 8 | type: string 9 | required: 10 | - api-key 11 | title: OpenAI Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/componentDef.yaml: -------------------------------------------------------------------------------- 1 | version: 1.0.0 2 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/base 3 | availableTasks: 4 | - TASK_TEXT_EMBEDDINGS 5 | custom: false 6 | documentationUrl: https://instill-ai.dev/docs/component/ai/openai 7 | icon: OpenAI/openai.svg 8 | id: openai 9 | public: true 10 | title: OpenAI 11 | type: COMPONENT_TYPE_AI 12 | uid: 9fb6a2cb-bff5-4c69-bc6d-4538dd8e3362 13 | vendor: OpenAI 14 | vendorAttributes: {} 15 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/componentTasks.yaml: -------------------------------------------------------------------------------- 1 | TASK_TEXT_EMBEDDINGS: 2 | shortDescription: Turn text into numbers, unlocking use cases like search. 3 | input: 4 | uiOrder: 0 5 | properties: 6 | model: 7 | $ref: additional.yaml#/components/schemas/CreateEmbeddingRequest/properties/model 8 | type: string 9 | shortDescription: ID of the model to use 10 | uiOrder: 0 11 | title: Model 12 | text: 13 | description: The text 14 | type: string 15 | uiOrder: 1 16 | title: Text 17 | required: 18 | - text 19 | - model 20 | title: Input 21 | type: object 22 | output: 23 | uiOrder: 0 24 | properties: 25 | embedding: 26 | $ref: schema.yaml#/$defs/instill-types/embedding 27 | uiOrder: 0 28 | title: Embedding 29 | required: 30 | - embedding 31 | title: Output 32 | type: object 33 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/operatorDef.yaml: -------------------------------------------------------------------------------- 1 | version: 1.0.0 2 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/base 3 | availableTasks: 4 | - TASK_MARSHAL 5 | custom: false 6 | documentationUrl: https://instill-ai.dev/docs/component/operator/json 7 | icon: Instill AI/json.svg 8 | iconUrl: '' 9 | id: json 10 | public: true 11 | spec: {} 12 | title: JSON 13 | uid: 28f53d15-6150-46e6-99aa-f76b70a926c0 14 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/operatorTasks.yaml: -------------------------------------------------------------------------------- 1 | TASK_MARSHAL: 2 | input: 3 | description: Input 4 | uiOrder: 0 5 | properties: 6 | object: 7 | description: Json object to be marshaled 8 | uiOrder: 0 9 | instillUpstreamTypes: 10 | - reference 11 | required: [] 12 | title: Object 13 | type: object 14 | required: 15 | - object 16 | title: Input 17 | type: object 18 | output: 19 | description: Output 20 | uiOrder: 0 21 | properties: 22 | string: 23 | description: Data 24 | uiOrder: 0 25 | title: Data 26 | type: string 27 | required: 28 | - string 29 | title: Output 30 | type: object 31 | -------------------------------------------------------------------------------- /pkg/component/base/testdata/test_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/base/testdata/test_image.png -------------------------------------------------------------------------------- /pkg/component/data/bigquery/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_INSERT 3 | - TASK_READ 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/data/bigquery 6 | icon: assets/bigquery.svg 7 | iconUrl: '' 8 | id: bigquery 9 | public: true 10 | title: BigQuery 11 | description: Insert data to BigQuery tables. 12 | tombstone: false 13 | type: COMPONENT_TYPE_DATA 14 | uid: e2ffe076-ab2c-4e5e-9587-a613a6b1c146 15 | vendor: Google 16 | vendorAttributes: {} 17 | version: 0.1.0 18 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/bigquery/v0 19 | releaseStage: RELEASE_STAGE_ALPHA 20 | -------------------------------------------------------------------------------- /pkg/component/data/bigquery/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | dataset-id: 4 | description: Fill in your BigQuery Dataset ID. 5 | type: string 6 | uiOrder: 2 7 | title: BigQuery Dataset ID 8 | json-key: 9 | description: Contents of the JSON key file with access to the bucket. 10 | type: string 11 | instillSecret: true 12 | uiOrder: 0 13 | title: JSON Key File contents 14 | project-id: 15 | description: Fill in your BigQuery Project ID. 16 | type: string 17 | uiOrder: 1 18 | title: BigQuery Project ID 19 | table-name: 20 | description: Fill in your BigQuery Table Name. 21 | type: string 22 | uiOrder: 3 23 | title: BigQuery Table Name 24 | required: 25 | - json-key 26 | - project-id 27 | - dataset-id 28 | - table-name 29 | title: BigQuery Connection 30 | type: object 31 | -------------------------------------------------------------------------------- /pkg/component/data/bigquery/v0/main_test.go: -------------------------------------------------------------------------------- 1 | // TODO: chuang8511, add test code 2 | // It will be done before 2024-06-26. 3 | package bigquery 4 | -------------------------------------------------------------------------------- /pkg/component/data/chroma/v0/client.go: -------------------------------------------------------------------------------- 1 | package chroma 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 8 | ) 9 | 10 | func newClient(setup *structpb.Struct, logger *zap.Logger) *httpclient.Client { 11 | c := httpclient.New("Chroma", getURL(setup), 12 | httpclient.WithLogger(logger), 13 | ) 14 | 15 | c.SetHeader("Authorization", "Bearer "+getAPIKey(setup)) 16 | c.SetHeader("Content-Type", "application/json") 17 | 18 | return c 19 | } 20 | 21 | func getURL(setup *structpb.Struct) string { 22 | return setup.GetFields()["url"].GetStringValue() 23 | } 24 | 25 | func getAPIKey(setup *structpb.Struct) string { 26 | return setup.GetFields()["api-key"].GetStringValue() 27 | } 28 | -------------------------------------------------------------------------------- /pkg/component/data/chroma/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_BATCH_UPSERT 3 | - TASK_UPSERT 4 | - TASK_QUERY 5 | - TASK_DELETE 6 | - TASK_CREATE_COLLECTION 7 | - TASK_DELETE_COLLECTION 8 | documentationUrl: https://instill-ai.dev/docs/component/data/chroma 9 | icon: assets/chroma.svg 10 | id: chroma 11 | public: true 12 | title: Chroma 13 | description: Build and search vector datasets. 14 | tombstone: false 15 | type: COMPONENT_TYPE_DATA 16 | uid: cb69cb22-c1e6-4ebd-aee5-6c1838429de7 17 | vendor: Chroma 18 | vendorAttributes: {} 19 | version: 0.1.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/chroma/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/data/chroma/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Chroma API key. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | url: 10 | description: Fill in your Chroma hosted public URL endpoint with port, e.g http://1.2.3:8000. 11 | type: string 12 | instillSecret: false 13 | uiOrder: 1 14 | title: Chroma URL Endpoint 15 | required: 16 | - api-key 17 | - url 18 | title: Chroma Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/chroma/v0/get_collection.go: -------------------------------------------------------------------------------- 1 | package chroma 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 7 | ) 8 | 9 | const ( 10 | getCollectionPath = "/api/v1/collections/%s" 11 | ) 12 | 13 | type GetCollectionResp struct { 14 | ID string `json:"id"` 15 | 16 | Detail []map[string]any `json:"detail"` 17 | } 18 | 19 | func getCollectionID(collectionName string, client *httpclient.Client) (string, error) { 20 | respGetColl := GetCollectionResp{} 21 | 22 | reqGetColl := client.R().SetResult(&respGetColl) 23 | 24 | resGetColl, err := reqGetColl.Get(fmt.Sprintf(getCollectionPath, collectionName)) 25 | 26 | if err != nil { 27 | return "", err 28 | } 29 | 30 | if resGetColl.StatusCode() != 200 { 31 | return "", fmt.Errorf("failed to get collection: %s", resGetColl.String()) 32 | } 33 | 34 | if respGetColl.Detail != nil { 35 | return "", fmt.Errorf("failed to get collection: %s", respGetColl.Detail[0]["msg"]) 36 | } 37 | 38 | return respGetColl.ID, nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/component/data/elasticsearch/v0/client.go: -------------------------------------------------------------------------------- 1 | package elasticsearch 2 | 3 | import ( 4 | "github.com/elastic/go-elasticsearch/v8" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | ) 7 | 8 | func newClient(setup *structpb.Struct) *ESClient { 9 | cfg := elasticsearch.Config{ 10 | CloudID: getCloudID(setup), 11 | APIKey: getAPIKey(setup), 12 | } 13 | 14 | es, _ := elasticsearch.NewClient(cfg) 15 | 16 | return &ESClient{ 17 | indexClient: es.Index, 18 | searchClient: es.Search, 19 | updateClient: es.UpdateByQuery, 20 | deleteClient: es.DeleteByQuery, 21 | createIndexClient: es.Indices.Create, 22 | deleteIndexClient: es.Indices.Delete, 23 | sqlTranslateClient: es.SQL.Translate, 24 | bulkClient: es.Bulk, 25 | } 26 | } 27 | 28 | // Need to confirm where the map is 29 | func getAPIKey(setup *structpb.Struct) string { 30 | return setup.GetFields()["api-key"].GetStringValue() 31 | } 32 | 33 | func getCloudID(setup *structpb.Struct) string { 34 | return setup.GetFields()["cloud-id"].GetStringValue() 35 | } 36 | -------------------------------------------------------------------------------- /pkg/component/data/elasticsearch/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_SEARCH 3 | - TASK_VECTOR_SEARCH 4 | - TASK_INDEX 5 | - TASK_MULTI_INDEX 6 | - TASK_UPDATE 7 | - TASK_DELETE 8 | - TASK_CREATE_INDEX 9 | - TASK_DELETE_INDEX 10 | documentationUrl: https://instill-ai.dev/docs/component/data/elasticsearch 11 | icon: assets/elasticsearch.svg 12 | id: elasticsearch 13 | public: true 14 | title: Elasticsearch 15 | description: Access the Elasticsearch database. 16 | tombstone: false 17 | type: COMPONENT_TYPE_DATA 18 | uid: f253a0c1-eb8e-45e1-a677-adb8895f5ceb 19 | vendor: Elastic 20 | version: 0.1.0 21 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/application/elasticsearch/v0 22 | releaseStage: RELEASE_STAGE_ALPHA 23 | -------------------------------------------------------------------------------- /pkg/component/data/elasticsearch/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in the API key for the Elasticsearch instance (please use encoded one). 5 | type: string 6 | instillSecret: true 7 | uiOrder: 1 8 | title: API Key 9 | cloud-id: 10 | description: Fill in the Cloud ID for the Elasticsearch instance. 11 | type: string 12 | instillSecret: true 13 | uiOrder: 0 14 | title: Cloud ID 15 | required: 16 | - api-key 17 | - cloud-id 18 | title: Elasticsearch Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/googlecloudstorage/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_UPLOAD 3 | - TASK_READ_OBJECTS 4 | - TASK_CREATE_BUCKET 5 | custom: false 6 | documentationUrl: https://instill-ai.dev/docs/component/data/gcs 7 | icon: assets/gcs.svg 8 | iconUrl: '' 9 | id: gcs 10 | public: true 11 | title: Google Cloud Storage 12 | description: Upload data to Google's Cloud Storage. 13 | tombstone: false 14 | type: COMPONENT_TYPE_DATA 15 | uid: 205cbeff-6f45-4abe-b0a8-cec1a310137f 16 | vendor: Google 17 | vendorAttributes: {} 18 | version: 0.1.0 19 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googlecloudstorage/v0 20 | releaseStage: RELEASE_STAGE_ALPHA 21 | -------------------------------------------------------------------------------- /pkg/component/data/googlecloudstorage/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | json-key: 4 | description: Contents of the JSON key file with access to the bucket. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 1 8 | title: JSON Key File contents 9 | required: 10 | - json-key 11 | title: Google Cloud Storage Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/data/googlecloudstorage/v0/main_test.go: -------------------------------------------------------------------------------- 1 | package googlecloudstorage 2 | 3 | // TODO: chuang8511: add test cases by mocking the GCS client 4 | // It will be done in 2024-06-26 5 | -------------------------------------------------------------------------------- /pkg/component/data/googledrive/v0/.compogen/intro.mdx: -------------------------------------------------------------------------------- 1 | **Note**: This component is only available on **🔮 Instill Core**. 2 | -------------------------------------------------------------------------------- /pkg/component/data/googledrive/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_READ_FILE 3 | - TASK_READ_FOLDER 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/data/google-drive 6 | icon: assets/google-drive.svg 7 | id: google-drive 8 | public: true 9 | title: Google Drive 10 | description: Connect to, and read files and folders within an existing Google Drive. 11 | tombstone: false 12 | type: COMPONENT_TYPE_DATA 13 | uid: cd220d2d-3d19-468e-8b95-37dd6a57c15f 14 | vendor: Google 15 | vendorAttributes: {} 16 | version: 0.1.0 17 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/googledrive/v0 18 | releaseStage: RELEASE_STAGE_ALPHA 19 | -------------------------------------------------------------------------------- /pkg/component/data/googledrive/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | properties: 2 | refresh-token: 3 | description: Refresh token for the Google Drive API. For more information about how to create tokens, please refer to the Google 4 | Drive API documentation and OAuth 2.0 documentation. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 1 8 | title: Refresh Token 9 | required: [] 10 | instillOAuthConfig: 11 | authUrl: https://accounts.google.com/o/oauth2/auth 12 | accessUrl: https://oauth2.googleapis.com/token 13 | scopes: 14 | - https://www.googleapis.com/auth/drive.readonly 15 | - https://www.googleapis.com/auth/drive.file 16 | - https://www.googleapis.com/auth/userinfo.email 17 | - https://www.googleapis.com/auth/userinfo.profile 18 | title: Google Drive Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/googledrive/v0/io.go: -------------------------------------------------------------------------------- 1 | package googledrive 2 | 3 | type readFileInput struct { 4 | SharedLink string `instill:"shared-link"` 5 | } 6 | 7 | type readFileOutput struct { 8 | File file `instill:"file"` 9 | } 10 | 11 | type file struct { 12 | ID string `instill:"id"` 13 | Name string `instill:"name"` 14 | Content string `instill:"content"` 15 | CreatedTime string `instill:"created-time"` 16 | ModifiedTime string `instill:"modified-time"` 17 | Size int64 `instill:"size"` 18 | MimeType string `instill:"mime-type"` 19 | Md5Checksum string `instill:"md5-checksum,omitempty"` 20 | Version int64 `instill:"version"` 21 | WebViewLink string `instill:"web-view-link"` 22 | WebContentLink string `instill:"web-content-link,omitempty"` 23 | } 24 | 25 | type readFolderInput struct { 26 | SharedLink string `instill:"shared-link"` 27 | ReadContent bool `instill:"read-content"` 28 | } 29 | 30 | type readFolderOutput struct { 31 | Files []*file `instill:"files"` 32 | } 33 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | properties: 2 | refresh-token: 3 | description: Refresh token for the Google Sheets API. For more information about how to create tokens, please refer to the Google 4 | Sheets API documentation and OAuth 2.0 documentation. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 1 8 | title: Refresh Token 9 | required: [] 10 | instillOAuthConfig: 11 | authUrl: https://accounts.google.com/o/oauth2/auth 12 | accessUrl: https://oauth2.googleapis.com/token 13 | scopes: 14 | - https://www.googleapis.com/auth/spreadsheets 15 | - https://www.googleapis.com/auth/drive.file 16 | - https://www.googleapis.com/auth/userinfo.email 17 | - https://www.googleapis.com/auth/userinfo.profile 18 | title: Google Drive Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_delete_row.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | ) 8 | 9 | func (e *execution) deleteRow(ctx context.Context, job *base.Job) error { 10 | input := &taskDeleteRowInput{} 11 | if err := job.Input.ReadData(ctx, input); err != nil { 12 | return err 13 | } 14 | 15 | err := e.deleteRowsHelper(ctx, input.SharedLink, input.SheetName, []int{input.RowNumber}) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | // TODO(huitang): reflect the real status 21 | output := &taskDeleteRowOutput{ 22 | Success: true, 23 | } 24 | if err := job.Output.WriteData(ctx, output); err != nil { 25 | return err 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_delete_spreadsheet.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | ) 8 | 9 | func (e *execution) deleteSpreadsheet(ctx context.Context, job *base.Job) error { 10 | input := &taskDeleteSpreadsheetInput{} 11 | if err := job.Input.ReadData(ctx, input); err != nil { 12 | return err 13 | } 14 | 15 | spreadsheetID, err := e.extractSpreadsheetID(input.SharedLink) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | // Delete the spreadsheet using Drive API 21 | err = e.driveService.Files.Delete(spreadsheetID).Context(ctx).Do() 22 | if err != nil { 23 | return err 24 | } 25 | 26 | output := &taskDeleteSpreadsheetOutput{ 27 | Success: true, 28 | } 29 | if err := job.Output.WriteData(ctx, output); err != nil { 30 | return err 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_get_row.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | ) 8 | 9 | func (e *execution) getRow(ctx context.Context, job *base.Job) error { 10 | input := &taskGetRowInput{} 11 | if err := job.Input.ReadData(ctx, input); err != nil { 12 | return err 13 | } 14 | 15 | rows, err := e.getRowsHelper(ctx, input.SharedLink, input.SheetName, []int{input.RowNumber}) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | output := &taskGetRowOutput{ 21 | Row: rows[0], 22 | } 23 | 24 | return job.Output.WriteData(ctx, output) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_insert_multiple_rows_test.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | // TODO: add tests 4 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_insert_row.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | "github.com/instill-ai/pipeline-backend/pkg/data/format" 8 | ) 9 | 10 | func (e *execution) insertRow(ctx context.Context, job *base.Job) error { 11 | input := &taskInsertRowInput{} 12 | if err := job.Input.ReadData(ctx, input); err != nil { 13 | return err 14 | } 15 | 16 | insertedRows, err := e.insertRowsHelper(ctx, input.SharedLink, input.SheetName, []map[string]format.Value{input.RowValue}) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | output := &taskInsertRowOutput{ 22 | Row: insertedRows[0], 23 | } 24 | if err := job.Output.WriteData(ctx, output); err != nil { 25 | return err 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_insert_row_test.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | // TODO: add tests 4 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_update_multiple_rows_test.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | // TODO: add tests 4 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_update_row.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | ) 8 | 9 | func (e *execution) updateRow(ctx context.Context, job *base.Job) error { 10 | input := &taskUpdateRowInput{} 11 | if err := job.Input.ReadData(ctx, input); err != nil { 12 | return err 13 | } 14 | 15 | updatedRows, err := e.updateRowsHelper(ctx, input.SharedLink, input.SheetName, []Row{input.Row}) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | output := &taskUpdateRowOutput{ 21 | Row: updatedRows[0], 22 | } 23 | if err := job.Output.WriteData(ctx, output); err != nil { 24 | return err 25 | } 26 | 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /pkg/component/data/googlesheets/v0/task_update_row_test.go: -------------------------------------------------------------------------------- 1 | package googlesheets 2 | 3 | // TODO: add tests 4 | -------------------------------------------------------------------------------- /pkg/component/data/instillartifact/v0/.compogen/extra-intro.mdx: -------------------------------------------------------------------------------- 1 | To use Artifact Component, you will need to set up the OpenAI API key for self-hosted deployment of Instill Core. 2 | You can do this by setting the `OPENAI_API_KEY` environment variable. 3 | Please refer to [configuring-the-embedding-feature](https://instill-ai.dev/docs/core/configuration#configuring-the-embedding-feature) 4 | p.s. In Instill Cloud case, you do not need to set up the OpenAI API key. 5 | -------------------------------------------------------------------------------- /pkg/component/data/instillartifact/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_UPLOAD_FILE 3 | - TASK_UPLOAD_FILES 4 | - TASK_GET_FILES_METADATA 5 | - TASK_GET_CHUNKS_METADATA 6 | - TASK_GET_FILE_IN_MARKDOWN 7 | - TASK_MATCH_FILE_STATUS 8 | - TASK_RETRIEVE 9 | - TASK_ASK 10 | documentationUrl: https://instill-ai.dev/docs/component/data/instill-artifact 11 | icon: assets/instill-artifact.svg 12 | id: instill-artifact 13 | public: true 14 | title: Instill Artifact 15 | description: Manipulate and smart search files and data in the artifact store. 16 | tombstone: false 17 | type: COMPONENT_TYPE_DATA 18 | uid: 6ec46048-f82f-4452-ba19-79698af9186e 19 | version: 0.1.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/instillartifact/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/data/instillartifact/v0/testdata/test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/data/instillartifact/v0/testdata/test.pdf -------------------------------------------------------------------------------- /pkg/component/data/instillartifact/v0/testdata/test_2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/data/instillartifact/v0/testdata/test_2.pdf -------------------------------------------------------------------------------- /pkg/component/data/instillartifact/v0/testdata/test_3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/data/instillartifact/v0/testdata/test_3.pdf -------------------------------------------------------------------------------- /pkg/component/data/milvus/v0/client.go: -------------------------------------------------------------------------------- 1 | package milvus 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 8 | ) 9 | 10 | func newClient(setup *structpb.Struct, logger *zap.Logger) *httpclient.Client { 11 | c := httpclient.New("Milvus", getURL(setup), 12 | httpclient.WithLogger(logger), 13 | ) 14 | 15 | c.SetHeader("Authorization", "Bearer "+getUsername(setup)+":"+getPassword(setup)) 16 | c.SetHeader("Content-Type", "application/json") 17 | 18 | return c 19 | } 20 | 21 | func getURL(setup *structpb.Struct) string { 22 | return setup.GetFields()["url"].GetStringValue() 23 | } 24 | 25 | func getUsername(setup *structpb.Struct) string { 26 | return setup.GetFields()["username"].GetStringValue() 27 | } 28 | 29 | func getPassword(setup *structpb.Struct) string { 30 | return setup.GetFields()["password"].GetStringValue() 31 | } 32 | -------------------------------------------------------------------------------- /pkg/component/data/milvus/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_VECTOR_SEARCH 3 | - TASK_UPSERT 4 | - TASK_BATCH_UPSERT 5 | - TASK_DELETE 6 | - TASK_CREATE_COLLECTION 7 | - TASK_DROP_COLLECTION 8 | - TASK_CREATE_PARTITION 9 | - TASK_DROP_PARTITION 10 | - TASK_CREATE_INDEX 11 | - TASK_DROP_INDEX 12 | documentationUrl: https://instill-ai.dev/docs/component/data/milvus 13 | icon: assets/milvus.svg 14 | id: milvus 15 | public: true 16 | title: Milvus 17 | description: Build and search vector datasets. 18 | tombstone: false 19 | type: COMPONENT_TYPE_DATA 20 | uid: 51a5246e-2f2c-4597-bbca-4baaa0dc8994 21 | vendor: Milvus 22 | vendorAttributes: {} 23 | version: 0.1.0 24 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/milvus/v0 25 | releaseStage: RELEASE_STAGE_ALPHA 26 | -------------------------------------------------------------------------------- /pkg/component/data/milvus/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | url: 4 | description: Fill in your Milvus public URL endpoint with port number, e.g http://3.25.202.142:19530. 5 | type: string 6 | instillSecret: false 7 | uiOrder: 0 8 | title: Milvus URL Endpoint 9 | username: 10 | description: Fill in your Milvus username. 11 | type: string 12 | instillSecret: false 13 | uiOrder: 1 14 | title: Milvus Username 15 | password: 16 | description: Fill in your Milvus password. 17 | type: string 18 | instillSecret: true 19 | uiOrder: 2 20 | title: Milvus Password 21 | required: 22 | - url 23 | - username 24 | - password 25 | title: Milvus Connection 26 | type: object 27 | -------------------------------------------------------------------------------- /pkg/component/data/mongodb/v0/assets/mongodb.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/data/mongodb/v0/client.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "context" 5 | 6 | "go.mongodb.org/mongo-driver/mongo" 7 | "go.mongodb.org/mongo-driver/mongo/options" 8 | "google.golang.org/protobuf/types/known/structpb" 9 | ) 10 | 11 | func newClient(ctx context.Context, setup *structpb.Struct) *mongo.Client { 12 | client, _ := mongo.Connect(ctx, options.Client().ApplyURI(getURI(setup))) 13 | 14 | return client 15 | } 16 | 17 | func getURI(setup *structpb.Struct) string { 18 | return setup.GetFields()["uri"].GetStringValue() 19 | } 20 | -------------------------------------------------------------------------------- /pkg/component/data/mongodb/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_INSERT 3 | - TASK_INSERT_MANY 4 | - TASK_FIND 5 | - TASK_UPDATE 6 | - TASK_DELETE 7 | - TASK_DROP_COLLECTION 8 | - TASK_DROP_DATABASE 9 | - TASK_CREATE_SEARCH_INDEX 10 | - TASK_DROP_SEARCH_INDEX 11 | - TASK_VECTOR_SEARCH 12 | documentationUrl: https://instill-ai.dev/docs/component/data/mongodb 13 | icon: assets/mongodb.svg 14 | id: mongodb 15 | public: true 16 | title: MongoDB 17 | description: Access the MongoDB NoSQL database. 18 | tombstone: false 19 | type: COMPONENT_TYPE_DATA 20 | uid: 1547711c-864b-4983-b0ca-cb79eafa2f7f 21 | vendor: MongoDB 22 | vendorAttributes: {} 23 | version: 0.1.0 24 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/mongodb/v0 25 | releaseStage: RELEASE_STAGE_ALPHA 26 | -------------------------------------------------------------------------------- /pkg/component/data/mongodb/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | uri: 4 | description: Fill in your MongoDB URI. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: URI 9 | required: 10 | - uri 11 | title: MongoDB Connection 12 | type: object 13 | -------------------------------------------------------------------------------- /pkg/component/data/pinecone/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_QUERY 3 | - TASK_UPSERT 4 | - TASK_BATCH_UPSERT 5 | - TASK_RERANK 6 | custom: false 7 | documentationUrl: https://instill-ai.dev/docs/component/data/pinecone 8 | icon: assets/pinecone.svg 9 | iconUrl: https://www.pinecone.io/favicon.ico 10 | id: pinecone 11 | public: true 12 | title: Pinecone 13 | description: Build and search vector datasets. 14 | tombstone: false 15 | type: COMPONENT_TYPE_DATA 16 | uid: 4b1dcf82-e134-4ba7-992f-f9a02536ec2b 17 | vendor: Pinecone 18 | vendorAttributes: {} 19 | version: 0.1.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/pinecone/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/data/pinecone/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Pinecone AI API key. You can create an api key in Pinecone Console. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | url: 10 | description: Fill in your Pinecone index URL. It is in the form. 11 | type: string 12 | instillSecret: false 13 | uiOrder: 1 14 | title: Pinecone Index URL 15 | required: 16 | - api-key 17 | title: Pinecone Connection 18 | type: object 19 | -------------------------------------------------------------------------------- /pkg/component/data/pinecone/v0/task_query.go: -------------------------------------------------------------------------------- 1 | // TODO: Implement the query task, it will be addressed in ins-7102 2 | package pinecone 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (e *execution) query(ctx context.Context, job *base.Job) error { 11 | return nil 12 | } 13 | -------------------------------------------------------------------------------- /pkg/component/data/pinecone/v0/task_rerank.go: -------------------------------------------------------------------------------- 1 | // TODO: Implement the rerank task, it will be addressed in ins-7102 2 | package pinecone 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 8 | ) 9 | 10 | func (e *execution) rerank(ctx context.Context, job *base.Job) error { 11 | return nil 12 | } 13 | -------------------------------------------------------------------------------- /pkg/component/data/qdrant/v0/client.go: -------------------------------------------------------------------------------- 1 | package qdrant 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 8 | ) 9 | 10 | func newClient(setup *structpb.Struct, logger *zap.Logger) *httpclient.Client { 11 | c := httpclient.New("Qdrant", getURL(setup), 12 | httpclient.WithLogger(logger), 13 | ) 14 | 15 | c.SetHeader("api-key", getAPIKey(setup)) 16 | c.SetHeader("Content-Type", "application/json") 17 | 18 | return c 19 | } 20 | 21 | func getURL(setup *structpb.Struct) string { 22 | return setup.GetFields()["url"].GetStringValue() 23 | } 24 | 25 | func getAPIKey(setup *structpb.Struct) string { 26 | return setup.GetFields()["api-key"].GetStringValue() 27 | } 28 | -------------------------------------------------------------------------------- /pkg/component/data/qdrant/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_VECTOR_SEARCH 3 | - TASK_BATCH_UPSERT 4 | - TASK_UPSERT 5 | - TASK_DELETE 6 | - TASK_CREATE_COLLECTION 7 | - TASK_DELETE_COLLECTION 8 | documentationUrl: https://instill-ai.dev/docs/component/data/qdrant 9 | icon: assets/qdrant.svg 10 | id: qdrant 11 | public: true 12 | title: Qdrant 13 | description: Build and search vector datasets. 14 | tombstone: false 15 | type: COMPONENT_TYPE_DATA 16 | uid: 628c91b8-1cf0-4141-b9e4-256b2ed109f2 17 | vendor: Qdrant 18 | vendorAttributes: {} 19 | version: 0.1.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/qdrant/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/data/qdrant/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Qdrant API key. Please refer to clusters in Qdrant data access control. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | url: 10 | description: Fill in your Qdrant URL endpoint. Please refer to clusters in Qdrant cluster details. 11 | type: string 12 | instillSecret: false 13 | uiOrder: 1 14 | title: Qdrant URL Endpoint 15 | required: 16 | - api-key 17 | - url 18 | title: Qdrant Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/redis/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_RETRIEVE_CHAT_HISTORY 3 | - TASK_WRITE_CHAT_MESSAGE 4 | - TASK_WRITE_MULTI_MODAL_CHAT_MESSAGE 5 | custom: false 6 | documentationUrl: https://instill-ai.dev/docs/component/data/redis 7 | icon: assets/redis.svg 8 | iconUrl: '' 9 | id: redis 10 | public: true 11 | title: Redis 12 | description: Manage data in NoSQL Redis databases. 13 | type: COMPONENT_TYPE_DATA 14 | uid: fd0ad325-f2f7-41f3-b247-6c71d571b1b8 15 | vendor: Redis Labs 16 | vendorAttributes: {} 17 | version: 0.1.1 18 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/redis/v0 19 | releaseStage: RELEASE_STAGE_ALPHA 20 | -------------------------------------------------------------------------------- /pkg/component/data/sql/v0/assets/sql.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/data/sql/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_INSERT 3 | - TASK_INSERT_MANY 4 | - TASK_UPDATE 5 | - TASK_SELECT 6 | - TASK_DELETE 7 | - TASK_CREATE_TABLE 8 | - TASK_DROP_TABLE 9 | documentationUrl: https://instill-ai.dev/docs/component/data/sql 10 | icon: assets/sql.svg 11 | id: sql 12 | public: true 13 | title: SQL 14 | description: Access the SQL database of your choice. 15 | tombstone: false 16 | type: COMPONENT_TYPE_DATA 17 | uid: 5861fc8f-1a07-42f6-a6b8-0e5a2664de00 18 | version: 0.1.0 19 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/sql/v0 20 | releaseStage: RELEASE_STAGE_ALPHA 21 | -------------------------------------------------------------------------------- /pkg/component/data/weaviate/v0/client.go: -------------------------------------------------------------------------------- 1 | package weaviate 2 | 3 | import ( 4 | "github.com/weaviate/weaviate-go-client/v4/weaviate" 5 | "github.com/weaviate/weaviate-go-client/v4/weaviate/auth" 6 | "google.golang.org/protobuf/types/known/structpb" 7 | ) 8 | 9 | func newClient(setup *structpb.Struct) *weaviate.Client { 10 | cfg := weaviate.Config{ 11 | Host: getURL(setup), 12 | Scheme: "https", 13 | AuthConfig: auth.ApiKey{Value: getAPIKey(setup)}, 14 | Headers: nil, 15 | } 16 | 17 | client, err := weaviate.NewClient(cfg) 18 | if err != nil { 19 | return nil 20 | } 21 | 22 | return client 23 | } 24 | 25 | func getURL(setup *structpb.Struct) string { 26 | return setup.GetFields()["url"].GetStringValue() 27 | } 28 | 29 | func getAPIKey(setup *structpb.Struct) string { 30 | return setup.GetFields()["api-key"].GetStringValue() 31 | } 32 | -------------------------------------------------------------------------------- /pkg/component/data/weaviate/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_VECTOR_SEARCH 3 | - TASK_BATCH_INSERT 4 | - TASK_INSERT 5 | - TASK_UPDATE 6 | - TASK_DELETE 7 | - TASK_DELETE_COLLECTION 8 | documentationUrl: https://instill-ai.dev/docs/component/data/weaviate 9 | icon: assets/weaviate.svg 10 | id: weaviate 11 | public: true 12 | title: Weaviate 13 | description: Build and search vector datasets. 14 | tombstone: false 15 | type: COMPONENT_TYPE_DATA 16 | uid: 8833d994-ab21-4627-910f-6612ae5526c0 17 | vendor: Weaviate 18 | vendorAttributes: {} 19 | version: 0.1.0 20 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/weaviate/v0 21 | releaseStage: RELEASE_STAGE_ALPHA 22 | -------------------------------------------------------------------------------- /pkg/component/data/weaviate/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | api-key: 4 | description: Fill in your Weaviate API key. Please refer to clusters in Weaviate Console. 5 | type: string 6 | instillSecret: true 7 | uiOrder: 0 8 | title: API Key 9 | url: 10 | description: Fill in your Weaviate base URL without https:// or http://. Please refer to clusters in Weaviate Console. 11 | type: string 12 | instillSecret: false 13 | uiOrder: 1 14 | title: Weaviate URL Endpoint 15 | required: 16 | - api-key 17 | - url 18 | title: Weaviate Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/data/zilliz/v0/client.go: -------------------------------------------------------------------------------- 1 | package zilliz 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 8 | ) 9 | 10 | func newClient(setup *structpb.Struct, logger *zap.Logger) *httpclient.Client { 11 | c := httpclient.New("Zilliz", getURL(setup), 12 | httpclient.WithLogger(logger), 13 | ) 14 | 15 | c.SetHeader("Authorization", "Bearer "+getAPIKey(setup)) 16 | c.SetHeader("Content-Type", "application/json") 17 | 18 | return c 19 | } 20 | 21 | func getURL(setup *structpb.Struct) string { 22 | return setup.GetFields()["url"].GetStringValue() 23 | } 24 | 25 | func getAPIKey(setup *structpb.Struct) string { 26 | return setup.GetFields()["api-key"].GetStringValue() 27 | } 28 | -------------------------------------------------------------------------------- /pkg/component/data/zilliz/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_VECTOR_SEARCH 3 | - TASK_UPSERT 4 | - TASK_BATCH_UPSERT 5 | - TASK_DELETE 6 | - TASK_CREATE_COLLECTION 7 | - TASK_DROP_COLLECTION 8 | - TASK_CREATE_PARTITION 9 | - TASK_DROP_PARTITION 10 | documentationUrl: https://instill-ai.dev/docs/component/data/zilliz 11 | icon: assets/zilliz.svg 12 | id: zilliz 13 | public: true 14 | title: Zilliz 15 | description: Build and search vector datasets. 16 | tombstone: false 17 | type: COMPONENT_TYPE_DATA 18 | uid: 7995e58f-de2c-4754-99d9-0876008faece 19 | vendor: Zilliz 20 | vendorAttributes: {} 21 | version: 0.1.0 22 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/data/zilliz/v0 23 | releaseStage: RELEASE_STAGE_ALPHA 24 | -------------------------------------------------------------------------------- /pkg/component/data/zilliz/v0/config/setup.yaml: -------------------------------------------------------------------------------- 1 | additionalProperties: false 2 | properties: 3 | url: 4 | description: Fill in your Zilliz public URL endpoint. 5 | type: string 6 | instillSecret: false 7 | uiOrder: 0 8 | title: Zilliz URL Endpoint 9 | api-key: 10 | description: Fill in your Zilliz API key. 11 | type: string 12 | instillSecret: true 13 | uiOrder: 1 14 | title: Zilliz API Key 15 | required: 16 | - url 17 | - api-key 18 | title: Zilliz Connection 19 | type: object 20 | -------------------------------------------------------------------------------- /pkg/component/generic/collection/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_APPEND 3 | - TASK_ASSIGN 4 | - TASK_CONCAT 5 | - TASK_DIFFERENCE 6 | - TASK_INTERSECTION 7 | - TASK_SPLIT 8 | - TASK_SYMMETRIC_DIFFERENCE 9 | - TASK_UNION 10 | custom: false 11 | documentationUrl: https://instill-ai.dev/docs/component/generic/collection 12 | icon: assets/collection.svg 13 | iconUrl: '' 14 | id: collection 15 | public: true 16 | spec: {} 17 | title: Collection 18 | type: COMPONENT_TYPE_GENERIC 19 | uid: eb611e31-fbe6-43ad-8671-5b9a2e351638 20 | version: 0.1.0 21 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/collection/v0 22 | description: Manipulate collection-type data. 23 | releaseStage: RELEASE_STAGE_ALPHA 24 | -------------------------------------------------------------------------------- /pkg/component/generic/http/v0/assets/http.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/generic/http/v0/client.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "google.golang.org/protobuf/types/known/structpb" 6 | 7 | "github.com/instill-ai/pipeline-backend/pkg/component/internal/util/httpclient" 8 | ) 9 | 10 | func newClient(setup *structpb.Struct, logger *zap.Logger) (*httpclient.Client, error) { 11 | c := httpclient.New("HTTP API", "", 12 | httpclient.WithLogger(logger), 13 | ) 14 | 15 | auth, err := getAuthentication(setup) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | if err := auth.setAuthInClient(c); err != nil { 21 | return nil, err 22 | } 23 | 24 | return c, nil 25 | } 26 | -------------------------------------------------------------------------------- /pkg/component/generic/http/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_GET 3 | - TASK_POST 4 | - TASK_PATCH 5 | - TASK_PUT 6 | - TASK_DELETE 7 | - TASK_HEAD 8 | - TASK_OPTIONS 9 | custom: false 10 | documentationUrl: https://instill-ai.dev/docs/component/generic/http 11 | icon: assets/http.svg 12 | iconUrl: '' 13 | id: http 14 | public: true 15 | title: HTTP 16 | description: Make requests to external HTTP APIs. 17 | tombstone: false 18 | type: COMPONENT_TYPE_GENERIC 19 | uid: 5ee55a5c-6e30-4c7a-80e8-90165a729e0a 20 | vendor: '' 21 | vendorAttributes: {} 22 | version: 0.2.0 23 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/http/v0 24 | releaseStage: RELEASE_STAGE_ALPHA 25 | -------------------------------------------------------------------------------- /pkg/component/generic/http/v0/io.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import "github.com/instill-ai/pipeline-backend/pkg/data/format" 4 | 5 | type httpInput struct { 6 | EndpointURL string `instill:"endpoint-url"` 7 | Header map[string][]string `instill:"header"` 8 | Body format.Value `instill:"body"` 9 | } 10 | 11 | type httpOutput struct { 12 | Header map[string][]string `instill:"header"` 13 | Body format.Value `instill:"body"` 14 | StatusCode int `instill:"status-code"` 15 | } 16 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/.compogen/bottom.mdx: -------------------------------------------------------------------------------- 1 | ## Supported Events 2 | 3 | ### Cron Job Triggered Event 4 | 5 | An event triggered on a cron job. 6 | 7 |
8 | 9 | | Configuration | ID | Type | Description | 10 | | :--- | :--- | :--- | :--- | 11 | | Cron Expression | `cron` | string | A cron expression in standard 5-field format (minute hour day-of-month month day-of-week). | 12 |
13 | 14 |
15 | 16 | | Event Message | ID | Type | Description | 17 | | :--- | :--- | :--- | :--- | 18 | | Triggered At | `triggered-at` | string | The ISO 8601 timestamp when the cron job was triggered. | 19 |
20 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/assets/scheduler.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: [] 2 | availableEvents: 3 | - EVENT_CRON_JOB_TRIGGERED 4 | documentationUrl: https://instill-ai.dev/docs/component/generic/scheduler 5 | icon: assets/scheduler.svg 6 | id: scheduler 7 | public: true 8 | title: Scheduler 9 | vendor: Generic 10 | description: Trigger pipelines on a scheduler using cron expressions. 11 | tombstone: false 12 | type: COMPONENT_TYPE_GENERIC 13 | uid: f2a89c6d-e315-4b7a-9d8f-c83e45a92c10 14 | version: 0.1.0 15 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/generic/scheduler/v0 16 | releaseStage: RELEASE_STAGE_ALPHA 17 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/config/events.yaml: -------------------------------------------------------------------------------- 1 | EVENT_CRON_JOB_TRIGGERED: 2 | title: Cron Job Triggered 3 | description: An event triggered on a cron job 4 | configSchema: 5 | required: 6 | - cron 7 | properties: 8 | cron: 9 | pattern: ^[0-9*,-/]+\s+[0-9*,-/]+\s+[0-9*,-/]+\s+[0-9*,-/]+\s+[0-9*,-/]+$ 10 | type: string 11 | type: object 12 | messageSchema: 13 | $schema: http://json-schema.org/draft-07/schema 14 | required: 15 | - triggered-at 16 | properties: 17 | triggered-at: 18 | description: The ISO 8601 timestamp when the cron job was triggered 19 | type: string 20 | additionalProperties: false 21 | title: Cron job triggered event 22 | type: object 23 | messageExamples: 24 | - triggered-at: '2024-01-01T00:00:00.000Z' 25 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/config/tasks.yaml: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/event_cron_job_triggered.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | "github.com/instill-ai/pipeline-backend/pkg/data" 8 | ) 9 | 10 | func (c *component) handleEventCronJobTriggered(ctx context.Context, rawEvent *base.RawEvent) (parsedEvent *base.ParsedEvent, err error) { 11 | 12 | return &base.ParsedEvent{ 13 | ParsedMessage: rawEvent.Message, 14 | Response: data.Map{}, 15 | }, nil 16 | } 17 | -------------------------------------------------------------------------------- /pkg/component/generic/scheduler/v0/event_structs.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | type EventCronJobTriggered struct { 4 | Cron string `instill:"cron"` 5 | } 6 | 7 | type EventCronJobTriggeredMessage struct { 8 | UID string `instill:"uid"` 9 | EventID string `instill:"eventID"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/component/internal/jsonref/interface.go: -------------------------------------------------------------------------------- 1 | // The following code is based on lestrrat's work, available at https://github.com/lestrrat-go/jsref. 2 | 3 | package jsonref 4 | 5 | import ( 6 | "errors" 7 | "net/url" 8 | "reflect" 9 | ) 10 | 11 | var zeroval = reflect.Value{} 12 | 13 | var ErrMaxRecursion = errors.New("reached max number of recursions") 14 | var ErrReferenceLoop = errors.New("reference loop detected") 15 | 16 | // Resolver is responsible for interpreting the provided JSON 17 | // reference. 18 | type Resolver struct { 19 | providers []Provider 20 | MaxRecursions int 21 | } 22 | 23 | // Provider resolves a URL into a ... thing. 24 | type Provider interface { 25 | Get(*url.URL) (interface{}, error) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/component/internal/jsonref/options.go: -------------------------------------------------------------------------------- 1 | // The following code is based on lestrrat's work, available at https://github.com/lestrrat-go/jsref. 2 | 3 | package jsonref 4 | 5 | import "github.com/lestrrat-go/option" 6 | 7 | type Option = option.Interface 8 | 9 | type identRecursiveResolution struct{} 10 | 11 | // WithRecursiveResolution allows you to enable recursive resolution 12 | // on the *result* data structure. This means that after resolving 13 | // the JSON reference in the structure at hand, it does another 14 | // pass at resolving the entire data structure. Depending on your 15 | // structure and size, this may incur significant cost. 16 | // 17 | // Please note that recursive resolution of the result is still 18 | // experimental. If you find problems, please submit a pull request 19 | // with a failing test case. 20 | func WithRecursiveResolution(b bool) Option { 21 | return option.New(identRecursiveResolution{}, b) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/component/internal/mock/README.md: -------------------------------------------------------------------------------- 1 | # Component mocks 2 | 3 | This directory holds the mocks required for the tests in the `component` 4 | package. Tests are defined in a dedicated directory (in opposition to being 5 | defined under `pipeline-backend/pkg/mock`) to avoid dependency cycles. 6 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/.compogen/bottom.mdx: -------------------------------------------------------------------------------- 1 | ## Example Recipes 2 | 3 | ```yaml 4 | version: v1beta 5 | component: 6 | audio-vad: 7 | type: audio 8 | input: 9 | audio: ${variable.audio} 10 | min-silence-duration: 300 11 | speech-pad: 10 12 | task: TASK_DETECT_ACTIVITY 13 | audio-segment: 14 | type: audio 15 | input: 16 | audio: ${variable.audio} 17 | segments: ${audio-vad.output.segments} 18 | task: TASK_SEGMENT 19 | variable: 20 | audio: 21 | title: Audio to test 22 | description: Audio to test VAD and extraction 23 | type: audio 24 | output: 25 | samples: 26 | title: Output audio segments 27 | description: Output extracted audio segments 28 | value: ${audio-segment.output.audio-segments} 29 | ``` 30 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/assets/audio.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_DETECT_ACTIVITY 3 | - TASK_SEGMENT 4 | documentationUrl: https://instill-ai.dev/docs/component/operator/audio 5 | icon: assets/audio.svg 6 | id: audio 7 | public: true 8 | spec: {} 9 | title: Audio 10 | type: COMPONENT_TYPE_OPERATOR 11 | uid: b5c75caa-9261-4757-bfbf-12e908f59f16 12 | version: 0.1.0 13 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/audio/v0 14 | description: Operate audio data. 15 | releaseStage: RELEASE_STAGE_ALPHA 16 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/io.go: -------------------------------------------------------------------------------- 1 | package audio 2 | 3 | import ( 4 | "github.com/instill-ai/pipeline-backend/pkg/data/format" 5 | ) 6 | 7 | type segmentData struct { 8 | StartTime float64 `instill:"start-time"` 9 | EndTime float64 `instill:"end-time"` 10 | } 11 | 12 | type detectActivityInput struct { 13 | Audio format.Audio `instill:"audio"` 14 | MinSilenceDuration int `instill:"min-silence-duration,default=100"` 15 | SpeechPad int `instill:"speech-pad,default=30"` 16 | } 17 | 18 | type detectActivityOutput struct { 19 | Segments []segmentData `instill:"segments"` 20 | } 21 | 22 | type segmentInput struct { 23 | Audio format.Audio `instill:"audio"` 24 | Segments []segmentData `instill:"segments"` 25 | } 26 | 27 | type segmentOutput struct { 28 | AudioSegments []format.Audio `instill:"audio-segments"` 29 | } 30 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/task_detect_activity_nontag.go: -------------------------------------------------------------------------------- 1 | //go:build !onnx 2 | // +build !onnx 3 | 4 | package audio 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | 10 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 11 | ) 12 | 13 | func detectActivity(ctx context.Context, job *base.Job) error { 14 | return fmt.Errorf("the Audio operator wasn't built with onnxruntime") 15 | } 16 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/testdata/voice1-activity-segments.json: -------------------------------------------------------------------------------- 1 | { 2 | "segments": [ 3 | { 4 | "start-time": 1.5, 5 | "end-time": 2.404 6 | }, 7 | { 8 | "start-time": 3.196, 9 | "end-time": 4.068 10 | }, 11 | { 12 | "start-time": 4.604, 13 | "end-time": 5.764 14 | }, 15 | { 16 | "start-time": 6.62, 17 | "end-time": 14.948 18 | }, 19 | { 20 | "start-time": 15.836, 21 | "end-time": 18.564 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/testdata/voice1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/audio/v0/testdata/voice1.wav -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/testdata/voice2-activity-segments.json: -------------------------------------------------------------------------------- 1 | { 2 | "segments": [ 3 | { 4 | "start-time": 0.002, 5 | "end-time": 9.406 6 | }, 7 | { 8 | "start-time": 10.146, 9 | "end-time": 18.782 10 | }, 11 | { 12 | "start-time": 19.234, 13 | "end-time": 30.878 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /pkg/component/operator/audio/v0/testdata/voice2.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/audio/v0/testdata/voice2.wav -------------------------------------------------------------------------------- /pkg/component/operator/base64/v0/assets/base64.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /pkg/component/operator/base64/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_ENCODE 3 | - TASK_DECODE 4 | custom: false 5 | documentationUrl: https://instill-ai.dev/docs/component/operator/base64 6 | icon: assets/base64.svg 7 | iconUrl: '' 8 | id: base64 9 | public: true 10 | spec: {} 11 | title: Base64 12 | type: COMPONENT_TYPE_OPERATOR 13 | tombstone: false 14 | uid: 3a836447-c211-4134-9cc5-ad45e1cc467e 15 | version: 0.1.0 16 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/base64/v0 17 | description: Encode or decode a string in Base64 format. 18 | releaseStage: RELEASE_STAGE_ALPHA 19 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/assets/document.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CONVERT_TO_MARKDOWN 3 | - TASK_CONVERT_TO_TEXT 4 | - TASK_CONVERT_TO_IMAGES 5 | custom: false 6 | documentationUrl: https://instill-ai.dev/docs/component/operator/document 7 | icon: assets/document.svg 8 | id: document 9 | public: true 10 | spec: {} 11 | title: Document 12 | type: COMPONENT_TYPE_OPERATOR 13 | uid: e5b290ae-ad53-47c9-a64e-efbc5358520b 14 | version: 0.1.3 15 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/document/v0 16 | description: Manipulate Document files. 17 | releaseStage: RELEASE_STAGE_ALPHA 18 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.csv: -------------------------------------------------------------------------------- 1 | test,test,tse 2 | 1,23,2 3 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.docx -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.html: -------------------------------------------------------------------------------- 1 | 2 |

This is test file

3 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.jpg -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Sample Markdown Document" 3 | author: "John Doe" 4 | date: "2023-05-31" 5 | description: "This is a sample Markdown document with YAML front matter." 6 | --- 7 | # Introduction 8 | This is a sample Markdown file to demonstrate extracting metadata. 9 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.odt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.odt -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.pdf -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.png -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.pptx -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.tif -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.xls -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/testdata/test.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/document/v0/testdata/test.xlsx -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/transformer/const.go: -------------------------------------------------------------------------------- 1 | package transformer 2 | 3 | import ( 4 | _ "embed" 5 | ) 6 | 7 | const ( 8 | pythonInterpreter string = "/opt/venv/bin/python" 9 | ) 10 | 11 | var ( 12 | //go:embed pdf_to_markdown/pdf_transformer.py 13 | pdfTransformer string 14 | //go:embed pdf_to_markdown/page_image_processor.py 15 | pageImageProcessor string 16 | 17 | //go:embed execution/docling_pdf_to_md_converter.py 18 | doclingPDFToMDConverter string 19 | 20 | //go:embed execution/pdfplumber_pdf_to_md_converter.py 21 | pdfPlumberPDFToMDConverter string 22 | 23 | //go:embed execution/image_converter.py 24 | imageConverter string 25 | 26 | //go:embed execution/pdf_checker.py 27 | pdfChecker string 28 | 29 | //go:embed execution/get_page_numbers.py 30 | getPageNumbersExecution string 31 | ) 32 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/transformer/execution/get_page_numbers.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | import json 3 | import sys 4 | import base64 5 | import pdfplumber 6 | 7 | if __name__ == "__main__": 8 | try: 9 | json_str = sys.stdin.buffer.read().decode('utf-8') 10 | params = json.loads(json_str) 11 | pdf_string = params["PDF"] 12 | decoded_bytes = base64.b64decode(pdf_string) 13 | pdf_file_obj = BytesIO(decoded_bytes) 14 | 15 | pdf = pdfplumber.open(pdf_file_obj) 16 | 17 | output = { 18 | "page_numbers": len(pdf.pages) 19 | } 20 | 21 | print(json.dumps(output)) 22 | except Exception as e: 23 | print(json.dumps({"error": str(e)})) 24 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/transformer/execution/pdf_checker.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | import json 3 | import base64 4 | import sys 5 | 6 | # TODO chuang8511: 7 | # Deal with the import error when running the code in the docker container. 8 | # Now, we combine all python code into one file to avoid the import error. 9 | # from pdf_to_markdown import PDFTransformer 10 | 11 | if __name__ == "__main__": 12 | json_str = sys.stdin.buffer.read().decode('utf-8') 13 | params = json.loads(json_str) 14 | pdf_string = params["PDF"] 15 | 16 | decoded_bytes = base64.b64decode(pdf_string) 17 | pdf_file_obj = BytesIO(decoded_bytes) 18 | pdf = PDFTransformer(x=pdf_file_obj) 19 | pages = pdf.raw_pages 20 | output = { 21 | "required": len(pages) == 0, 22 | } 23 | print(json.dumps(output)) 24 | -------------------------------------------------------------------------------- /pkg/component/operator/document/v0/transformer/pdf_to_markdown/__init__.py: -------------------------------------------------------------------------------- 1 | from page_image_processor import PageImageProcessor 2 | from pdf_transformer import PDFTransformer 3 | -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/assets/image.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CONCAT 3 | - TASK_CROP 4 | - TASK_RESIZE 5 | - TASK_DRAW_CLASSIFICATION 6 | - TASK_DRAW_DETECTION 7 | - TASK_DRAW_KEYPOINT 8 | - TASK_DRAW_OCR 9 | - TASK_DRAW_INSTANCE_SEGMENTATION 10 | - TASK_DRAW_SEMANTIC_SEGMENTATION 11 | custom: false 12 | documentationUrl: https://instill-ai.dev/docs/component/operator/image 13 | icon: assets/image.svg 14 | iconUrl: '' 15 | id: image 16 | public: true 17 | spec: {} 18 | title: Image 19 | type: COMPONENT_TYPE_OPERATOR 20 | uid: e9eb8fc8-f249-4e11-ad50-5035d79ffc18 21 | version: 0.1.0 22 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/image/v0 23 | description: Manipulate image files. 24 | releaseStage: RELEASE_STAGE_ALPHA 25 | -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/task_draw_classification.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/component/base" 7 | ) 8 | 9 | func drawClassification(ctx context.Context, job *base.Job) error { 10 | 11 | var inputStruct drawClassificationInput 12 | if err := job.Input.ReadData(ctx, &inputStruct); err != nil { 13 | return err 14 | } 15 | 16 | img, err := decodeImage(inputStruct.Image) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | imgRGBA := convertToRGBA(img) 22 | 23 | if err := drawImageLabel(imgRGBA, inputStruct.Category, inputStruct.Score, inputStruct.ShowScore); err != nil { 24 | return err 25 | } 26 | 27 | imgData, err := encodeImage(imgRGBA) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | outputData := drawClassificationOutput{ 33 | Image: imgData, 34 | } 35 | 36 | if err := job.Output.WriteData(ctx, outputData); err != nil { 37 | return err 38 | } 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/cls-dog.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/cls-dog.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/cls-dog.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "golden retriever", 3 | "score": 0.898938 4 | } 5 | -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/det-coco-1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/det-coco-1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/det-coco-2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/det-coco-2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/inst-seg-coco-1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/inst-seg-coco-1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/inst-seg-coco-2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/inst-seg-coco-2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/inst-seg-stomata.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/inst-seg-stomata.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/kp-coco-1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/kp-coco-1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/kp-coco-2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/kp-coco-2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/ocr-mm.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/ocr-mm.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/sem-seg-cityscape.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/sem-seg-cityscape.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_1x4_grid_without_padding.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_1x4_grid_without_padding.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_2x2_grid_with_padding.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_2x2_grid_with_padding.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_default_square_grid.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_concat_testconcat_default_square_grid.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_circular_crop.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_circular_crop.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_corner_radius_crop.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_corner_radius_crop.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_no_crop_(all_offsets_0).jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_no_crop_(all_offsets_0).jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_rectangular_crop.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_crop_testcrop_rectangular_crop.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_classification_testdrawclassification_classification_dog.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_classification_testdrawclassification_classification_dog.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_detection_testdrawdetection_detection_coco_1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_detection_testdrawdetection_detection_coco_1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_detection_testdrawdetection_detection_coco_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_detection_testdrawdetection_detection_coco_2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_coco_2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_stomata.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_instance_segmentation_testdrawinstancesegmentation_instance_segmentation_stomata.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_1.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_keypoint_testdrawkeypoint_keypoint_coco_2.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_ocr_testdrawocr_ocr_mm.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_ocr_testdrawocr_ocr_mm.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_draw_semantic_segmentation_testdrawsemanticsegmentation_semantic_segmentation_cityscape.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_draw_semantic_segmentation_testdrawsemanticsegmentation_semantic_segmentation_cityscape.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_ratio.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_ratio.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_ratio_0.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_ratio_0.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_0.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_0.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_and_height.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_and_height.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_and_height_0.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/image/v0/testdata/test_output_task_resize_testresize_resize_by_width_and_height_0.jpeg -------------------------------------------------------------------------------- /pkg/component/operator/json/v0/assets/json.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/json/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_MARSHAL 3 | - TASK_UNMARSHAL 4 | - TASK_JQ 5 | - TASK_RENAME_FIELDS 6 | custom: false 7 | documentationUrl: https://instill-ai.dev/docs/component/operator/json 8 | icon: assets/json.svg 9 | iconUrl: '' 10 | id: json 11 | public: true 12 | spec: {} 13 | title: JSON 14 | type: COMPONENT_TYPE_OPERATOR 15 | uid: 28f53d15-6150-46e6-99aa-f76b70a926c0 16 | version: 0.1.0 17 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/json/v0 18 | description: Manipulate and convert JSON entities. 19 | releaseStage: RELEASE_STAGE_ALPHA 20 | -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/assets/text.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CHUNK_TEXT 3 | custom: false 4 | documentationUrl: https://instill-ai.dev/docs/component/operator/text 5 | icon: assets/text.svg 6 | iconUrl: '' 7 | id: text 8 | public: true 9 | spec: {} 10 | title: Text 11 | type: COMPONENT_TYPE_OPERATOR 12 | uid: 5b7aca5b-1ae3-477f-bf60-d34e1c993c87 13 | version: 0.1.1 14 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/text/v0 15 | description: Extract and manipulate text from different sources. 16 | releaseStage: RELEASE_STAGE_ALPHA 17 | -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/chinese/chunk1_1.txt: -------------------------------------------------------------------------------- 1 | # A 2 | 我怎麼知道誰比較強1 3 | 我怎麼知道誰比較強2 4 | 我怎麼知道誰比較強3 -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/chinese/chunk1_2.txt: -------------------------------------------------------------------------------- 1 | 我怎麼知道誰比較強3 2 | 我怎麼知道誰比較強4 -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/chinese/chunk1_3.txt: -------------------------------------------------------------------------------- 1 | 我怎麼知道誰比較強5 2 | 我怎麼知道誰比較強6 3 | 我怎麼知道誰比較強7 -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/chinese/text1.txt: -------------------------------------------------------------------------------- 1 | # A 2 | 我怎麼知道誰比較強1 3 | 我怎麼知道誰比較強2 4 | 我怎麼知道誰比較強3 5 | 我怎麼知道誰比較強4 6 | 7 | 我怎麼知道誰比較強5 8 | 我怎麼知道誰比較強6 9 | 我怎麼知道誰比較強7 10 | 我怎麼知道誰比較強8 11 | ## B 12 | 我怎麼知道誰比較若9 13 | 我怎麼知道誰比較若0 14 | 我怎麼知道誰比較若1 15 | 若若若若若若若若若2 16 | 我怎麼知道誰比較若3 17 | 若若若若若若若若若4 18 | 我怎麼知道誰比較若5 19 | 若若若若若若若若若6 20 | 21 | 22 | ### C 23 | 不知道就不要亂說話7 24 | 不知道就不要亂說話8 25 | 不知道就不要亂說話9 26 | 不知道就不要亂說話0 27 | 不知道就不要亂說話1 28 | 不知道就不要亂說話2 29 | 不知道就不要亂說話3 30 | 31 | 不知道就不要亂說話4 32 | 不知道就不要亂說話5 -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/list_position_test.md: -------------------------------------------------------------------------------- 1 | # Progress Notes 2 | LUTS 3 | => **LUTS:Lower Urinary Tract Symptoms，表示有下泌尿道感染** 4 | 5 | # 護理 6 | 主訴有困難解尿情形，有尿液感但都解不出來 7 | => **符合有症狀的泌尿道感染之徵象或症狀:解尿困難或疼痛(dysuria)** 8 | 9 | # 個案泌尿道感染判定 10 | 1. 沒有導尿管或導尿管留置未超過2天 11 | 2. 病患有UTI感染症狀:解尿困難或疼痛(dysuria)、Lower Urinary Tract Symptoms 12 | => **判斷為:非導尿管相關泌尿道感染(Non-CAUTI)** -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/test.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/text/v0/testdata/test.docx -------------------------------------------------------------------------------- /pkg/component/operator/text/v0/testdata/test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/text/v0/testdata/test.pdf -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/assets/video.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_SEGMENT 3 | - TASK_SUBSAMPLE 4 | - TASK_EXTRACT_AUDIO 5 | - TASK_EXTRACT_FRAMES 6 | - TASK_EMBED_AUDIO 7 | documentationUrl: https://instill-ai.dev/docs/component/operator/video 8 | icon: assets/video.svg 9 | id: video 10 | public: true 11 | spec: {} 12 | title: Video 13 | type: COMPONENT_TYPE_OPERATOR 14 | uid: f0be2fd3-7266-4eeb-88eb-3bbbcc2a6b32 15 | version: 0.1.0 16 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/video/v0 17 | description: Operate video data. 18 | releaseStage: RELEASE_STAGE_ALPHA 19 | -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/audio.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/audio.ogg -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/embed-video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/embed-video.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/frame-0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/frame-0.png -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/frame-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/frame-1.png -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/frame-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/frame-2.png -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/frame-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/frame-3.png -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/frame-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/frame-4.png -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-sample-bunny.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-sample-bunny.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-0.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-1.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-2.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-2.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-3.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-3.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-4.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-4.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segment-5.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-segment-5.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-segments.json: -------------------------------------------------------------------------------- 1 | { 2 | "segments": [ 3 | { 4 | "start-time": 0, 5 | "end-time": 10 6 | }, 7 | { 8 | "start-time": 10, 9 | "end-time": 20 10 | }, 11 | { 12 | "start-time": 20, 13 | "end-time": 30 14 | }, 15 | { 16 | "start-time": 30, 17 | "end-time": 40 18 | }, 19 | { 20 | "start-time": 40, 21 | "end-time": 50 22 | }, 23 | { 24 | "start-time": 50, 25 | "end-time": 60 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-subsample-280x0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-subsample-280x0.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-subsample-abr-128.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-subsample-abr-128.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-subsample-fps-15.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-subsample-fps-15.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video-subsample-vbr-1000.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video-subsample-vbr-1000.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/video/v0/testdata/video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/operator/video/v0/testdata/video.mp4 -------------------------------------------------------------------------------- /pkg/component/operator/web/v0/assets/web.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /pkg/component/operator/web/v0/config/definition.yaml: -------------------------------------------------------------------------------- 1 | availableTasks: 2 | - TASK_CRAWL_SITE 3 | - TASK_SCRAPE_PAGES 4 | - TASK_SCRAPE_SITEMAP 5 | documentationUrl: https://instill-ai.dev/docs/component/operator/web 6 | icon: assets/web.svg 7 | id: web 8 | public: true 9 | title: Web 10 | type: COMPONENT_TYPE_OPERATOR 11 | uid: 98909958-db7d-4dfe-9858-7761904be17e 12 | version: 0.4.0 13 | sourceUrl: https://github.com/instill-ai/pipeline-backend/blob/main/pkg/component/operator/web/v0 14 | description: Scrape websites. 15 | releaseStage: RELEASE_STAGE_ALPHA 16 | -------------------------------------------------------------------------------- /pkg/component/resources/onnx/silero_vad.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/component/resources/onnx/silero_vad.onnx -------------------------------------------------------------------------------- /pkg/component/resources/schemas/schema.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | _ "embed" 5 | ) 6 | 7 | //go:embed schema.yaml 8 | var SchemaYAML []byte 9 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/Makefile: -------------------------------------------------------------------------------- 1 | COMPONENT_BASE := ../../ 2 | 3 | #============================================================================ 4 | # Generate the documentation of one of more components with the local version 5 | # of `compogen`. 6 | # - `t`: component type 7 | # - `c`: component name 8 | # Example: make local-gen-doc t=application c=slack 9 | local-gen-doc: 10 | @go install . 11 | @cd ${COMPONENT_BASE} && go generate -run compogen $(shell echo ./$t/$c/... | tr -s /) 12 | 13 | test: 14 | go test -mod=mod -race -cover -json ./... | tparse --all --notests 15 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/cmd/execute_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "testing" 7 | 8 | "github.com/rogpeppe/go-internal/testscript" 9 | ) 10 | 11 | var updateScripts = flag.Bool("update", false, "update testdata/*.txt files with actual command output") 12 | 13 | func TestExecute(t *testing.T) { 14 | testscript.Run(t, testscript.Params{ 15 | TestWork: true, 16 | Dir: "testdata", 17 | UpdateScripts: *updateScripts, 18 | }) 19 | } 20 | 21 | func TestMain(m *testing.M) { 22 | os.Exit(testscript.RunMain(m, map[string]func() int{ 23 | "compogen": Execute, 24 | })) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/cmd/testdata/usage.txt: -------------------------------------------------------------------------------- 1 | # OK - CLI help. 2 | 3 | compogen --version 4 | cmp stdout want-version 5 | 6 | compogen --help 7 | cmp stdout want-help 8 | 9 | compogen help 10 | cmp stdout want-help 11 | 12 | # NOK - Passing unknown flags should produce a usage message. 13 | 14 | ! compogen --unknown 15 | cmp stderr want-usage 16 | 17 | -- want-version -- 18 | compogen version 0.1.2 19 | -- want-usage -- 20 | Error: unknown flag: --unknown 21 | Run 'compogen --help' for usage. 22 | -- want-help -- 23 | compogen is the Instill AI component schema generation tool 24 | 25 | Usage: 26 | compogen [command] 27 | 28 | Available Commands: 29 | completion Generate the autocompletion script for the specified shell 30 | help Help about any command 31 | readme Generate component README 32 | 33 | Flags: 34 | -h, --help help for compogen 35 | -v, --version version for compogen 36 | 37 | Use "compogen [command] --help" for more information about a command. 38 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/main.go: -------------------------------------------------------------------------------- 1 | // compogen is a generation tool for Instill AI component schemas. It is a 2 | // command line application that should guide users through the usage, 3 | // documentation and maintenance of Instill AI components. 4 | package main 5 | 6 | import ( 7 | "os" 8 | 9 | "github.com/instill-ai/pipeline-backend/pkg/component/tools/compogen/cmd" 10 | ) 11 | 12 | func main() { 13 | os.Exit(cmd.Execute()) 14 | } 15 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/pkg/gen/utils.go: -------------------------------------------------------------------------------- 1 | package gen 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "gopkg.in/yaml.v3" 7 | ) 8 | 9 | func convertYAMLToJSON(yamlBytes []byte) ([]byte, error) { 10 | if yamlBytes == nil { 11 | return nil, nil 12 | } 13 | var d any 14 | err := yaml.Unmarshal(yamlBytes, &d) 15 | if err != nil { 16 | return nil, err 17 | } 18 | return json.Marshal(d) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/component/tools/compogen/pkg/gen/validation.go: -------------------------------------------------------------------------------- 1 | package gen 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/go-playground/validator/v10" 8 | ) 9 | 10 | func fieldErrorMessage(fe validator.FieldError) string { 11 | var msg string 12 | switch fe.Tag() { 13 | case "required", "required_if": 14 | msg = "is required" 15 | case "len": 16 | msg = "has an invalid length" 17 | case "gt": 18 | msg = "doesn't reach the minimum value / number of elements" 19 | case "semver": 20 | msg = "must be valid SemVer 2.0.0" 21 | case "url": 22 | msg = "must be a valid URL" 23 | default: 24 | return fe.Error() // default error 25 | } 26 | 27 | return fmt.Sprintf("%s: %s field %s", fe.Namespace(), fe.Field(), msg) 28 | } 29 | 30 | func asValidationError(err error) error { 31 | var ve validator.ValidationErrors 32 | if !errors.As(err, &ve) { 33 | return err 34 | } 35 | 36 | errs := make([]error, len(ve)) 37 | for i, fe := range ve { 38 | errs[i] = fmt.Errorf("%s", fieldErrorMessage(fe)) 39 | } 40 | 41 | return errors.Join(errs...) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/data/format/value.go: -------------------------------------------------------------------------------- 1 | package format 2 | 3 | import ( 4 | "google.golang.org/protobuf/types/known/structpb" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/data/path" 7 | ) 8 | 9 | type Value interface { 10 | IsValue() 11 | ToStructValue() (v *structpb.Value, err error) 12 | ToJSONValue() (v any, err error) 13 | Get(p *path.Path) (v Value, err error) 14 | Equal(other Value) bool 15 | String() string 16 | } 17 | -------------------------------------------------------------------------------- /pkg/data/testdata/original_links.txt: -------------------------------------------------------------------------------- 1 | Original links: 2 | https://filesamples.com/samples/video/mp4/sample_640x360.mp4 3 | https://filesamples.com/samples/video/mov/sample_640x360.mov 4 | https://filesamples.com/samples/video/wmv/sample_640x360.wmv 5 | https://filesamples.com/samples/audio/wav/sample1.wav 6 | https://filesamples.com/samples/audio/mp3/sample1.mp3 7 | https://filesamples.com/samples/audio/ogg/sample1.ogg 8 | https://filesamples.com/samples/image/png/sample_640×426.png 9 | https://filesamples.com/samples/image/jpeg/sample_640×426.jpeg 10 | https://filesamples.com/samples/image/tiff/sample_640×426.tiff 11 | https://filesamples.com/samples/document/pdf/sample2.pdf 12 | https://filesamples.com/samples/document/txt/sample2.txt 13 | https://filesamples.com/samples/document/docx/sample1.docx 14 | -------------------------------------------------------------------------------- /pkg/data/testdata/readme.md: -------------------------------------------------------------------------------- 1 | # Test Data Files 2 | 3 | The files in this directory are sample video, audio, and image files used for testing purposes. These files have been downloaded from https://filesamples.com. 4 | 5 | 6 | -------------------------------------------------------------------------------- /pkg/data/testdata/sample1.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample1.docx -------------------------------------------------------------------------------- /pkg/data/testdata/sample1.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample1.mp3 -------------------------------------------------------------------------------- /pkg/data/testdata/sample1.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample1.ogg -------------------------------------------------------------------------------- /pkg/data/testdata/sample1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample1.wav -------------------------------------------------------------------------------- /pkg/data/testdata/sample2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample2.pdf -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_360.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_360.mov -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_360.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_360.mp4 -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_360.wmv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_360.wmv -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_426.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_426.jpeg -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_426.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_426.png -------------------------------------------------------------------------------- /pkg/data/testdata/sample_640_426.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/data/testdata/sample_640_426.tiff -------------------------------------------------------------------------------- /pkg/datamodel/datamodel_test.go: -------------------------------------------------------------------------------- 1 | package datamodel 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/frankban/quicktest" 7 | ) 8 | 9 | func TestDatamodel_TagNames(t *testing.T) { 10 | c := quicktest.New(t) 11 | 12 | testCases := []struct { 13 | pipeline *Pipeline 14 | expected []string 15 | }{ 16 | { 17 | pipeline: &Pipeline{ 18 | Tags: []*Tag{ 19 | { 20 | TagName: "tag1", 21 | }, 22 | { 23 | TagName: "tag2", 24 | }, 25 | }, 26 | }, 27 | expected: []string{"tag1", "tag2"}, 28 | }, 29 | { 30 | pipeline: &Pipeline{}, 31 | expected: []string{}, 32 | }, 33 | } 34 | 35 | for _, tc := range testCases { 36 | tagNames := tc.pipeline.TagNames() 37 | c.Assert(tagNames, quicktest.DeepEquals, tc.expected) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pkg/db/migration/000001_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS pipelines; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000002_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000002_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000002_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000002_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000003_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000003_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000003_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000003_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000004_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000004_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000005_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000005_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000005_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline ADD COLUMN "permission" JSONB DEFAULT '{}'; 4 | ALTER TABLE public.pipeline ADD COLUMN "share_code" VARCHAR(255) DEFAULT '' NOT NULL; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000006_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000006_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000006_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline ADD COLUMN "metadata" JSONB DEFAULT '{}'; 4 | ALTER TABLE public.pipeline_release ADD COLUMN "metadata" JSONB DEFAULT '{}'; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000007_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000007_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000007_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000007_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000008_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000008_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000009_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000009_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000009_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline ADD COLUMN "readme" TEXT DEFAULT ''; 4 | ALTER TABLE public.pipeline_release ADD COLUMN "readme" TEXT DEFAULT ''; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000010_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000010_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000010_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline RENAME COLUMN permission TO sharing; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000011_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TYPE CONNECTOR_VALID_CONNECTOR_TYPE RENAME VALUE 'CONNECTOR_TYPE_APPLICATION' TO 'CONNECTOR_TYPE_BLOCKCHAIN'; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000011_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TYPE CONNECTOR_VALID_CONNECTOR_TYPE RENAME VALUE 'CONNECTOR_TYPE_BLOCKCHAIN' TO 'CONNECTOR_TYPE_APPLICATION'; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000012_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP INDEX IF EXISTS component_definition_index_filter; 4 | 5 | DROP TABLE IF EXISTS component_definition_index CASCADE; 6 | 7 | DROP TYPE IF EXISTS VALID_RELEASE_STAGE; 8 | 9 | DROP TYPE IF EXISTS COMPONENT_DEFINITION_VALID_COMPONENT_TYPE ; 10 | 11 | COMMIT; 12 | -------------------------------------------------------------------------------- /pkg/db/migration/000013_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS secret; 4 | DROP INDEX IF EXISTS secret_unique_owner_id; 5 | DROP INDEX IF EXISTS secret_uid_create_time_pagination; 6 | 7 | COMMIT; 8 | -------------------------------------------------------------------------------- /pkg/db/migration/000013_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | CREATE TABLE IF NOT EXISTS public.secret ( 3 | uid UUID NOT NULL, 4 | id VARCHAR(255) NOT NULL, 5 | owner VARCHAR(255) NOT NULL, 6 | description VARCHAR(1023) NULL, 7 | value text NULL, 8 | create_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL, 9 | update_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL 10 | ); 11 | CREATE UNIQUE INDEX secret_unique_owner_id ON public.secret (owner, id); 12 | CREATE INDEX secret_uid_create_time_pagination ON public.secret (uid, create_time); 13 | 14 | COMMIT; 15 | -------------------------------------------------------------------------------- /pkg/db/migration/000014_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS tag; 4 | DROP INDEX IF EXISTS tag_pipeline_uid; 5 | DROP INDEX IF EXISTS tag_tag_name; 6 | DROP INDEX IF EXISTS tag_unique_pipeline_tag; 7 | 8 | COMMIT; 9 | -------------------------------------------------------------------------------- /pkg/db/migration/000014_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | CREATE TABLE IF NOT EXISTS public.tag( 4 | pipeline_uid UUID NOT NULL, 5 | tag_name VARCHAR(255) NOT NULL, 6 | create_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL, 7 | update_time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL 8 | ); 9 | 10 | 11 | CREATE INDEX tag_pipeline_uid ON public.tag(pipeline_uid); 12 | CREATE INDEX tag_tag_name on public.tag(tag_name); 13 | CREATE UNIQUE INDEX tag_unique_pipeline_tag ON public.tag(pipeline_uid, tag_name); 14 | 15 | COMMIT; 16 | -------------------------------------------------------------------------------- /pkg/db/migration/000015_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_AI' TO 'COMPONENT_TYPE_CONNECTOR_AI'; 4 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_DATA' TO 'COMPONENT_TYPE_CONNECTOR_DATA'; 5 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_APPLICATION' TO 'COMPONENT_TYPE_CONNECTOR_APPLICATION'; 6 | 7 | ALTER TABLE public.pipeline DROP COLUMN number_of_runs; 8 | ALTER TABLE public.pipeline DROP COLUMN last_run_time; 9 | DROP INDEX IF EXISTS pipeline_number_of_runs; 10 | DROP INDEX IF EXISTS pipeline_last_run_time; 11 | 12 | COMMIT; 13 | -------------------------------------------------------------------------------- /pkg/db/migration/000015_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_CONNECTOR_AI' TO 'COMPONENT_TYPE_AI'; 4 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_CONNECTOR_DATA' TO 'COMPONENT_TYPE_DATA'; 5 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE RENAME VALUE 'COMPONENT_TYPE_CONNECTOR_APPLICATION' TO 'COMPONENT_TYPE_APPLICATION'; 6 | 7 | ALTER TABLE public.pipeline ADD COLUMN number_of_runs INTEGER DEFAULT 0; 8 | ALTER TABLE public.pipeline ADD COLUMN last_run_time TIMESTAMPTZ DEFAULT '0001-01-01T00:00:00Z'; 9 | CREATE INDEX pipeline_number_of_runs ON public.pipeline (number_of_runs); 10 | CREATE INDEX pipeline_last_run_time ON public.pipeline (last_run_time); 11 | 12 | COMMIT; 13 | -------------------------------------------------------------------------------- /pkg/db/migration/000016_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline DROP COLUMN recipe_yaml; 4 | ALTER TABLE public.pipeline_release DROP COLUMN recipe_yaml; 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000016_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline ADD COLUMN recipe_yaml TEXT DEFAULT ''; 4 | ALTER TABLE public.pipeline_release ADD COLUMN recipe_yaml TEXT DEFAULT ''; 5 | ALTER TABLE public.pipeline ALTER COLUMN recipe DROP NOT NULL; 6 | ALTER TABLE public.pipeline_release ALTER COLUMN recipe DROP NOT NULL; 7 | 8 | COMMIT; 9 | -------------------------------------------------------------------------------- /pkg/db/migration/000018_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline DROP COLUMN source_url; 4 | ALTER TABLE public.pipeline DROP COLUMN documentation_url; 5 | ALTER TABLE public.pipeline DROP COLUMN license; 6 | ALTER TABLE public.pipeline DROP COLUMN profile_image; 7 | 8 | COMMIT; 9 | -------------------------------------------------------------------------------- /pkg/db/migration/000018_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS source_url VARCHAR(255) DEFAULT ''; 4 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS documentation_url VARCHAR(255) DEFAULT ''; 5 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS license VARCHAR(255) DEFAULT ''; 6 | 7 | -- `profile_image` stores the profile image of the pipeline in base64 format. 8 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS profile_image TEXT DEFAULT NULL; 9 | 10 | COMMIT; 11 | -------------------------------------------------------------------------------- /pkg/db/migration/000019_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000019_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000019_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000019_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000020_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP EXTENSION pg_trgm; 4 | ALTER TABLE public.pipeline DROP COLUMN namespace_id; 5 | ALTER TABLE public.pipeline DROP COLUMN namespace_type; 6 | ALTER TABLE public.secret DROP COLUMN namespace_id; 7 | ALTER TABLE public.secret DROP COLUMN namespace_type; 8 | 9 | COMMIT; 10 | -------------------------------------------------------------------------------- /pkg/db/migration/000020_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | CREATE EXTENSION IF NOT EXISTS pg_trgm; 4 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS namespace_id VARCHAR(255) DEFAULT ''; 5 | ALTER TABLE public.pipeline ADD COLUMN IF NOT EXISTS namespace_type VARCHAR(255) DEFAULT ''; 6 | ALTER TABLE public.secret ADD COLUMN IF NOT EXISTS namespace_id VARCHAR(255) DEFAULT ''; 7 | ALTER TABLE public.secret ADD COLUMN IF NOT EXISTS namespace_type VARCHAR(255) DEFAULT ''; 8 | 9 | COMMIT; 10 | -------------------------------------------------------------------------------- /pkg/db/migration/000021_init.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | UPDATE public.component_definition_index SET component_type='COMPONENT_TYPE_APPLICATION' WHERE id='restapi'; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000021_init.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | ALTER TYPE COMPONENT_DEFINITION_VALID_COMPONENT_TYPE ADD VALUE 'COMPONENT_TYPE_GENERIC'; 3 | COMMIT; 4 | 5 | BEGIN; 6 | UPDATE public.component_definition_index SET component_type='COMPONENT_TYPE_GENERIC' WHERE id='restapi'; 7 | COMMIT; 8 | -------------------------------------------------------------------------------- /pkg/db/migration/000022_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000022_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000022_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000022_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000023_run_logging.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP TABLE IF EXISTS component_run; 4 | DROP TABLE IF EXISTS pipeline_run; 5 | 6 | DROP TYPE IF EXISTS valid_trigger_status; 7 | DROP TYPE IF EXISTS valid_trigger_source; 8 | 9 | COMMIT; 10 | -------------------------------------------------------------------------------- /pkg/db/migration/000024_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000024_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000024_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000024_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000025_add_integration_column.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP INDEX IF EXISTS integration_index; 4 | ALTER TABLE component_definition_index 5 | DROP COLUMN IF EXISTS has_integration, 6 | DROP COLUMN IF EXISTS vendor; 7 | 8 | COMMIT; 9 | -------------------------------------------------------------------------------- /pkg/db/migration/000025_add_integration_column.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE component_definition_index 4 | ADD COLUMN IF NOT EXISTS has_integration BOOL DEFAULT FALSE NOT NULL, 5 | ADD COLUMN IF NOT EXISTS vendor VARCHAR(255) DEFAULT '' NOT NULL; 6 | 7 | CREATE INDEX IF NOT EXISTS integration_index ON component_definition_index (feature_score DESC, uid DESC) WHERE is_visible IS TRUE AND has_integration IS TRUE; 8 | 9 | COMMIT; 10 | -------------------------------------------------------------------------------- /pkg/db/migration/000026_add_connections.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | DROP INDEX IF EXISTS unique_connection_id_namespace; 4 | DROP TABLE IF EXISTS connection; 5 | DROP TYPE valid_connection_method; 6 | 7 | COMMIT; 8 | -------------------------------------------------------------------------------- /pkg/db/migration/000026_add_connections.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | CREATE TYPE valid_connection_method AS ENUM ( 4 | 'METHOD_DICTIONARY', 5 | 'METHOD_OAUTH' 6 | ); 7 | 8 | CREATE TABLE IF NOT EXISTS connection ( 9 | uid UUID PRIMARY KEY, 10 | id VARCHAR(255) NOT NULL, 11 | namespace_uid UUID NOT NULL, 12 | integration_uid UUID NOT NULL REFERENCES component_definition_index, 13 | method valid_connection_method NOT NULL, 14 | setup JSONB NOT NULL, 15 | create_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, 16 | update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, 17 | delete_time TIMESTAMPTZ 18 | ); 19 | 20 | CREATE UNIQUE INDEX unique_connection_id_namespace ON connection (id, namespace_uid) WHERE delete_time IS NULL; 21 | 22 | COMMIT; 23 | -------------------------------------------------------------------------------- /pkg/db/migration/000027_run_logging_namespace.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | alter table pipeline_run 4 | drop column namespace; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000027_run_logging_namespace.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | alter table pipeline_run 4 | add namespace varchar(255) not null default ''; 5 | 6 | comment on column pipeline_run.namespace is 'run by namespace, which is the credit owner'; 7 | 8 | update pipeline_run 9 | set namespace=triggered_by 10 | where pipeline_run.namespace = ''; 11 | 12 | COMMIT; 13 | -------------------------------------------------------------------------------- /pkg/db/migration/000028_oauth_integration.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE connection DROP COLUMN IF EXISTS o_auth_access_details; 4 | ALTER TABLE connection DROP COLUMN IF EXISTS scopes; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000028_oauth_integration.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE connection ADD COLUMN IF NOT EXISTS scopes TEXT[]; 4 | ALTER TABLE connection ADD COLUMN IF NOT EXISTS o_auth_access_details JSONB; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000029_init.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000029_init.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000029_init.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000029_init.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000030_connection_identity.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE connection DROP COLUMN IF EXISTS identity; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000030_connection_identity.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE connection ADD COLUMN IF NOT EXISTS identity TEXT; 4 | 5 | COMMIT; 6 | -------------------------------------------------------------------------------- /pkg/db/migration/000031_migrate_slack_setup.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000031_migrate_slack_setup.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000031_migrate_slack_setup.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000031_migrate_slack_setup.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000032_migrate_web_recipes.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000032_migrate_web_recipes.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000032_migrate_web_recipes.down.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000032_migrate_web_recipes.down.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000033_migrate_web_recipes.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000033_migrate_web_recipes.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000033_migrate_web_recipes.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000033_migrate_web_recipes.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000034_rename_http_component.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000034_rename_http_component.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000034_rename_http_component.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000034_rename_http_component.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000035_run_requester_uid.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | alter table pipeline_run 4 | rename column runner_uid to triggered_by; 5 | 6 | alter table pipeline_run 7 | rename column requester_uid to namespace; 8 | 9 | comment on column pipeline_run.namespace is 'run by namespace, which is the credit owner'; 10 | 11 | alter table pipeline_run 12 | alter column namespace type varchar(255) using ''::character varying; 13 | 14 | alter table pipeline_run 15 | alter column triggered_by type varchar(255); 16 | 17 | COMMIT; 18 | -------------------------------------------------------------------------------- /pkg/db/migration/000035_run_requester_uid.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | alter table pipeline_run 4 | rename column triggered_by to runner_uid; 5 | 6 | alter table pipeline_run 7 | rename column namespace to requester_uid; 8 | 9 | comment on column pipeline_run.requester_uid is null; 10 | 11 | alter table pipeline_run 12 | alter column runner_uid type uuid using runner_uid::uuid; 13 | 14 | alter table pipeline_run 15 | alter column requester_uid drop default; 16 | 17 | alter table pipeline_run 18 | alter column requester_uid type uuid using requester_uid::uuid; 19 | 20 | COMMIT; 21 | -------------------------------------------------------------------------------- /pkg/db/migration/000036_rename_instill_format.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000036_rename_instill_format.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000036_rename_instill_format.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000036_rename_instill_format.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/000037_create_pipeline_run_on_table.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | DROP INDEX IF EXISTS pipeline_run_on_pipeline_uid_release_uid_event_id_run_on_type; 3 | DROP INDEX IF EXISTS pipeline_run_on_run_on_type_identifier_pagination; 4 | DROP INDEX IF EXISTS pipeline_run_on_pipeline_uid_release_uid_create_time_pagination; 5 | DROP TABLE IF EXISTS public.pipeline_run_on; 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000038_add_pipeline_run_on_table_column.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE public.pipeline_run_on 4 | DROP COLUMN config, 5 | DROP COLUMN setup; 6 | 7 | COMMIT; 8 | -------------------------------------------------------------------------------- /pkg/db/migration/000038_add_pipeline_run_on_table_column.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | -- Store config and setup for pipeline run events to enable proper event unregistration with original settings 4 | ALTER TABLE public.pipeline_run_on 5 | ADD COLUMN config JSONB, 6 | ADD COLUMN setup JSONB; 7 | 8 | COMMIT; 9 | -------------------------------------------------------------------------------- /pkg/db/migration/000039_add_pipeline_run_expiration_time.down.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE component_run DROP COLUMN IF EXISTS blob_data_expiration_time; 4 | ALTER TABLE pipeline_run DROP COLUMN IF EXISTS blob_data_expiration_time; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000039_add_pipeline_run_expiration_time.up.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | ALTER TABLE pipeline_run ADD COLUMN IF NOT EXISTS blob_data_expiration_time TIMESTAMPTZ; 4 | ALTER TABLE component_run ADD COLUMN IF NOT EXISTS blob_data_expiration_time TIMESTAMPTZ; 5 | 6 | COMMIT; 7 | -------------------------------------------------------------------------------- /pkg/db/migration/000040_rename_format_to_type.down.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000040_rename_format_to_type.down.sql -------------------------------------------------------------------------------- /pkg/db/migration/000040_rename_format_to_type.up.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/instill-ai/pipeline-backend/5254cbee726592e347116ad0c88d53fe4b4c7819/pkg/db/migration/000040_rename_format_to_type.up.sql -------------------------------------------------------------------------------- /pkg/db/migration/convert/converter.go: -------------------------------------------------------------------------------- 1 | package convert 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "gorm.io/gorm" 6 | ) 7 | 8 | // Basic contains the basic elements to execute a conversion migration. 9 | type Basic struct { 10 | DB *gorm.DB 11 | Logger *zap.Logger 12 | } 13 | -------------------------------------------------------------------------------- /pkg/handler/component_definition.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | 6 | "go.opentelemetry.io/otel/trace" 7 | 8 | pb "github.com/instill-ai/protogen-go/pipeline/pipeline/v1beta" 9 | ) 10 | 11 | // ListComponentDefinitions returns a paginated list of component definitions. 12 | func (h *PublicHandler) ListComponentDefinitions(ctx context.Context, req *pb.ListComponentDefinitionsRequest) (*pb.ListComponentDefinitionsResponse, error) { 13 | ctx, span := tracer.Start(ctx, "ListComponentDefinitions", trace.WithSpanKind(trace.SpanKindServer)) 14 | defer span.End() 15 | 16 | resp, err := h.service.ListComponentDefinitions(ctx, req) 17 | if err != nil { 18 | span.SetStatus(1, err.Error()) 19 | return nil, err 20 | } 21 | 22 | h.log.Info("ListComponentDefinitions") 23 | return resp, nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/handler/errors.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import "errors" 4 | 5 | var ErrCheckUpdateImmutableFields = errors.New("update immutable fields error") 6 | var ErrCheckOutputOnlyFields = errors.New("can not contain output only fields") 7 | var ErrCheckRequiredFields = errors.New("required fields missing") 8 | var ErrFieldMask = errors.New("field mask error") 9 | var ErrSematicVersion = errors.New("not a legal version, should be the format vX.Y.Z or vX.Y.Z-identifiers") 10 | var ErrUpdateMask = errors.New("update mask error") 11 | -------------------------------------------------------------------------------- /pkg/handler/utils.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/instill-ai/pipeline-backend/pkg/constant" 7 | "github.com/instill-ai/pipeline-backend/pkg/service" 8 | "github.com/instill-ai/x/resource" 9 | 10 | constantx "github.com/instill-ai/x/constant" 11 | ) 12 | 13 | func authenticateUser(ctx context.Context, allowVisitor bool) error { 14 | if resource.GetRequestSingleHeader(ctx, constant.HeaderServiceKey) == "instill" { 15 | return nil 16 | } 17 | 18 | if resource.GetRequestSingleHeader(ctx, constantx.HeaderAuthTypeKey) == "user" { 19 | if resource.GetRequestSingleHeader(ctx, constantx.HeaderUserUIDKey) == "" { 20 | return service.ErrUnauthenticated 21 | } 22 | return nil 23 | } 24 | 25 | if !allowVisitor { 26 | return service.ErrUnauthenticated 27 | } 28 | 29 | if resource.GetRequestSingleHeader(ctx, constantx.HeaderVisitorUIDKey) == "" { 30 | return service.ErrUnauthenticated 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/mock/generator.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | //go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/repository.Repository -o ./ -s "_mock.gen.go" 4 | //go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/acl.ACLClientInterface -o ./ -s "_mock.gen.go" 5 | //go:generate minimock -g -i github.com/instill-ai/pipeline-backend/pkg/service.Converter -o ./ -s "_mock.gen.go" 6 | //go:generate minimock -g -i github.com/instill-ai/protogen-go/core/mgmt/v1beta.MgmtPrivateServiceClient -o ./ -s "_mock.gen.go" 7 | -------------------------------------------------------------------------------- /pkg/recipe/schema.go: -------------------------------------------------------------------------------- 1 | package recipe 2 | 3 | import ( 4 | _ "embed" 5 | ) 6 | 7 | //go:embed schema.json 8 | var RecipeSchema []byte 9 | -------------------------------------------------------------------------------- /pkg/repository/errors.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | errdomain "github.com/instill-ai/pipeline-backend/pkg/errors" 8 | ) 9 | 10 | var ErrOwnerTypeNotMatch = errors.New("owner type not match") 11 | var ErrNoDataDeleted = errors.New("no data deleted") 12 | var ErrNoDataUpdated = errors.New("no data updated") 13 | 14 | func newPageTokenErr(err error) error { 15 | return fmt.Errorf("%w: invalid page token: %w", errdomain.ErrInvalidArgument, err) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/repository/utils.go: -------------------------------------------------------------------------------- 1 | package repository 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | ) 7 | 8 | func transformBoolToDescString(b bool) string { 9 | if b { 10 | return " DESC" 11 | } 12 | return "" 13 | } 14 | 15 | // TODO: we should refactor this to have a flexible format and merge it into x package. 16 | 17 | // DecodeToken decodes the token string into create_time and UUID 18 | func DecodeToken(encodedToken string) (map[string]any, error) { 19 | byt, err := base64.StdEncoding.DecodeString(encodedToken) 20 | if err != nil { 21 | return nil, err 22 | } 23 | tokens := map[string]any{} 24 | err = json.Unmarshal(byt, &tokens) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return tokens, nil 29 | } 30 | 31 | // EncodeToken encodes create_time and UUID into a single string 32 | func EncodeToken(tokens map[string]any) (string, error) { 33 | b, err := json.Marshal(tokens) 34 | if err != nil { 35 | return "", err 36 | } 37 | return base64.StdEncoding.EncodeToString(b), nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/service/errors.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/instill-ai/x/errmsg" 7 | 8 | errdomain "github.com/instill-ai/pipeline-backend/pkg/errors" 9 | ) 10 | 11 | var ErrUnauthenticated = fmt.Errorf("unauthenticated") 12 | var ErrRateLimiting = fmt.Errorf("rate limiting") 13 | var ErrCanNotTriggerNonLatestPipelineRelease = fmt.Errorf("can not trigger non-latest pipeline release") 14 | var ErrExceedMaxBatchSize = fmt.Errorf("the batch size can not exceed 32") 15 | var ErrTriggerFail = fmt.Errorf("failed to trigger the pipeline") 16 | 17 | var errCanNotUsePlaintextSecret = errmsg.AddMessage( 18 | fmt.Errorf("%w: plaintext value in credential field", errdomain.ErrInvalidArgument), 19 | "Plaintext values are forbidden in credential fields. You can create a secret and reference it with the syntax ${secret.my-secret}.", 20 | ) 21 | -------------------------------------------------------------------------------- /pkg/utils/async.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "fmt" 4 | 5 | func GoSafe(fn func()) { 6 | go func() { 7 | defer func() { 8 | if r := recover(); r != nil { 9 | fmt.Printf("go routine recovered from panic: %s\n", r) 10 | } 11 | }() 12 | 13 | fn() 14 | }() 15 | } 16 | -------------------------------------------------------------------------------- /release-please/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | ".": { 4 | "release-type": "go", 5 | "draft": false, 6 | "prerelease": true, 7 | "bump-minor-pre-major": true, 8 | "bump-patch-for-minor-pre-major": false 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /release-please/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "0.54.0-rc" 3 | } 4 | -------------------------------------------------------------------------------- /resp.json: -------------------------------------------------------------------------------- 1 | { 2 | "componentDefinitions": [ 3 | "collection": [ 4 | { 5 | "name": "component-definitions/collection", 6 | "id": "collection", 7 | "uid": "eb611e31-fbe6-43ad-8671-5b9a2e351638", 8 | "title": "Collection", 9 | "icon": "assets/collection.svg", 10 | ... 11 | "version": "0.1.0" 12 | } 13 | ], 14 | "http": [ 15 | { 16 | "name": "component-definitions/http", 17 | "id": "http", 18 | "uid": "5ee55a5c-6e30-4c7a-80e8-90165a729e0a", 19 | "id": "http", 20 | "title": "HTTP", 21 | "documentationUrl": "https://instill-ai.dev/docs/component/generic/http", 22 | "icon": "assets/http.svg", 23 | ... 24 | "version": "0.2.0" 25 | } 26 | ] 27 | 28 | ], 29 | "totalSize": 2, 30 | "pageSize": 10, 31 | "page": 0 32 | } 33 | --------------------------------------------------------------------------------