) {
7 | return (
8 |
12 | )
13 | }
14 |
15 | export { Skeleton }
16 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/context/toaster-context.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { Toaster } from "react-hot-toast";
4 |
5 | const ToasterContext = () => {
6 | return ;
7 | };
8 |
9 | export default ToasterContext;
10 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/lib/color.ts:
--------------------------------------------------------------------------------
1 | const colors: readonly string[] = [
2 | "slate",
3 | "red",
4 | "orange",
5 | "amber",
6 | "yellow",
7 | "lime",
8 | "green",
9 | "emerald",
10 | "teal",
11 | "cyan",
12 | "sky",
13 | "blue",
14 | "indigo",
15 | "violet",
16 | "purple",
17 | "fuchsia",
18 | "pink",
19 | "rose",
20 | ];
21 |
22 | const intensities: readonly string[] = ["300", "500", "700"];
23 |
24 | export const getAvatarColor = (owner: string): string => {
25 | return (
26 | colors[owner.charCodeAt(0) % colors.length] +
27 | "-" +
28 | intensities[owner.charCodeAt(1) % intensities.length]
29 | );
30 | };
31 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/lib/constants.ts:
--------------------------------------------------------------------------------
1 | export const MY_DRIVE_URL: string = "/my-drive";
2 | export const DEFAULT_OWNER: string = "anonymous";
3 | export const ROOT_FOLDER_ID: string = "Root";
4 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/lib/firebase/admin.ts:
--------------------------------------------------------------------------------
1 | import "server-only";
2 |
3 | import {
4 | initializeApp,
5 | applicationDefault,
6 | AppOptions,
7 | getApp,
8 | App,
9 | } from "firebase-admin/app";
10 |
11 | const ADMIN = "ADMIN";
12 | const firebaseAdminConfig: AppOptions = {
13 | credential: applicationDefault(),
14 | };
15 |
16 | const initialize = () => {
17 | let adminApp: App;
18 | try {
19 | adminApp = getApp(ADMIN);
20 | } catch (e) {
21 | adminApp = initializeApp(firebaseAdminConfig, ADMIN);
22 | }
23 | return adminApp;
24 | };
25 |
26 | const firebaseAdminApp = initialize();
27 |
28 | export default firebaseAdminApp;
29 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/lib/logging.ts:
--------------------------------------------------------------------------------
1 | export const logError = (obj: Object) => {
2 | console.log(JSON.stringify({ ...obj, severity: "ERROR" }));
3 | };
4 |
5 | export const logDebug = (obj: Object) => {
6 | console.log(JSON.stringify({ ...obj, severity: "DEBUG" }));
7 | };
8 |
9 | export const logInfo = (obj: Object) => {
10 | console.log(JSON.stringify({ ...obj, severity: "INFO" }));
11 | };
12 |
13 | export const logWarn = (obj: Object) => {
14 | console.log(JSON.stringify({ ...obj, severity: "WARN" }));
15 | };
16 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/lib/types.ts:
--------------------------------------------------------------------------------
1 | export type ItemSelector = {
2 | text: string;
3 | };
4 |
5 | export type TItem = {
6 | id: string;
7 | name: string;
8 | size: number | null;
9 | type: string | null;
10 | isFolder: boolean;
11 | createdAt: string;
12 | parent: string | null;
13 | owner: string | null;
14 | description: string | null;
15 | embedded: boolean;
16 | };
17 |
18 | export type FolderForBreadcrumb = {
19 | id: string;
20 | name: string;
21 | };
22 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/middleware.ts:
--------------------------------------------------------------------------------
1 | import { NextResponse } from "next/server";
2 | import type { NextRequest } from "next/server";
3 | import { MY_DRIVE_URL } from "./lib/constants";
4 |
5 | export const middleware = (request: NextRequest) => {
6 | return NextResponse.redirect(new URL(MY_DRIVE_URL, request.url));
7 | };
8 |
9 | export const config = {
10 | matcher: "/",
11 | };
12 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/src/providers/query-client.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { useState } from "react";
4 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
5 |
6 | export default function Providers({ children }: { children: React.ReactNode }) {
7 | const [queryClient] = useState(
8 | () =>
9 | new QueryClient({
10 | defaultOptions: {
11 | queries: {
12 | staleTime: 60 * 1000,
13 | },
14 | },
15 | }),
16 | );
17 |
18 | return (
19 | {children}
20 | );
21 | }
22 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive-firestore/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "lib": ["dom", "dom.iterable", "esnext"],
4 | "allowJs": true,
5 | "skipLibCheck": true,
6 | "strict": true,
7 | "noEmit": true,
8 | "esModuleInterop": true,
9 | "module": "esnext",
10 | "moduleResolution": "bundler",
11 | "resolveJsonModule": true,
12 | "isolatedModules": true,
13 | "jsx": "preserve",
14 | "incremental": true,
15 | "plugins": [
16 | {
17 | "name": "next",
18 | },
19 | ],
20 | "paths": {
21 | "@/*": ["./src/*"],
22 | },
23 | },
24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
25 | "exclude": ["node_modules"],
26 | }
27 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/.env:
--------------------------------------------------------------------------------
1 | DB_USER=
2 | DB_PASSWORD=
3 | DB_NAME=
4 | INSTANCE_CONNECTION_NAME=
5 | BUCKET_NAME=
6 | SEARCH_HOST=
7 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/.eslintrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "next/core-web-vitals"
3 | }
4 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.js
7 | .yarn/install-state.gz
8 |
9 | # testing
10 | /coverage
11 |
12 | # next.js
13 | /.next/
14 | /out/
15 |
16 | # production
17 | /build
18 |
19 | # misc
20 | .DS_Store
21 | *.pem
22 |
23 | # debug
24 | npm-debug.log*
25 | yarn-debug.log*
26 | yarn-error.log*
27 |
28 | # local env files
29 | .env*.local
30 |
31 | # vercel
32 | .vercel
33 |
34 | # typescript
35 | *.tsbuildinfo
36 | next-env.d.ts
37 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "plugins": ["prettier-plugin-tailwindcss"]
3 | }
4 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/components.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "https://ui.shadcn.com/schema.json",
3 | "style": "default",
4 | "rsc": true,
5 | "tsx": true,
6 | "tailwind": {
7 | "config": "tailwind.config.ts",
8 | "css": "src/app/globals.css",
9 | "baseColor": "slate",
10 | "cssVariables": true,
11 | "prefix": ""
12 | },
13 | "aliases": {
14 | "components": "@/components",
15 | "utils": "@/lib/utils"
16 | }
17 | }
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/next.config.mjs:
--------------------------------------------------------------------------------
1 | /** @type {import('next').NextConfig} */
2 | const nextConfig = {
3 | output: "standalone",
4 | };
5 |
6 | export default nextConfig;
7 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | };
7 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/public/images/404.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/public/images/404.png
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/public/images/check.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/public/images/check.png
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/public/images/error.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/public/images/error.png
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/public/images/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/public/images/logo.png
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/public/images/notfound.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/public/images/notfound.png
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/(pages)/folders/[slug]/page.tsx:
--------------------------------------------------------------------------------
1 | import Content from "@/components/content";
2 | import Screen from "@/components/screen";
3 | import Sidebar from "@/components/sidebar";
4 |
5 | export default async function FoldersPage({
6 | params,
7 | }: {
8 | params: { slug: string };
9 | }) {
10 | const { slug } = params;
11 | return (
12 |
13 |
14 |
15 |
16 | );
17 | }
18 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/(pages)/items/[slug]/page.tsx:
--------------------------------------------------------------------------------
1 | import { getDownloadURL, getOwner, getSourceIP } from "@/lib/actions";
2 | import { logWarn } from "@/lib/logging";
3 | import { headers } from "next/headers";
4 | import { notFound, redirect } from "next/navigation";
5 |
6 | export default async function PerItemPage({
7 | params,
8 | }: {
9 | params: { slug: string };
10 | }) {
11 | const action = "showItemDetails";
12 | const headersList = headers();
13 | const owner = await getOwner(headersList);
14 | const sourceIP = await getSourceIP(headersList);
15 | const { slug: id } = params;
16 |
17 | const res = await getDownloadURL(id);
18 |
19 | if (!res.url) {
20 | logWarn({
21 | owner: owner,
22 | sourceIP: sourceIP,
23 | action: action,
24 | message: `${sourceIP}/${owner}/${action}/${id}: Failed to find item with id ${id}`,
25 | });
26 | return notFound();
27 | }
28 |
29 | redirect(res.url);
30 | }
31 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/(pages)/my-drive/page.tsx:
--------------------------------------------------------------------------------
1 | import Content from "@/components/content";
2 | import Screen from "@/components/screen";
3 | import Sidebar from "@/components/sidebar";
4 | import { ROOT_FOLDER_ID } from "@/lib/constants";
5 |
6 | export default async function MyDrivePage() {
7 | return (
8 |
9 |
10 |
11 |
12 | );
13 | }
14 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/(pages)/search/page.tsx:
--------------------------------------------------------------------------------
1 | import SearchContent from "@/components/search-content";
2 | import Sidebar from "@/components/sidebar";
3 | import { MY_DRIVE_URL } from "@/lib/constants";
4 | import { redirect } from "next/navigation";
5 | import Screen from "@/components/screen";
6 |
7 | type SearchPageProps = {
8 | searchParams: { [key: string]: string | string[] };
9 | };
10 |
11 | export default async function SearchPage({ searchParams }: SearchPageProps) {
12 | const queryText = searchParams["q"];
13 | if (queryText === undefined) {
14 | redirect(MY_DRIVE_URL);
15 | }
16 |
17 | return (
18 |
19 |
20 |
21 |
22 | );
23 | }
24 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/error.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | export default function Error({
4 | error,
5 | reset,
6 | }: {
7 | error: Error & { digest?: string };
8 | reset: () => void;
9 | }) {
10 | return (
11 |
12 |
Something went wrong!
13 |
14 |
15 | );
16 | }
17 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_genai_googlecloud/src/knowledge-drive/src/app/favicon.ico
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/layout.tsx:
--------------------------------------------------------------------------------
1 | import type { Metadata } from "next";
2 | import { Inter } from "next/font/google";
3 | import "./globals.css";
4 | import ToasterContext from "@/context/toaster-context";
5 | import Providers from "@/providers/query-client";
6 |
7 | const inter = Inter({ subsets: ["latin"] });
8 |
9 | export const metadata: Metadata = {
10 | title: "Knowledge Drive",
11 | description: "Store your knowledge into Cloud",
12 | };
13 |
14 | export default function RootLayout({
15 | children,
16 | }: Readonly<{
17 | children: React.ReactNode;
18 | }>) {
19 | return (
20 |
21 |
24 |
25 | {children}
26 |
27 |
28 |
29 |
30 | );
31 | }
32 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/app/not-found.tsx:
--------------------------------------------------------------------------------
1 | import Image from "next/image";
2 |
3 | function NotFoundPage() {
4 | return (
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 | 404.{" "}
13 | エラーが発生しました。
14 |
15 |
16 |
17 | リクエストされた URL はこのサーバーで見つかりませんでした。
18 |
19 | 確認できたことは以上です。
20 |
21 |
22 |
23 |
24 |
25 |
ドライブ
;
3 | };
4 |
5 | export default AppName;
6 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/avatar.tsx:
--------------------------------------------------------------------------------
1 | import { getOwner } from "@/lib/actions";
2 | import { getAvatarColor } from "@/lib/color";
3 | import { DEFAULT_OWNER } from "@/lib/constants";
4 | import { headers } from "next/headers";
5 | import Link from "next/link";
6 | import { FaUser } from "react-icons/fa6";
7 |
8 | const Avatar = async () => {
9 | const headersList = headers();
10 | const owner = await getOwner(headersList);
11 | const color = getAvatarColor(owner);
12 | const bgColor = `bg-${color}`;
13 |
14 | return (
15 |
16 | {owner === DEFAULT_OWNER ? (
17 |
18 |
19 |
20 | ) : (
21 |
22 |
25 | {owner.charAt(0).toUpperCase()}
26 |
27 |
28 | )}
29 |
23 |
24 | {folder.name}
25 |
26 |
13 | {folders.map((folder, index) => (
14 |
15 | {index !== 0 && (
16 |
20 | )}
21 |
22 |
23 | ))}
24 |
13 | {children}
14 |
15 |
16 |
17 |
18 |
19 |
22 |
23 |
24 |
17 |
18 |
19 | }>
20 |
21 |
22 |
23 |
24 | }>
25 |
26 |
27 |
28 |
29 |
6 |
7 |
8 | );
9 | };
10 |
11 | export default DetailsButton;
12 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/empty-list.tsx:
--------------------------------------------------------------------------------
1 | import Image from "next/image";
2 |
3 | type EmptyListProps = {
4 | children: React.ReactNode;
5 | };
6 |
7 | const EmptyList = ({ children }: EmptyListProps) => {
8 | return (
9 |
10 |
11 |
17 |
18 | {children}
19 |
5 |
名前
6 |
オーナー
7 |
作成日時
8 |
ファイルサイズ
9 |
10 |
14 | {itemSelectors.map((itemSelector) => {
15 | return (
16 |
20 | );
21 | })}
22 |
23 | );
24 | };
25 |
26 | export default ItemSelector;
27 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/llm-search-dialog.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { Dialog, DialogContent } from "@/components/ui/dialog";
4 | import LLMSearchResult from "@/components/llm-search-result";
5 |
6 | type LLMSearchDialogProps = {
7 | open: boolean;
8 | onOpenChange: (open: boolean) => void;
9 | question: string;
10 | };
11 |
12 | const LLMSearchDialog = ({
13 | open,
14 | onOpenChange,
15 | question,
16 | }: LLMSearchDialogProps) => {
17 | return (
18 |
23 | );
24 | };
25 |
26 | export default LLMSearchDialog;
27 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/logo.tsx:
--------------------------------------------------------------------------------
1 | import Image from "next/image";
2 |
3 | const Logo = () => {
4 | return (
5 |
6 |
7 |
8 | );
9 | };
10 |
11 | export default Logo;
12 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/new-folder-button.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { AiOutlineFolderAdd } from "react-icons/ai";
4 | import { Dialog, DialogTrigger } from "@/components/ui/dialog";
5 | import NewFolderForm from "@/components/new-folder-form";
6 | import { useState } from "react";
7 |
8 | type NewFolderButtonProps = {
9 | closeDropdownMenu: () => void;
10 | };
11 |
12 | const NewFolderButton = ({ closeDropdownMenu }: NewFolderButtonProps) => {
13 | const [dialogOpen, setDialogOpen] = useState(false);
14 |
15 | const closeDialog = () => {
16 | setDialogOpen(false);
17 | };
18 |
19 | return (
20 |
32 | );
33 | };
34 |
35 | export default NewFolderButton;
36 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/refresh.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { useRouter } from "next/navigation";
4 | import { useEffect } from "react";
5 |
6 | type RefreshProps = {
7 | intervalSecond: number;
8 | };
9 |
10 | const Refresh = ({ intervalSecond }: RefreshProps) => {
11 | const router = useRouter();
12 |
13 | useEffect(() => {
14 | const id = setInterval(() => {
15 | console.log("refreshed");
16 | router.refresh();
17 | }, intervalSecond * 1000);
18 | return () => clearInterval(id);
19 | }, []);
20 |
21 | return <>;
22 | };
23 |
24 | export default Refresh;
25 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/screen.tsx:
--------------------------------------------------------------------------------
1 | import Header from "@/components/header";
2 |
3 | type ScreenProps = {
4 | children: React.ReactNode;
5 | };
6 |
7 | const Screen = ({ children }: ScreenProps) => {
8 | return (
9 | <>
10 |
11 | {children}
12 |
13 | );
14 | };
15 |
16 | export default Screen;
17 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/search-content-text.tsx:
--------------------------------------------------------------------------------
1 | const SearchContentText = () => {
2 | return (
3 |
10 | );
11 | };
12 |
13 | export default SearchContentText;
14 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/search-content.tsx:
--------------------------------------------------------------------------------
1 | import ItemSelector from "@/components/item-selector";
2 | import ContentHeader from "@/components/content-header";
3 | import SearchContentText from "@/components/search-content-text";
4 | import SearchItemTable from "@/components/search-item-table";
5 | import { Suspense } from "react";
6 | import SkeletonItemTable from "./skeleton-item-table";
7 | import { randomUUID } from "crypto";
8 |
9 | type SearchContentProps = {
10 | queryText: string;
11 | };
12 |
13 | const SearchContent = ({ queryText }: SearchContentProps) => {
14 | return (
15 |
16 |
17 |
18 |
19 |
20 |
21 | }>
22 |
23 |
24 |
25 |
26 |
21 | 検索条件にヒットするファイル、
22 |
23 | フォルダが見つかりませんでした
24 |
25 |
26 | );
27 | }
28 |
29 | return (
30 |
31 |
32 |
33 | {items.map((item) => (
34 |
37 |
6 |
7 |
8 | );
9 | };
10 |
11 | export default SkeletonBreadcrumbFolder;
12 |
--------------------------------------------------------------------------------
/appdev_genai_googlecloud/src/knowledge-drive/src/components/skeleton-breadcrumb.tsx:
--------------------------------------------------------------------------------
1 | import { MdKeyboardArrowRight } from "react-icons/md";
2 | import SkeletonBreadcrumbFolder from "./skeleton-breadcrumb-folder";
3 |
4 | const SkeletonBreadcrumb = () => {
5 | return (
6 |
7 |
8 |
9 |
13 |
14 |
18 |
19 |
20 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
4 |
5 |
名前
6 |
オーナー
7 |
最終更新（自分）
8 |
ファイルサイズ
9 |
︙
10 |
11 |
5 |
6 |
7 |
候補リスト
8 |
9 |
16 |
17 |
22 | {title && {title}}
23 | {description && (
24 | {description}
25 | )}
26 |
27 | {action}
28 |
29 |
30 | )
31 | })}
32 |
33 |
34 | )
35 | }
36 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/api/auth.ts:
--------------------------------------------------------------------------------
1 | export const signUpApi = async (request: SignUpRequest, csrfToken: string) => {
2 | await fetch("/api/register", {
3 | method: "POST",
4 | headers: {
5 | "X-CSRF-Token": csrfToken,
6 | },
7 | body: JSON.stringify(request),
8 | });
9 | };
10 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/firebase/admin.ts:
--------------------------------------------------------------------------------
1 | import {
2 | initializeApp,
3 | applicationDefault,
4 | AppOptions,
5 | getApp,
6 | App,
7 | } from "firebase-admin/app";
8 |
9 | const ADMIN = "ADMIN";
10 | const firebaseAdminConfig: AppOptions = {
11 | credential: applicationDefault(),
12 | };
13 |
14 | const initialize = () => {
15 | let adminApp: App;
16 | try {
17 | adminApp = getApp(ADMIN);
18 | } catch (e) {
19 | adminApp = initializeApp(firebaseAdminConfig, ADMIN);
20 | }
21 | return adminApp;
22 | };
23 |
24 | const firebaseAdminApp = initialize();
25 |
26 | export default firebaseAdminApp;
27 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/firebase/auth.ts:
--------------------------------------------------------------------------------
1 | import {
2 | signInWithEmailAndPassword,
3 | getAuth,
4 | UserCredential,
5 | } from "firebase/auth";
6 | import firebaseClientApp from "./client";
7 | import { FirebaseError } from "firebase/app";
8 | import { signUpApi } from "../api/auth";
9 |
10 | const auth = getAuth(firebaseClientApp);
11 |
12 | export const signIn = async ({
13 | email,
14 | password,
15 | }: SignInRequest): Promise => {
16 | try {
17 | const user = await signInWithEmailAndPassword(auth, email, password);
18 | return user;
19 | } catch (e) {
20 | if (e instanceof FirebaseError) {
21 | console.error(e.code);
22 | throw e;
23 | }
24 | throw e;
25 | }
26 | };
27 |
28 | export const signUp = async (
29 | request: SignUpRequest,
30 | csrfToken: string
31 | ): Promise => {
32 | try {
33 | await signUpApi(request, csrfToken);
34 | const user = await signInWithEmailAndPassword(
35 | auth,
36 | request.email,
37 | request.password
38 | );
39 | return user;
40 | } catch (e) {
41 | if (e instanceof FirebaseError) {
42 | console.error(e.code);
43 | throw e;
44 | }
45 | throw e;
46 | }
47 | };
48 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/firebase/client.ts:
--------------------------------------------------------------------------------
1 | import {
2 | initializeApp,
3 | FirebaseOptions,
4 | getApp,
5 | FirebaseApp,
6 | } from "firebase/app";
7 |
8 | const CLIENT = "CLIENT";
9 |
10 | const firebaseClientConfig: FirebaseOptions = {
11 | apiKey: process.env.NEXT_PUBLIC_FIREBASE_API_KEY,
12 | authDomain: process.env.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN,
13 | projectId: process.env.NEXT_PUBLIC_FIREBASE_PROJECT_ID,
14 | storageBucket: process.env.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET,
15 | messagingSenderId: process.env.NEXT_PUBLIC_FIREBASE_MESSAGING_ID,
16 | appId: process.env.NEXT_PUBLIC_FIREBASE_APP_ID,
17 | };
18 |
19 | const initialize = () => {
20 | let clientApp: FirebaseApp;
21 | try {
22 | clientApp = getApp(CLIENT);
23 | } catch (e) {
24 | clientApp = initializeApp(firebaseClientConfig, CLIENT);
25 | }
26 | return clientApp;
27 | };
28 |
29 | const firebaseClientApp = initialize();
30 |
31 | export default firebaseClientApp;
32 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/firebase/firestore.ts:
--------------------------------------------------------------------------------
1 | import firebaseClientApp from "./client";
2 | import {
3 | getFirestore,
4 | addDoc,
5 | serverTimestamp,
6 | collection,
7 | setDoc,
8 | doc,
9 | } from "firebase/firestore";
10 |
11 | const firestore = getFirestore(firebaseClientApp);
12 |
13 | type Folder = {
14 | name: string;
15 | parent: string;
16 | uid: string;
17 | };
18 |
19 | type File = {
20 | id: string;
21 | name: string;
22 | parent: string;
23 | uid: string;
24 | size: number;
25 | url: string;
26 | };
27 |
28 | export const createFolder = async ({ name, parent, uid }: Folder) => {
29 | await addDoc(collection(firestore, "users", uid, "items"), {
30 | name: name,
31 | parent: parent,
32 | isFolder: true,
33 | timestamp: serverTimestamp(),
34 | size: 0,
35 | });
36 | };
37 |
38 | export const createFile = async ({
39 | id,
40 | name,
41 | parent,
42 | uid,
43 | size,
44 | url,
45 | }: File) => {
46 | await setDoc(doc(firestore, "users", uid, "items", id), {
47 | name: name,
48 | parent: parent,
49 | isFolder: false,
50 | timestamp: serverTimestamp(),
51 | size: size,
52 | url: url,
53 | });
54 | };
55 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/lib/utils.ts:
--------------------------------------------------------------------------------
1 | import { type ClassValue, clsx } from "clsx";
2 | import { twMerge } from "tailwind-merge";
3 |
4 | export function cn(...inputs: ClassValue[]) {
5 | return twMerge(clsx(inputs));
6 | }
7 |
8 | export const printFilesize = (filesize: number) => {
9 | if (filesize >= 1024 * 1024 * 1024)
10 | return (filesize / (1024 * 1024 * 1024)).toFixed(2).toString() + " GB";
11 | if (filesize >= 1024 * 1024)
12 | return (filesize / (1024 * 1024)).toFixed(2).toString() + " MB";
13 | if (filesize >= 1024) return (filesize / 1024).toFixed(2).toString() + " KB";
14 | return filesize.toString() + " B";
15 | };
16 |
17 | export const printTimestamp = (timestamp: Date) => {
18 | if (!timestamp) return "";
19 | return (
20 | timestamp.getFullYear().toString() +
21 | "年" +
22 | (timestamp.getMonth() + 1).toString() +
23 | "月" +
24 | timestamp.getDate().toString() +
25 | "日"
26 | );
27 | };
28 |
29 | export const getFolderId = (pathname: string) => {
30 | if (pathname === "/drive") return "ROOT_FOLDER";
31 | return pathname.replace("/folders", "");
32 | };
33 |
34 | export const isSearchPage = (pathname: string) => {
35 | return pathname === "/search";
36 | };
37 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/middleware.ts:
--------------------------------------------------------------------------------
1 | import csrf from "edge-csrf";
2 | import { NextResponse } from "next/server";
3 | import type { NextRequest } from "next/server";
4 |
5 | // initalize protection function
6 | const csrfProtect = csrf({
7 | cookie: {
8 | secure: process.env.NODE_ENV === "production",
9 | },
10 | excludePathPrefixes: ["/api/search"],
11 | });
12 |
13 | export async function middleware(request: NextRequest) {
14 | const response = NextResponse.next();
15 |
16 | // csrf protection
17 | const csrfError = await csrfProtect(request, response);
18 |
19 | // check result
20 | if (csrfError) {
21 | return new NextResponse("invalid csrf token", { status: 403 });
22 | }
23 |
24 | return response;
25 | }
26 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/next.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('next').NextConfig} */
2 | const nextConfig = {
3 | output: "standalone",
4 | images: {
5 | remotePatterns: [
6 | {
7 | protocol: "https",
8 | hostname: "upload.wikimedia.org",
9 | port: "",
10 | },
11 | ],
12 | },
13 | };
14 |
15 | module.exports = nextConfig;
16 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/public/images/check.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_with_generative_ai/src/knowledge-drive/public/images/check.png
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/public/images/error.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_with_generative_ai/src/knowledge-drive/public/images/error.png
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/public/images/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_with_generative_ai/src/knowledge-drive/public/images/logo.png
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/public/images/notfound.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_with_generative_ai/src/knowledge-drive/public/images/notfound.png
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/public/images/placeholder.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/appdev_with_generative_ai/src/knowledge-drive/public/images/placeholder.png
--------------------------------------------------------------------------------
/appdev_with_generative_ai/src/knowledge-drive/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es5",
4 | "lib": ["dom", "dom.iterable", "esnext"],
5 | "allowJs": true,
6 | "skipLibCheck": true,
7 | "strict": true,
8 | "noEmit": true,
9 | "esModuleInterop": true,
10 | "module": "esnext",
11 | "moduleResolution": "bundler",
12 | "resolveJsonModule": true,
13 | "isolatedModules": true,
14 | "jsx": "preserve",
15 | "incremental": true,
16 | "plugins": [
17 | {
18 | "name": "next"
19 | }
20 | ],
21 | "paths": {
22 | "@/*": ["./*"]
23 | }
24 | },
25 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
26 | "exclude": ["node_modules"]
27 | }
28 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/artifactregistry.tf:
--------------------------------------------------------------------------------
1 | resource "time_sleep" "wait_2_minutes_for_artifact_registry" {
2 | create_duration = "2m"
3 | }
4 |
5 | resource "google_artifact_registry_repository" "drive_repo" {
6 | location = var.region
7 | repository_id = "drive-repo"
8 | description = "Docker repository for knowledge drive"
9 | format = "DOCKER"
10 | depends_on = [
11 | time_sleep.wait_2_minutes_for_artifact_registry,
12 | null_resource.update_dot_env_for_knowledge_drive
13 | ]
14 | }
15 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/firebase_config.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | CONFIG_FILE=firebase-config.json
4 | ENV_PATH=../src/knowledge-drive
5 |
6 | function reset_configuration() {
7 | env_key=$1
8 | sed -i -e "s/$env_key=.*$/$env_key=/g" $ENV_PATH/.env
9 | }
10 |
11 | function update_env() {
12 | json_key=$1
13 | env_key=$2
14 | reset_configuration $env_key
15 | json_value=$(cat $CONFIG_FILE | jq ".$json_key")
16 | sed -i -e "s/$env_key=$/$env_key=$json_value/g" $ENV_PATH/.env
17 | }
18 |
19 | update_env projectId NEXT_PUBLIC_FIREBASE_PROJECT_ID
20 | update_env appId NEXT_PUBLIC_FIREBASE_APP_ID
21 | update_env storageBucket NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET
22 | update_env apiKey NEXT_PUBLIC_FIREBASE_API_KEY
23 | update_env authDomain NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN
24 | update_env messagingSenderId NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID
25 |
26 | echo "Updated configuration..."
27 | echo
28 | cat $ENV_PATH/.env
29 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/firestore.rules:
--------------------------------------------------------------------------------
1 | rules_version = '2';
2 | service cloud.firestore {
3 | match /databases/{database}/documents {
4 | match /users/{userId} {
5 | allow read: if request.auth != null
6 | && request.auth.uid == userId;
7 | match /items/{itemId} {
8 | allow read, write: if request.auth != null
9 | && request.auth.uid == userId;
10 | }
11 | }
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/iam.tf:
--------------------------------------------------------------------------------
1 | resource "google_service_account" "knowledge_drive" {
2 | account_id = "knowledge-drive"
3 | }
4 |
5 | resource "google_project_iam_member" "knowledge_drive" {
6 | project = var.project_id
7 | for_each = toset([
8 | "roles/firebase.sdkAdminServiceAgent",
9 | "roles/firebaseauth.admin",
10 | "roles/iam.serviceAccountTokenCreator",
11 | ])
12 | role = each.key
13 | member = "serviceAccount:${google_service_account.knowledge_drive.email}"
14 | depends_on = [
15 | google_service_account.knowledge_drive
16 | ]
17 | }
18 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/main.tf:
--------------------------------------------------------------------------------
1 | terraform {
2 | required_providers {
3 | google = {
4 | source = "hashicorp/google"
5 | version = "5.8.0"
6 | }
7 | google-beta = {
8 | source = "hashicorp/google-beta"
9 | version = "5.8.0"
10 | }
11 | }
12 | }
13 |
14 | provider "google" {
15 | project = var.project_id
16 | region = var.region
17 | }
18 |
19 | provider "google-beta" {
20 | project = var.project_id
21 | region = var.region
22 | }
23 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/output.tf:
--------------------------------------------------------------------------------
1 | output "Service_URL" {
2 | value = google_cloud_run_service.knowledge_drive.status[0].url
3 | }
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/storage.rules:
--------------------------------------------------------------------------------
1 | rules_version = '2';
2 | service firebase.storage {
3 | match /b/{bucket}/o {
4 | match /files/{userId}/{itemId} {
5 | allow read, write: if request.auth != null
6 | && request.auth.uid == userId;
7 | }
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/appdev_with_generative_ai/tf/variables.tf:
--------------------------------------------------------------------------------
1 | variable "project_id" {}
2 |
3 | variable "region" {
4 | default = "asia-northeast1"
5 | }
6 |
7 | variable "zone" {
8 | default = "asia-northeast1-c"
9 | }
10 |
--------------------------------------------------------------------------------
/data_analytics/README.md:
--------------------------------------------------------------------------------
1 | # GCP Hands on materials for Data Analytics
2 |
3 | **This is not an officially supported Google product**. This directory
4 | contains some scripts that are used to teach GCP beginners how
5 | to use GCP's data analytics products in more efficient way.
6 |
--------------------------------------------------------------------------------
/data_analytics/sample.csv:
--------------------------------------------------------------------------------
1 | Taro,20,Osaka
2 | Hanako,25,Tokyo
3 | Jiro,30,Aichi
4 |
--------------------------------------------------------------------------------
/data_analytics/sample.sql:
--------------------------------------------------------------------------------
1 | # 分析1のクエリ
2 | SELECT
3 | COUNT(DISTINCT station_id) as cnt
4 | FROM
5 | `bigquery-public-data.new_york_citibike.citibike_stations`
6 |
7 |
8 | # 分析2のクエリ
9 | SELECT
10 | COUNT(station_id) as cnt
11 | FROM
12 | `bigquery-public-data.new_york_citibike.citibike_stations`
13 | WHERE
14 | is_installed = TRUE
15 | AND is_renting = TRUE
16 | AND is_returning = TRUE
17 |
18 |
19 | # 分析3のクエリ
20 | SELECT
21 | usertype,
22 | gender,
23 | COUNT(gender) AS cnt
24 | FROM
25 | `bigquery-public-data.new_york_citibike.citibike_trips`
26 | GROUP BY
27 | usertype,
28 | gender
29 | ORDER BY
30 | cnt DESC
31 |
32 |
33 | # 分析4のクエリ
34 | SELECT
35 | start_station_name,
36 | end_station_name,
37 | COUNT(end_station_name) AS cnt
38 | FROM
39 | `bigquery-public-data.new_york_citibike.citibike_trips`
40 | GROUP BY
41 | start_station_name,
42 | end_station_name
43 | ORDER BY
44 | cnt DESC
45 |
--------------------------------------------------------------------------------
/fundamental/gce/index.html:
--------------------------------------------------------------------------------
1 |
13 |
14 |
17 |
18 |
19 |
20 | Hello World! running on `hostname`
21 |
22 |
23 |
--------------------------------------------------------------------------------
/fundamental/gke/helloNode/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM node:12.4.0
2 | EXPOSE 8080
3 | COPY server.js /server.js
4 | CMD node server.js
5 |
--------------------------------------------------------------------------------
/fundamental/gke/helloNode/hello-node-deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | name: hello-node
6 | name: hello-node
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | name: hello-node
12 | template:
13 | metadata:
14 | labels:
15 | name: hello-node
16 | spec:
17 | containers:
18 | - image: asia.gcr.io//hello-node:v1
19 | name: hello-node
20 | ports:
21 | - containerPort: 8080
22 |
--------------------------------------------------------------------------------
/fundamental/gke/helloNode/hello-node-service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | labels:
5 | name: hello-node
6 | name: hello-node
7 | spec:
8 | ports:
9 | - port: 8080
10 | selector:
11 | name: hello-node
12 | type: LoadBalancer
13 |
--------------------------------------------------------------------------------
/fundamental/gke/helloNode/server.js:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | /*
18 | * This file is used for GCP handson.
19 | */
20 |
21 | var http = require('http');
22 | var handleRequest = function(request, response) {
23 | var os = require('os');
24 | var hostname = os.hostname();
25 | response.writeHead(200);
26 | response.end("Hello World! == " + hostname + "
\n");
27 | }
28 | var www = http.createServer(handleRequest);
29 | www.listen(8080);
30 |
--------------------------------------------------------------------------------
/fundamental/readme.md:
--------------------------------------------------------------------------------
1 | # GCP Hands on materials for Fundamental
2 |
3 | **This is not an officially supported Google product**. This directory
4 | contains some files that are used to teach GCP beginners how
5 | to use GCP's products.
6 |
--------------------------------------------------------------------------------
/gke-ai/lab-01/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM pytorch/pytorch:2.3.0-cuda12.1-cudnn8-runtime
2 |
3 | WORKDIR /app
4 |
5 | RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
6 |
7 | COPY requirements.txt .
8 | RUN pip install --no-cache-dir -r requirements.txt
9 |
10 | COPY main.py .
11 |
12 | EXPOSE 8080
13 |
14 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-keep-alive", "60", "--workers", "1"]
15 |
--------------------------------------------------------------------------------
/gke-ai/lab-01/requirements.txt:
--------------------------------------------------------------------------------
1 | fastapi
2 | uvicorn[standard]
3 | torch --index-url https://download.pytorch.org/whl/cu121
4 | transformers
5 | peft
6 | accelerate
7 | trl
8 | datasets
9 | sentencepiece
10 | protobuf
--------------------------------------------------------------------------------
/gke-ai/lab-01/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: gemma3-server-service
5 | spec:
6 | type: LoadBalancer
7 | selector:
8 | app: gemma3-server # deployment.yamlのラベルと一致
9 | ports:
10 | - protocol: TCP
11 | port: 80
12 | targetPort: 8080
--------------------------------------------------------------------------------
/gke-ai/lab-02/.gitignore:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/gke-ai/lab-02/.gitignore
--------------------------------------------------------------------------------
/gke-ai/lab-02/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM pytorch/pytorch:2.3.0-cuda12.1-cudnn8-runtime
2 |
3 | WORKDIR /app
4 |
5 | # 必要なツールをインストール (git は Hugging Face モデルダウンロードに必要)
6 | RUN apt-get update && apt-get install -y \
7 | git \
8 | && rm -rf /var/lib/apt/lists/*
9 |
10 | # requirements.txt をコピーしてライブラリをインストール
11 | COPY requirements_finetune.txt .
12 | RUN pip install --no-cache-dir -r requirements_finetune.txt
13 |
14 | # 学習スクリプトをコピー
15 | COPY finetune_gozaru.py .
16 |
17 | # スクリプトを実行可能にする
18 | RUN chmod +x finetune_gozaru.py
19 |
20 | # コンテナ起動時に学習スクリプトを実行
21 | ENTRYPOINT ["python3", "finetune_gozaru.py"]
22 |
--------------------------------------------------------------------------------
/gke-ai/lab-02/requirements_finetune.txt:
--------------------------------------------------------------------------------
1 | trl==0.17.0
2 | peft==0.10.0
3 | tensorboard
4 | sentencepiece
5 | protobuf<4
6 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM pytorch/pytorch:2.3.0-cuda12.1-cudnn8-runtime
2 |
3 | WORKDIR /app
4 |
5 | # 必要なツールをインストール (git は Hugging Face モデルダウンロードに必要)
6 | RUN apt-get update && apt-get install -y \
7 | git \
8 | && rm -rf /var/lib/apt/lists/*
9 |
10 | # requirements.txtをコピーしてライブラリをインストール
11 | COPY requirements_gozaru.txt .
12 | RUN pip install --no-cache-dir -r requirements_gozaru.txt
13 |
14 | # アプリケーションコードをコピー
15 | COPY main_gozaru.py .
16 |
17 | # ポート8080を公開
18 | EXPOSE 8080
19 |
20 | # アプリケーションの起動コマンド
21 | CMD ["uvicorn", "main_gozaru:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-keep-alive", "60", "--workers", "1"]
22 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/gateway.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: gateway.networking.k8s.io/v1
2 | kind: Gateway
3 | metadata:
4 | name: inference-gateway
5 | spec:
6 | gatewayClassName: gke-l7-regional-external-managed
7 | listeners:
8 | - name: http
9 | protocol: HTTP
10 | port: 80
11 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/gozaru-gemma-pool-epp-svc.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: gozaru-gemma-pool-epp # ★ InferencePool から参照する名前
5 | spec:
6 | selector:
7 | app: gozaru-gemma-server # モデル Pod のラベル
8 | ports:
9 | - name: grpc
10 | port: 9002 # ここは自由 (9002 がデフォルト推奨)
11 | targetPort: 9002
12 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/httproute.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: gateway.networking.k8s.io/v1
2 | kind: HTTPRoute
3 | metadata:
4 | name: gemma-route
5 | spec:
6 | parentRefs:
7 | - name: inference-gateway
8 | rules:
9 | # ござる LoRA
10 | - matches:
11 | - headers:
12 | - name: model
13 | value: gemma-gozaru
14 | backendRefs:
15 | - group: inference.networking.x-k8s.io
16 | kind: InferencePool
17 | name: gozaru-gemma-pool
18 | weight: 100
19 | # ベース
20 | - matches:
21 | - headers:
22 | - name: model
23 | value: gemma-base
24 | backendRefs:
25 | - group: inference.networking.x-k8s.io
26 | kind: InferencePool
27 | name: gozaru-gemma-pool
28 | weight: 100
29 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/inferencemodel.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: inference.networking.x-k8s.io/v1alpha2
2 | kind: InferenceModel
3 | metadata:
4 | name: gemma-gozaru
5 | spec:
6 | modelName: tarota0226/gemma-gozaru-adapter # LoRA
7 | criticality: Standard
8 | poolRef:
9 | name: gozaru-gemma-pool
10 | ---
11 | apiVersion: inference.networking.x-k8s.io/v1alpha2
12 | kind: InferenceModel
13 | metadata:
14 | name: gemma-base
15 | spec:
16 | modelName: google/gemma-3-4b-it # ベース
17 | criticality: Critical
18 | poolRef:
19 | name: gozaru-gemma-pool
20 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/inferencepool.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: inference.networking.x-k8s.io/v1alpha2
2 | kind: InferencePool
3 | metadata:
4 | name: gozaru-gemma-pool
5 | spec:
6 | # --- Pod を選ぶラベル (単なる key:value の map) ---
7 | selector:
8 | app: gozaru-gemma-server
9 |
10 | # --- Pod 内で公開しているポート番号 ---
11 | targetPortNumber: 8080
12 |
13 | # --- Gateway が呼び出す拡張サービス (固定値で OK) ---
14 | extensionRef:
15 | name: gke # Service 名。変更しない
16 | kind: Service # 既定値なので省略可
17 | group: "" # Core API group（省略可）
18 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/requirements_gozaru.txt:
--------------------------------------------------------------------------------
1 | fastapi
2 | uvicorn[standard]
3 | torch --index-url https://download.pytorch.org/whl/cu121
4 | transformers
5 | peft
6 | accelerate
7 | trl
8 | datasets
9 | bitsandbytes
10 | sentencepiece
11 | protobuf
12 |
--------------------------------------------------------------------------------
/gke-ai/lab-03/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: gozaru-gemma-service
5 | spec:
6 | type: LoadBalancer
7 | selector:
8 | app: gozaru-gemma-server
9 | ports:
10 | - protocol: TCP
11 | port: 80
12 | targetPort: 8080
13 |
--------------------------------------------------------------------------------
/gke-basic/lab-01/gateway/gateway.yaml:
--------------------------------------------------------------------------------
1 | kind: Gateway
2 | apiVersion: gateway.networking.k8s.io/v1beta1
3 | metadata:
4 | name: external-http
5 | spec:
6 | gatewayClassName: gke-l7-gxlb
7 | listeners:
8 | - name: http
9 | protocol: HTTP
10 | port: 80
11 | addresses:
12 | - type: NamedAddress
13 | value: gatewayip
--------------------------------------------------------------------------------
/gke-basic/lab-01/gateway/httproute.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: gateway.networking.k8s.io/v1beta1
2 | kind: HTTPRoute
3 | metadata:
4 | name: frontend-route
5 | spec:
6 | parentRefs:
7 | - name: external-http
8 | hostnames:
9 | - "x-x-x-x.nip.io"
10 | rules:
11 | - matches:
12 | - path:
13 | value: /
14 | backendRefs:
15 | - name: frontend
16 | port: 80
17 |
--------------------------------------------------------------------------------
/gke-basic/lab-02/balloon-deploy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: balloon-deploy
5 | spec:
6 | replicas: 10
7 | selector:
8 | matchLabels:
9 | app: balloon
10 | template:
11 | metadata:
12 | labels:
13 | app: balloon
14 | spec:
15 | priorityClassName: balloon-priority
16 | terminationGracePeriodSeconds: 0
17 | containers:
18 | - name: busybox
19 | image: busybox:latest
20 | command: ["sleep"]
21 | args: ["infinity"]
22 | resources:
23 | requests:
24 | cpu: 200m
25 | memory: 250Mi
--------------------------------------------------------------------------------
/gke-basic/lab-02/balloon-priority.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: scheduling.k8s.io/v1
2 | kind: PriorityClass
3 | metadata:
4 | name: balloon-priority
5 | value: -10
6 | preemptionPolicy: Never
7 | globalDefault: false
8 | description: "Balloon pod priority."
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/.python-version:
--------------------------------------------------------------------------------
1 | >= 3.10
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/Procfile:
--------------------------------------------------------------------------------
1 | web: gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'gcr.io/k8s-skaffold/pack'
3 | entrypoint: 'pack'
4 | args: ['build', '--builder=gcr.io/buildpacks/builder', '--publish', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/gke-dojo/gke-dojo-app:v1']
5 |
6 |
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/clouddeploy.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | apiVersion: deploy.cloud.google.com/v1beta1
3 | kind: DeliveryPipeline
4 | metadata:
5 | name: gke-dojo
6 | description: gke-dojo
7 | serialPipeline:
8 | stages:
9 | - targetId: staging
10 | - targetId: production
11 | ---
12 | apiVersion: deploy.cloud.google.com/v1beta1
13 | kind: Target
14 | metadata:
15 | name: staging
16 | description: Staging Environment
17 | gke:
18 | cluster: projects/PROJECT_ID/locations/asia-northeast1/clusters/gke-dojo-cluster
19 | ---
20 | apiVersion: deploy.cloud.google.com/v1beta1
21 | kind: Target
22 | metadata:
23 | name: production
24 | description: Production Environment
25 | gke:
26 | cluster: projects/PROJECT_ID/locations/asia-northeast1/clusters/gke-dojo-cluster-prod
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/k8s/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: gke-dojo
5 | spec:
6 | selector:
7 | matchLabels:
8 | app: gke-dojo
9 | template:
10 | metadata:
11 | labels:
12 | app: gke-dojo
13 | spec:
14 | containers:
15 | - name: gke-dojo
16 | image: gke-dojo
17 | ports:
18 | - containerPort: 8080
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/k8s/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: gke-dojo-service
5 | spec:
6 | ports:
7 | - port: 80
8 | targetPort: 8080
9 | type: LoadBalancer
10 | selector:
11 | app: gke-dojo
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/main.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, render_template_string
2 |
3 | app = Flask(__name__)
4 |
5 | @app.route('/')
6 | def index():
7 | template = """
8 |
9 |
10 |
11 | GKE Dojo Handson 2023
12 |
24 |
25 |
26 |
27 | Welcome to gke-dojo 2023
28 |
29 |
30 |
31 | """
32 |
33 | return render_template_string(template)
34 |
35 | if __name__ == '__main__':
36 | app.run(host='0.0.0.0', port=8080)
37 |
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/requirements.txt:
--------------------------------------------------------------------------------
1 | # https://pypi.org/project/flask/
2 | Flask >= 2.2
3 |
4 | # https://pypi.org/project/gunicorn/
5 | gunicorn==20.1.0
6 |
--------------------------------------------------------------------------------
/gke-basic/lab-ex01/skaffold.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: skaffold/v2beta16
2 | kind: Config
3 | deploy:
4 | kubectl:
5 | manifests: ["k8s/*.yaml"]
--------------------------------------------------------------------------------
/gke-basics-2025/lab-01/app/app.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: helloweb
5 | labels:
6 | app: hello
7 | spec:
8 | selector:
9 | matchLabels:
10 | app: hello
11 | tier: web
12 | template:
13 | metadata:
14 | labels:
15 | app: hello
16 | tier: web
17 | spec:
18 | containers:
19 | - name: hello-app
20 | image: us-docker.pkg.dev/google-samples/containers/gke/hello-app:1.0
21 | ports:
22 | - containerPort: 8080
23 | resources:
24 | requests:
25 | cpu: 200m
26 | memory: 256Mi
27 | limits:
28 | cpu: 500m
29 | memory: 512Mi
--------------------------------------------------------------------------------
/gke-basics-2025/lab-01/app/svc.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: helloweb-lb
5 | labels:
6 | app: hello
7 | spec:
8 | type: LoadBalancer
9 | selector:
10 | app: hello
11 | tier: web
12 | ports:
13 | - port: 80
14 | targetPort: 8080
15 | protocol: TCP
--------------------------------------------------------------------------------
/gke-basics-2025/lab-02/balloon-deploy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: balloon-deploy
5 | spec:
6 | replicas: 4
7 | selector:
8 | matchLabels:
9 | app: balloon
10 | template:
11 | metadata:
12 | labels:
13 | app: balloon
14 | spec:
15 | priorityClassName: balloon-priority
16 | terminationGracePeriodSeconds: 0
17 | containers:
18 | - name: busybox
19 | image: busybox:latest
20 | command: ["sleep"]
21 | args: ["infinity"]
22 | resources:
23 | requests:
24 | cpu: 200m
25 | memory: 250Mi
--------------------------------------------------------------------------------
/gke-basics-2025/lab-02/balloon-priority.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: scheduling.k8s.io/v1
2 | kind: PriorityClass
3 | metadata:
4 | name: balloon-priority
5 | value: -10
6 | preemptionPolicy: Never
7 | globalDefault: false
8 | description: "Balloon pod priority."
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/.python-version:
--------------------------------------------------------------------------------
1 | >= 3.10
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/Procfile:
--------------------------------------------------------------------------------
1 | web: gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'gcr.io/k8s-skaffold/pack'
3 | entrypoint: 'pack'
4 | args: ['build', '--builder=gcr.io/buildpacks/builder', '--publish', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/gke-dojo/gke-dojo-app:v1']
5 |
6 |
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/clouddeploy.yaml:
--------------------------------------------------------------------------------
1 | ---
2 | apiVersion: deploy.cloud.google.com/v1beta1
3 | kind: DeliveryPipeline
4 | metadata:
5 | name: gke-dojo
6 | description: gke-dojo
7 | serialPipeline:
8 | stages:
9 | - targetId: staging
10 | - targetId: production
11 | ---
12 | apiVersion: deploy.cloud.google.com/v1beta1
13 | kind: Target
14 | metadata:
15 | name: staging
16 | description: Staging Environment
17 | gke:
18 | cluster: projects/PROJECT_ID/locations/asia-northeast1/clusters/gke-dojo-cluster
19 | ---
20 | apiVersion: deploy.cloud.google.com/v1beta1
21 | kind: Target
22 | metadata:
23 | name: production
24 | description: Production Environment
25 | gke:
26 | cluster: projects/PROJECT_ID/locations/asia-northeast1/clusters/gke-dojo-cluster-prod
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/k8s/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: gke-dojo
5 | spec:
6 | selector:
7 | matchLabels:
8 | app: gke-dojo
9 | template:
10 | metadata:
11 | labels:
12 | app: gke-dojo
13 | spec:
14 | containers:
15 | - name: gke-dojo
16 | image: gke-dojo
17 | ports:
18 | - containerPort: 8080
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/k8s/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: gke-dojo-service
5 | spec:
6 | ports:
7 | - port: 80
8 | targetPort: 8080
9 | type: LoadBalancer
10 | selector:
11 | app: gke-dojo
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/main.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, render_template_string
2 |
3 | app = Flask(__name__)
4 |
5 | @app.route('/')
6 | def index():
7 | template = """
8 |
9 |
10 |
11 | GKE Dojo Handson 2023
12 |
24 |
25 |
26 |
27 | Welcome to gke-dojo 2025
28 |
29 |
30 |
31 | """
32 |
33 | return render_template_string(template)
34 |
35 | if __name__ == '__main__':
36 | app.run(host='0.0.0.0', port=8080)
37 |
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/requirements.txt:
--------------------------------------------------------------------------------
1 | # https://pypi.org/project/flask/
2 | Flask >= 2.2
3 |
4 | # https://pypi.org/project/gunicorn/
5 | gunicorn==20.1.0
6 |
--------------------------------------------------------------------------------
/gke-basics-2025/lab-ex01/skaffold.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: skaffold/v2beta16
2 | kind: Config
3 | deploy:
4 | kubectl:
5 | manifests: ["k8s/*.yaml"]
--------------------------------------------------------------------------------
/machine_learning/README.md:
--------------------------------------------------------------------------------
1 | # GCP Hands on materials for Machine Learning
2 |
3 | **This is not an officially supported Google product**. This directory
4 | contains Colab-based hands-on materials that are used to teach GCP beginners how
5 | to use Google Cloud AI in more efficient way.
6 |
--------------------------------------------------------------------------------
/machine_learning/cloud_ai_building_blocks/speech-to-speech/README.md:
--------------------------------------------------------------------------------
1 | # Simple Speech-to-speech
2 |
3 | This directory contains simple speech-to-speech demo which uses Google Cloud
4 | Machine Learning APIs.
5 |
6 | 1. Record your voice
7 |
8 | ```bash
9 | bash voice_recorder.sh
10 | ```
11 |
12 | 2. Install required packages
13 |
14 | ```bash
15 | pip install -r requirements.txt
16 | ```
17 |
18 | 3. Execute speech-to-speech
19 |
20 | ```bash
21 | python speech-to-speech.py
22 | ```
23 |
24 | 4. Check output audio file with your music player
25 |
26 | An audio file should be saved as 'en-sample.mp3'.
27 |
--------------------------------------------------------------------------------
/machine_learning/cloud_ai_building_blocks/speech-to-speech/requirements.txt:
--------------------------------------------------------------------------------
1 | google-api-python-client
2 |
--------------------------------------------------------------------------------
/machine_learning/cloud_ai_building_blocks/speech-to-speech/voice_recorder.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | read -p "エンターキーを押下すると音声録音(10秒間)が始まります" rec
4 | if [ -z $rec ]; then
5 | rec --channels=1 --bits=16 --rate=16000 ja-sample.flac trim 0 10
6 | fi
7 |
8 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM gcr.io/YOUR-PROJECT-ID/tensorrtserver_client
2 |
3 | RUN pip install --upgrade locust
4 |
5 | COPY locust locust
6 | COPY data data
7 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/data/00001.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/machine_learning/ml_infrastructure/inference-server-performance/client/data/00001.jpg
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/data/00002.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/machine_learning/ml_infrastructure/inference-server-performance/client/data/00002.jpg
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/data/00003.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/machine_learning/ml_infrastructure/inference-server-performance/client/data/00003.jpg
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/deployment_master.yaml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: extensions/v1beta1
3 | kind: Deployment
4 | metadata:
5 | name: locust-master
6 | labels:
7 | name: locust-master
8 | spec:
9 | replicas: 1
10 | selector:
11 | matchLabels:
12 | app: locust-master
13 | template:
14 | metadata:
15 | labels:
16 | app: locust-master
17 | spec:
18 | containers:
19 | - name: locust-master
20 | image: gcr.io/YOUR-PROJECT-ID/locust_tester
21 | ports:
22 | - name: loc-master
23 | containerPort: 8089
24 | protocol: TCP
25 | - name: loc-master-p1
26 | containerPort: 5557
27 | protocol: TCP
28 | - name: loc-master-p2
29 | containerPort: 5558
30 | protocol: TCP
31 | command: ["locust","-f","locust/trtis_grpc_client.py"]
32 | args: ["--host", "CLUSTER-IP-TRTIS", "--master"]
33 | resources:
34 | requests:
35 | cpu: 200m
36 | env:
37 | - name: MODEL_NAME
38 | valueFrom:
39 | configMapKeyRef:
40 | name: locust-config
41 | key: model
42 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/deployment_slave.yaml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: extensions/v1beta1
3 | kind: Deployment
4 | metadata:
5 | name: locust-slave
6 | labels:
7 | name: locust-slave
8 | spec:
9 | replicas: 3
10 | selector:
11 | matchLabels:
12 | app: locust-slave
13 | template:
14 | metadata:
15 | labels:
16 | app: locust-slave
17 | spec:
18 | containers:
19 | - name: locust-slave
20 | image: gcr.io/YOUR-PROJECT-ID/locust_tester
21 | command: ["locust","-f","locust/trtis_grpc_client.py"]
22 | args: ["--slave", "--master-host=CLUSTER-IP-LOCUST-MASTER"]
23 | resources:
24 | requests:
25 | cpu: 100m
26 | env:
27 | - name: MODEL_NAME
28 | valueFrom:
29 | configMapKeyRef:
30 | name: locust-config
31 | key: model
32 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/client/service_master.yaml:
--------------------------------------------------------------------------------
1 |
2 | kind: Service
3 | apiVersion: v1
4 | metadata:
5 | name: locust-master
6 | labels:
7 | app: locust-master
8 | spec:
9 | ports:
10 | - name: loc-master
11 | port: 8089
12 | targetPort: 8089
13 | protocol: TCP
14 | - name: loc-master-p1
15 | port: 5557
16 | targetPort: 5557
17 | protocol: TCP
18 | - name: loc-master-p2
19 | port: 5558
20 | targetPort: 5558
21 | protocol: TCP
22 | selector:
23 | app: locust-master
24 | type: LoadBalancer
25 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM nvcr.io/nvidia/tensorflow:19.05-py3
2 |
3 | COPY scripts scripts
4 |
5 | ENTRYPOINT ["python3", "scripts/tensorrt-optimization.py"]
6 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/clusterRole.yml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: rbac.authorization.k8s.io/v1beta1
3 | kind: ClusterRole
4 | metadata:
5 | name: prometheus
6 | rules:
7 | - apiGroups: [""]
8 | resources:
9 | - nodes
10 | - nodes/proxy
11 | - services
12 | - endpoints
13 | - pods
14 | verbs: ["get", "list", "watch"]
15 | - apiGroups:
16 | - extensions
17 | resources:
18 | - ingresses
19 | verbs: ["get", "list", "watch"]
20 | - nonResourceURLs: ["/metrics"]
21 | verbs: ["get"]
22 | ---
23 | apiVersion: rbac.authorization.k8s.io/v1beta1
24 | kind: ClusterRoleBinding
25 | metadata:
26 | name: prometheus
27 | roleRef:
28 | apiGroup: rbac.authorization.k8s.io
29 | kind: ClusterRole
30 | name: prometheus
31 | subjects:
32 | - kind: ServiceAccount
33 | name: default
34 | namespace: monitoring
35 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/grafana-deployment.yml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: extensions/v1beta1
3 | kind: Deployment
4 | metadata:
5 | name: grafana-deployment
6 | namespace: monitoring
7 | spec:
8 | replicas: 1
9 | template:
10 | metadata:
11 | labels:
12 | app: grafana-server
13 | spec:
14 | containers:
15 | - name: grafana
16 | image: grafana/grafana:latest
17 | #args:
18 | # - "--config.file=/root/prometheus.yml"
19 | # - "--storage.tsdb.path=/prometheus/"
20 | ports:
21 | - containerPort: 3000
22 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/grafana-service.yml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: v1
3 | kind: Service
4 | metadata:
5 | name: grafana-service
6 | spec:
7 | selector:
8 | app: grafana-server
9 | type: LoadBalancer
10 | ports:
11 | - port: 8100
12 | targetPort: 3000
13 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/models/resnet/original/00001/README:
--------------------------------------------------------------------------------
1 | This model is created based on Cloud TPU Resnet-50 example. One difference from Cloud TPU Resnet-50 example is we change input_tensor signature from tf.string to Tensor shape [None, 224, 224, 3] with tf.float32.
2 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/original/config.pbtxt:
--------------------------------------------------------------------------------
1 | name: "original"
2 | platform: "tensorflow_savedmodel"
3 | max_batch_size: 64
4 | input {
5 | name: "input"
6 | data_type: TYPE_FP32
7 | format: FORMAT_NHWC
8 | dims: [ 224, 224, 3 ]
9 | }
10 | output {
11 | name: "probabilities"
12 | data_type: TYPE_FP32
13 | dims: 1000
14 | label_filename: "imagenet1k_labels.txt"
15 | }
16 | default_model_filename: "model"
17 | instance_group [
18 | {
19 | count: 1
20 | kind: KIND_GPU
21 | }
22 | ]
23 | dynamic_batching {
24 | preferred_batch_size: [ 64 ]
25 | max_queue_delay_microseconds: 20000
26 | }
27 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/prometheus-deployment.yml:
--------------------------------------------------------------------------------
1 | apiVersion: extensions/v1beta1
2 | kind: Deployment
3 | metadata:
4 | name: prometheus-deployment
5 | namespace: monitoring
6 | spec:
7 | replicas: 1
8 | template:
9 | metadata:
10 | labels:
11 | app: prometheus-server
12 | spec:
13 | containers:
14 | - name: prometheus
15 | image: prom/prometheus:latest
16 | args:
17 | - "--config.file=/etc/prometheus/prometheus.yml"
18 | - "--storage.tsdb.path=/prometheus/"
19 | ports:
20 | - containerPort: 9090
21 | volumeMounts:
22 | - name: prometheus-config-volume
23 | mountPath: /etc/prometheus
24 | - name: prometheus-storage-volume
25 | mountPath: /prometheus
26 | volumes:
27 | - name: prometheus-config-volume
28 | configMap:
29 | defaultMode: 420
30 | name: prometheus-server-conf
31 | - name: prometheus-storage-volume
32 | emptyDir: {}
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/prometheus-service.yml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: v1
3 | kind: Service
4 | metadata:
5 | name: prometheus-service
6 | spec:
7 | selector:
8 | app: prometheus-server
9 | type: ClusterIP
10 | ports:
11 | - port: 8080
12 | targetPort: 9090
13 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/prometheus.yaml:
--------------------------------------------------------------------------------
1 | # my global config
2 | global:
3 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
4 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
5 | # scrape_timeout is set to the global default (10s).
6 |
7 | # Alertmanager configuration
8 | alerting:
9 | alertmanagers:
10 | - static_configs:
11 | - targets:
12 | # - alertmanager:9093
13 |
14 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
15 | rule_files:
16 | # - "first_rules.yml"
17 | # - "second_rules.yml"
18 |
19 | # A scrape configuration containing exactly one endpoint to scrape:
20 | # Here it's Prometheus itself.
21 | scrape_configs:
22 | # The job name is added as a label `job=` to any timeseries scraped from this config.
23 | - job_name: 'prometheus'
24 |
25 | # metrics_path defaults to '/metrics'
26 | # scheme defaults to 'http'.
27 |
28 | static_configs:
29 | - targets: ['${CLUSTER_EXTERNAL_IP}:8002']
30 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/tftrt_fp16/config.pbtxt:
--------------------------------------------------------------------------------
1 | name: "tftrt_fp16"
2 | platform: "tensorflow_savedmodel"
3 | max_batch_size: 64
4 | input {
5 | name: "input"
6 | data_type: TYPE_FP32
7 | format: FORMAT_NHWC
8 | dims: [ 224, 224, 3 ]
9 | }
10 | output {
11 | name: "probabilities"
12 | data_type: TYPE_FP32
13 | dims: 1000
14 | label_filename: "imagenet1k_labels.txt"
15 | }
16 | default_model_filename: "model"
17 | instance_group [
18 | {
19 | count: 1
20 | kind: KIND_GPU
21 | }
22 | ]
23 | dynamic_batching {
24 | preferred_batch_size: [ 64 ]
25 | max_queue_delay_microseconds: 20000
26 | }
27 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/tftrt_fp16_bs8_count4/config.pbtxt:
--------------------------------------------------------------------------------
1 | name: "tftrt_fp16_bs8_count4"
2 | platform: "tensorflow_savedmodel"
3 | max_batch_size: 8
4 | input {
5 | name: "input"
6 | data_type: TYPE_FP32
7 | format: FORMAT_NHWC
8 | dims: [ 224, 224, 3 ]
9 | }
10 | output {
11 | name: "probabilities"
12 | data_type: TYPE_FP32
13 | dims: 1000
14 | label_filename: "imagenet1k_labels.txt"
15 | }
16 | default_model_filename: "model"
17 | instance_group [
18 | {
19 | count: 4
20 | kind: KIND_GPU
21 | }
22 | ]
23 | dynamic_batching {
24 | preferred_batch_size: [ 8 ]
25 | max_queue_delay_microseconds: 20000
26 | }
27 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/tftrt_fp32/config.pbtxt:
--------------------------------------------------------------------------------
1 | name: "tftrt_fp32"
2 | platform: "tensorflow_savedmodel"
3 | max_batch_size: 64
4 | input {
5 | name: "input"
6 | data_type: TYPE_FP32
7 | format: FORMAT_NHWC
8 | dims: [ 224, 224, 3 ]
9 | }
10 | output {
11 | name: "probabilities"
12 | data_type: TYPE_FP32
13 | dims: 1000
14 | label_filename: "imagenet1k_labels.txt"
15 | }
16 | default_model_filename: "model"
17 | instance_group [
18 | {
19 | count: 1
20 | kind: KIND_GPU
21 | }
22 | ]
23 | dynamic_batching {
24 | preferred_batch_size: [ 64 ]
25 | max_queue_delay_microseconds: 20000
26 | }
27 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/tftrt_int8_bs8_count4/config.pbtxt:
--------------------------------------------------------------------------------
1 | name: "tftrt_int8_bs8_count4"
2 | platform: "tensorflow_savedmodel"
3 | max_batch_size: 8
4 | input {
5 | name: "input"
6 | data_type: TYPE_FP32
7 | format: FORMAT_NHWC
8 | dims: [ 224, 224, 3 ]
9 | }
10 | output {
11 | name: "probabilities"
12 | data_type: TYPE_FP32
13 | dims: 1000
14 | label_filename: "imagenet1k_labels.txt"
15 | }
16 | default_model_filename: "model"
17 | instance_group [
18 | {
19 | count: 4
20 | kind: KIND_GPU
21 | }
22 | ]
23 | dynamic_batching {
24 | preferred_batch_size: [ 8 ]
25 | max_queue_delay_microseconds: 20000
26 | }
27 |
--------------------------------------------------------------------------------
/machine_learning/ml_infrastructure/inference-server-performance/server/trtis_service.yaml:
--------------------------------------------------------------------------------
1 |
2 | apiVersion: v1
3 | kind: Service
4 | metadata:
5 | labels:
6 | name: inference-server
7 | name: inference-server
8 | namespace: default
9 | spec:
10 | #externalTrafficPolicy: Cluster
11 | ports:
12 | - name: http-inference-server
13 | port: 8000
14 | protocol: TCP
15 | targetPort: 8000
16 | - name: grpc-inference-server
17 | port: 8001
18 | protocol: TCP
19 | targetPort: 8001
20 | - name: metrics-inference-server
21 | port: 8002
22 | protocol: TCP
23 | targetPort: 8002
24 | selector:
25 | app: inference-server
26 | sessionAffinity: None
27 | type: ClusterIP
28 |
--------------------------------------------------------------------------------
/microservices/README.md:
--------------------------------------------------------------------------------
1 | # GCP Hands-on material for microservices development
2 |
3 | **This is not an official Google product**.
4 |
5 | This directory contains hands-on lab material for microservices development on GCP.
6 |
7 | See [tutorial-1.md](tutorial-1.md), [tutorial-2.md](tutorial-2.md) and [tutorial-3.md](tutorial-3.md) for more details.
8 |
--------------------------------------------------------------------------------
/microservices/hello_world/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile
2 | README.md
3 | *.pyc
4 | *.pyo
5 | *.pyd
6 | __pycache__
7 | .pytest_cache
8 |
--------------------------------------------------------------------------------
/microservices/hello_world/Dockerfile:
--------------------------------------------------------------------------------
1 | # Use the official lightweight Python image.
2 | # https://hub.docker.com/_/python
3 | FROM python:3.8-slim
4 |
5 | # Allow statements and log messages to immediately appear in the Knative logs
6 | ENV PYTHONUNBUFFERED True
7 |
8 | # Copy local code to the container image.
9 | ENV APP_HOME /app
10 | WORKDIR $APP_HOME
11 | COPY . ./
12 |
13 | # Install production dependencies.
14 | RUN pip install -r requirements.txt
15 |
16 | # Run the web service on container startup. Here we use the gunicorn
17 | # webserver, with one worker process and 8 threads.
18 | # For environments with multiple CPU cores, increase the number of workers
19 | # to be equal to the cores available.
20 | CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app
21 |
--------------------------------------------------------------------------------
/microservices/hello_world/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | gunicorn
3 |
--------------------------------------------------------------------------------
/microservices/message_board/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile
2 | README.md
3 | index.yaml
4 | *.pyc
5 | *.pyo
6 | *.pyd
7 | __pycache__
8 | .pytest_cache
9 |
--------------------------------------------------------------------------------
/microservices/message_board/Dockerfile:
--------------------------------------------------------------------------------
1 | # Use the official lightweight Python image.
2 | # https://hub.docker.com/_/python
3 | FROM python:3.8-slim
4 |
5 | # Allow statements and log messages to immediately appear in the Knative logs
6 | ENV PYTHONUNBUFFERED True
7 |
8 | # Copy local code to the container image.
9 | ENV APP_HOME /app
10 | WORKDIR $APP_HOME
11 | COPY . ./
12 |
13 | # Install production dependencies.
14 | RUN pip install -r requirements.txt
15 |
16 | # Run the web service on container startup. Here we use the gunicorn
17 | # webserver, with one worker process and 8 threads.
18 | # For environments with multiple CPU cores, increase the number of workers
19 | # to be equal to the cores available.
20 | CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app
21 |
--------------------------------------------------------------------------------
/microservices/message_board/index.yaml:
--------------------------------------------------------------------------------
1 | # Copyright 2020 Google Inc.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | indexes:
16 | - kind: Message
17 | properties:
18 | - name: name
19 | - name: timestamp
20 |
--------------------------------------------------------------------------------
/microservices/message_board/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | gunicorn
3 | google-cloud-datastore
4 |
--------------------------------------------------------------------------------
/microservices/storage_logging/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile
2 | README.md
3 | *.pyc
4 | *.pyo
5 | *.pyd
6 | __pycache__
7 | .pytest_cache
8 |
--------------------------------------------------------------------------------
/microservices/storage_logging/Dockerfile:
--------------------------------------------------------------------------------
1 | # Use the official lightweight Python image.
2 | # https://hub.docker.com/_/python
3 | FROM python:3.8-slim
4 |
5 | # Allow statements and log messages to immediately appear in the Knative logs
6 | ENV PYTHONUNBUFFERED True
7 |
8 | # Copy local code to the container image.
9 | ENV APP_HOME /app
10 | WORKDIR $APP_HOME
11 | COPY . ./
12 |
13 | # Install production dependencies.
14 | RUN pip install -r requirements.txt
15 |
16 | # Run the web service on container startup. Here we use the gunicorn
17 | # webserver, with one worker process and 8 threads.
18 | # For environments with multiple CPU cores, increase the number of workers
19 | # to be equal to the cores available.
20 | CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app
21 |
--------------------------------------------------------------------------------
/microservices/storage_logging/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | gunicorn
3 | google-cloud-datastore
4 |
--------------------------------------------------------------------------------
/movie_search/custom_ver/backend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | .venv
--------------------------------------------------------------------------------
/movie_search/custom_ver/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9
2 |
3 | WORKDIR /code
4 |
5 | COPY ./requirements.txt /code/requirements.txt
6 |
7 | RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
8 |
9 | COPY . /code
10 |
11 | CMD ["fastapi", "run", "main.py", "--port", "8080"]
12 |
--------------------------------------------------------------------------------
/movie_search/custom_ver/backend/main.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/movie_search/custom_ver/backend/main.py
--------------------------------------------------------------------------------
/movie_search/custom_ver/backend/prompt_content_search.py:
--------------------------------------------------------------------------------
1 | PROMPT_CONTENT_SEARCH = """
2 | You are a video content editor.
3 |
4 | Given the following information of a movie:
5 | - The [summary] section contains the summary of the movie.
6 | - The [important scenes] section contains the important scenes of the movie with timestamps.
7 | - The [visual info] section contains the visual information on what's happening in each scene with timestamps.
8 | - The [transcription] section contains speech transcription with timestamps.
9 | - The [text] section contains text information with timestamps.
10 |
11 | Find one to three scenes that matches the user query with timestamps.
12 |
13 | [format instruction]
14 | Output in Japanese. Output is a JSON list with "scene dict".
15 | Each "scene dict" is a JSON dict with the following format:
16 | {{
17 | "Timestamp": "",
18 | "Description": ""
19 | }}
20 |
21 |
22 | [user query]
23 | {query}
24 |
25 | {metatext}
26 | """
27 |
--------------------------------------------------------------------------------
/movie_search/custom_ver/backend/requirements.txt:
--------------------------------------------------------------------------------
1 | google-cloud-discoveryengine==0.13.3
2 | google-cloud-aiplatform==1.70.0
3 | google-cloud-storage==2.18.2
4 | google-auth==2.36.0
5 | fastapi[standard]==0.115.4
6 | uvicorn==0.32.0
--------------------------------------------------------------------------------
/movie_search/custom_ver/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | .venv
3 |
--------------------------------------------------------------------------------
/movie_search/custom_ver/frontend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9-slim
2 |
3 | WORKDIR /app
4 |
5 | COPY . /app
6 |
7 | RUN pip3 install -r requirements.txt
8 |
9 | EXPOSE 8080
10 |
11 | HEALTHCHECK CMD curl --fail http://localhost:8080/_stcore/health
12 |
13 | ENTRYPOINT ["streamlit", "run", "main.py", "--server.port=8080", "--server.address=0.0.0.0"]
14 |
--------------------------------------------------------------------------------
/movie_search/custom_ver/frontend/main.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/movie_search/custom_ver/frontend/main.py
--------------------------------------------------------------------------------
/movie_search/custom_ver/frontend/requirements.txt:
--------------------------------------------------------------------------------
1 | streamlit==1.40.0
2 | requests==2.32.3
3 | google-auth==2.36.0
4 |
5 |
--------------------------------------------------------------------------------
/movie_search/final_ver/backend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | .venv
--------------------------------------------------------------------------------
/movie_search/final_ver/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9
2 |
3 | WORKDIR /code
4 |
5 | COPY ./requirements.txt /code/requirements.txt
6 |
7 | RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
8 |
9 | COPY . /code
10 |
11 | CMD ["fastapi", "run", "main.py", "--port", "8080"]
12 |
--------------------------------------------------------------------------------
/movie_search/final_ver/backend/prompt_content_search.py:
--------------------------------------------------------------------------------
1 | PROMPT_CONTENT_SEARCH = """
2 | You are a video content editor.
3 |
4 | Given the following information of a movie:
5 | - The [summary] section contains the summary of the movie.
6 | - The [important scenes] section contains the important scenes of the movie with timestamps.
7 | - The [visual info] section contains the visual information on what's happening in each scene with timestamps.
8 | - The [transcription] section contains speech transcription with timestamps.
9 | - The [text] section contains text information with timestamps.
10 |
11 | Find one to three scenes that matches the user query with timestamps.
12 |
13 | [format instruction]
14 | Output in Japanese. Output is a JSON list with "scene dict".
15 | Each "scene dict" is a JSON dict with the following format:
16 | {{
17 | "Timestamp": "",
18 | "Description": ""
19 | }}
20 |
21 |
22 | [user query]
23 | {query}
24 |
25 | {metatext}
26 | """
27 |
--------------------------------------------------------------------------------
/movie_search/final_ver/backend/requirements.txt:
--------------------------------------------------------------------------------
1 | google-cloud-discoveryengine==0.13.3
2 | google-cloud-aiplatform==1.70.0
3 | google-cloud-storage==2.18.2
4 | google-auth==2.36.0
5 | fastapi[standard]==0.115.4
6 | uvicorn==0.32.0
7 |
--------------------------------------------------------------------------------
/movie_search/final_ver/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 | .venv
3 |
--------------------------------------------------------------------------------
/movie_search/final_ver/frontend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9-slim
2 |
3 | WORKDIR /app
4 |
5 | COPY . /app
6 |
7 | RUN pip3 install -r requirements.txt
8 |
9 | EXPOSE 8080
10 |
11 | HEALTHCHECK CMD curl --fail http://localhost:8080/_stcore/health
12 |
13 | ENTRYPOINT ["streamlit", "run", "main.py", "--server.port=8080", "--server.address=0.0.0.0"]
--------------------------------------------------------------------------------
/movie_search/final_ver/frontend/requirements.txt:
--------------------------------------------------------------------------------
1 | streamlit==1.40.0
2 | requests==2.32.3
3 | google-auth==2.36.0
--------------------------------------------------------------------------------
/movie_search_metadata/Notebooks2/temp.txt:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/.gitignore:
--------------------------------------------------------------------------------
1 | handson_resource*
2 | _setup_env
3 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/backend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9-slim
2 |
3 | WORKDIR /app
4 |
5 | COPY . /app
6 |
7 | RUN pip3 install -r requirements.txt
8 |
9 | CMD ["fastapi", "run", "main.py", "--port", "8080"]
10 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/backend/file_search.py:
--------------------------------------------------------------------------------
1 | from typing import List
2 |
3 | from utils import get_bucket_and_blobnames
4 | from utils import generate_download_signed_url_v4
5 | from search_document import search_documents_by_query
6 |
7 |
8 | def search_files(query: str) -> List[dict]:
9 | response = search_documents_by_query(query)
10 | results = []
11 | results.append({'summary': response.summary.summary_text})
12 |
13 | for c, item in enumerate(response.results):
14 | title = item.document.derived_struct_data['title']
15 | url = item.document.derived_struct_data['link']
16 | bucket_name, _, blob_mp4 = get_bucket_and_blobnames(url)
17 | signed_url = generate_download_signed_url_v4(bucket_name, blob_mp4)
18 | results.append({
19 | 'id': c+1, 'title': title,
20 | 'bucket_name': bucket_name, 'blob_name': blob_mp4,
21 | 'url': url, 'signed_url': signed_url
22 | })
23 |
24 | return results
25 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/backend/prompt_content_search.py:
--------------------------------------------------------------------------------
1 | PROMPT_CONTENT_SEARCH = """
2 | You are a video content editor.
3 |
4 | Given the following information of a movie:
5 | - The [summary] section contains the summary of the movie.
6 | - The [important scenes] section contains the important scenes of the movie with timestamps.
7 | - The [visual info] section contains the visual information on what's happening in each scene with timestamps.
8 | - The [transcription] section contains speech transcription with timestamps.
9 | - The [text] section contains text information with timestamps.
10 |
11 | Find one to three scenes that match the user query with timestamps.
12 |
13 | [format instruction]
14 | Output in Japanese. Output is a JSON list with "scene dict".
15 | Each "scene dict" is a JSON dict with the following format:
16 | {{
17 | "Timestamp": "",
18 | "Description": ""
19 | }}
20 |
21 |
22 | [user query]
23 | {query}
24 |
25 | {metatext}
26 | """
27 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/backend/requirements.txt:
--------------------------------------------------------------------------------
1 | google-cloud-discoveryengine==0.13.3
2 | google-cloud-aiplatform==1.70.0
3 | google-cloud-storage==2.18.2
4 | google-auth==2.36.0
5 | fastapi[standard]==0.115.4
6 | uvicorn==0.32.0
7 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/docs/images/movie_search_demo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/movie_search_metadata/demo_app/docs/images/movie_search_demo.gif
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__
2 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/frontend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.9-slim
2 |
3 | WORKDIR /app
4 |
5 | COPY . /app
6 |
7 | RUN pip3 install -r requirements.txt
8 |
9 | EXPOSE 8080
10 |
11 | HEALTHCHECK CMD curl --fail http://localhost:8080/_stcore/health
12 |
13 | CMD ["streamlit", "run", "main.py", "--server.port=8080", "--server.address=0.0.0.0"]
14 |
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/frontend/requirements.txt:
--------------------------------------------------------------------------------
1 | streamlit==1.40.0
2 | requests==2.32.3
3 | google-auth==2.36.0
--------------------------------------------------------------------------------
/movie_search_metadata/demo_app/vais_setup.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | export PROJECT_ID=$(gcloud config list --format "value(core.project)")
4 | export BUCKET="gs://${PROJECT_ID}-movie-search"
5 |
6 | echo ""
7 | echo "## Enabling APIs..."
8 |
9 | services=(
10 | "discoveryengine.googleapis.com"
11 | )
12 | services_list="(""$(IFS='|'; echo "${services[*]}")"")"
13 | enabled=$(gcloud services list --format json | jq .[].config.name |\
14 | grep -E "$services_list" | wc -l)
15 | if [[ $enabled != ${#services[@]} ]]; then
16 | echo "Enabling APIs."
17 | services_list="$(IFS=' '; echo "${services[*]}")"
18 | gcloud services enable $services_list
19 |
20 | echo "Wait 60 seconds for APIs to be ready."
21 | sleep 60
22 | fi
23 |
24 | echo ""
25 | echo "## Creating bucket..."
26 | gsutil mb -b on -l us-central1 $BUCKET
27 |
28 | curl -OL https://github.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/releases/download/v0.1.0/handson_resource.tgz
29 | tar -xvzf handson_resource.tgz
30 | pushd handson_resource
31 | gsutil -m cp -r metadata $BUCKET/
32 | gsutil -m cp -r mp4 $BUCKET/
33 | popd
34 |
35 | python3 -m venv _setup_env
36 | source _setup_env/bin/activate
37 | pip3 install google-cloud-discoveryengine==0.13.3
38 |
39 | python3 _vais_setup.py
40 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/.gitignore:
--------------------------------------------------------------------------------
1 | venv
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.12-alpine
2 |
3 | WORKDIR /app
4 |
5 | COPY requirements.txt requirements.txt
6 | RUN pip install -r requirements.txt
7 |
8 | COPY app.py app.py
9 |
10 | EXPOSE 5000
11 | CMD ["python", "app.py"]
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, jsonify
2 | import random
3 |
4 | app = Flask(__name__)
5 |
6 | breeds = [
7 | "Labrador Retriever",
8 | "German Shepherd",
9 | "Golden Retriever",
10 | "French Bulldog",
11 | "Bulldog",
12 | "Poodle",
13 | "Beagle",
14 | "Rottweiler",
15 | "German Shorthaired Pointer",
16 | "Yorkshire Terrier"
17 | ]
18 |
19 |
20 | @app.route('/random-pets', methods=['GET'])
21 | def get_random_dog():
22 | random_breed = random.choice(breeds)
23 | return jsonify({'breed': random_breed})
24 |
25 |
26 | if __name__ == '__main__':
27 | app.run(host='0.0.0.0', port=5000)
28 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/app.txt:
--------------------------------------------------------------------------------
1 | from flask import Flask, jsonify
2 | import random
3 |
4 | app = Flask(__name__)
5 |
6 |
7 | dog_breeds = [
8 | "Labrador Retriever",
9 | "German Shepherd",
10 | "Golden Retriever",
11 | "French Bulldog",
12 | "Bulldog",
13 | "Poodle",
14 | "Beagle",
15 | "Rottweiler",
16 | "German Shorthaired Pointer",
17 | "Yorkshire Terrier"
18 | ]
19 |
20 |
21 | cat_breeds = [
22 | "Persian Cat",
23 | "Maine Coon",
24 | "Ragdoll",
25 | "Siamese Cat",
26 | "British Shorthair",
27 | "Sphynx Cat",
28 | "Scottish Fold",
29 | "Abyssinian",
30 | "Bengal Cat",
31 | "Russian Blue"
32 | ]
33 |
34 |
35 | @app.route('/random-pets', methods=['GET'])
36 | def get_random_pet():
37 | if random.choice([True, False]):
38 | random_breed = random.choice(dog_breeds)
39 | animal_type = "dog"
40 | else:
41 | random_breed = random.choice(cat_breeds)
42 | animal_type = "cat"
43 | return jsonify({"animal": animal_type, "breed": random_breed})
44 |
45 |
46 | if __name__ == '__main__':
47 | app.run(host='0.0.0.0', port=5000)
48 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/cloudbuild-2.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'python:3.12-alpine'
3 | entrypoint: 'sh'
4 | args:
5 | - '-c'
6 | - |
7 | pip install -r requirements.txt
8 | pip install flake8
9 | flake8 .
10 |
11 | - name: 'python:3.12-alpine'
12 | entrypoint: 'sh'
13 | args:
14 | - '-c'
15 | - |
16 | pip install -r requirements.txt
17 | python -m unittest discover
18 |
19 | - name: 'gcr.io/cloud-builders/docker'
20 | args: ['build', '-t', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2', '.']
21 |
22 | - name: 'gcr.io/cloud-builders/docker'
23 | args: ['push', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2']
24 |
25 | - name: 'gcr.io/cloud-builders/gcloud'
26 | entrypoint: 'bash'
27 | args:
28 | - '-c'
29 | - |
30 | sed -i "s|image: .*|image: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2|g" kubernetes-manifests/deployment.yaml
31 |
32 | - name: 'gcr.io/cloud-builders/gcloud'
33 | entrypoint: 'bash'
34 | args:
35 | - '-c'
36 | - |
37 | gcloud deploy releases create release-$(date +%Y%m%d%H%M%S) --delivery-pipeline=pfe-cicd --region=asia-northeast1 --source=./ --project=$PROJECT_ID
38 |
39 | images:
40 | - 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2'
41 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'python:3.12-alpine'
3 | entrypoint: 'sh'
4 | args:
5 | - '-c'
6 | - |
7 | pip install -r requirements.txt
8 | pip install flake8
9 | flake8 .
10 |
11 | - name: 'python:3.12-alpine'
12 | entrypoint: 'sh'
13 | args:
14 | - '-c'
15 | - |
16 | pip install -r requirements.txt
17 | python -m unittest discover
18 |
19 | - name: 'gcr.io/cloud-builders/docker'
20 | args: ['build', '-t', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1', '.']
21 |
22 | - name: 'gcr.io/cloud-builders/docker'
23 | args: ['push', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1']
24 |
25 | images:
26 | - 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1'
27 |
28 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/clouddeploy-2.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: deploy.cloud.google.com/v1beta1
2 | kind: DeliveryPipeline
3 | metadata:
4 | name: pfe-cicd
5 | description: pfe-cicd
6 | serialPipeline:
7 | stages:
8 | - targetId: dev
9 | - targetId: prod
10 | ---
11 | apiVersion: deploy.cloud.google.com/v1beta1
12 | kind: Target
13 | metadata:
14 | name: dev
15 | description: Dev Environment
16 | gke:
17 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/dev-cluster
18 | ---
19 | apiVersion: deploy.cloud.google.com/v1beta1
20 | kind: Target
21 | metadata:
22 | name: prod
23 | description: Production Environment
24 | gke:
25 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/prod-cluster
26 | ---
27 | apiVersion: deploy.cloud.google.com/v1
28 | kind: Automation
29 | metadata:
30 | name: pfe-cicd/promote
31 | description: promotes a release
32 | suspended: false
33 | serviceAccount: ${PROJECT_NUMBER}-compute@developer.gserviceaccount.com
34 | selector:
35 | targets:
36 | - id: dev
37 | rules:
38 | - promoteReleaseRule:
39 | name: "promote-release"
40 | wait: 1m
41 | toTargetId: "@next"
42 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/clouddeploy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: deploy.cloud.google.com/v1beta1
2 | kind: DeliveryPipeline
3 | metadata:
4 | name: pfe-cicd
5 | description: pfe-cicd
6 | serialPipeline:
7 | stages:
8 | - targetId: dev
9 | - targetId: prod
10 | ---
11 | apiVersion: deploy.cloud.google.com/v1beta1
12 | kind: Target
13 | metadata:
14 | name: dev
15 | description: Dev Environment
16 | gke:
17 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/dev-cluster
18 | ---
19 | apiVersion: deploy.cloud.google.com/v1beta1
20 | kind: Target
21 | metadata:
22 | name: prod
23 | description: Production Environment
24 | gke:
25 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/prod-cluster
26 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/kubernetes-manifests/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: pets-deployment
5 | labels:
6 | app: pets
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | app: pets
12 | template:
13 | metadata:
14 | labels:
15 | app: pets
16 | spec:
17 | containers:
18 | - name: pets
19 | image: pets
20 | ports:
21 | - containerPort: 5000
22 | env:
23 | - name: FLASK_ENV
24 | value: production
25 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/kubernetes-manifests/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: pets-service
5 | spec:
6 | selector:
7 | app: pets
8 | ports:
9 | - protocol: TCP
10 | port: 80
11 | targetPort: 5000
12 | type: LoadBalancer
13 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - deployment.yaml
3 | - service.yaml
4 |
5 | images:
6 | - name: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets
7 | newName: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets
8 | newTag: ${SHORT_SHA}
9 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | flake8
3 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/skaffold.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: skaffold/v2beta28
2 | kind: Config
3 | metadata:
4 | name: pets-app
5 | deploy:
6 | kubectl:
7 | manifests: ["kubernetes-manifests/*.yaml"]
8 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-01/test_app.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from app import app
3 |
4 |
5 | class FlaskTestCase(unittest.TestCase):
6 |
7 | def setUp(self):
8 | self.app = app.test_client()
9 | self.app.testing = True
10 |
11 | def test_random_dog(self):
12 | response = self.app.get('/random-pets')
13 | self.assertEqual(response.status_code, 200)
14 | self.assertIn('breed', response.json)
15 |
16 |
17 | if __name__ == '__main__':
18 | unittest.main()
19 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.12
2 |
3 | WORKDIR /app
4 |
5 | COPY requirements.txt requirements.txt
6 | RUN pip install -r requirements.txt
7 |
8 | COPY app.py app.py
9 |
10 | EXPOSE 5000
11 | CMD ["python", "app.py"]
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, jsonify
2 | import random
3 |
4 | app = Flask(__name__)
5 |
6 |
7 | dog_breeds = [
8 | "Labrador Retriever",
9 | "German Shepherd",
10 | "Golden Retriever",
11 | "French Bulldog",
12 | "Bulldog",
13 | "Poodle",
14 | "Beagle",
15 | "Rottweiler",
16 | "German Shorthaired Pointer",
17 | "Yorkshire Terrier"
18 | ]
19 |
20 |
21 | cat_breeds = [
22 | "Persian Cat",
23 | "Maine Coon",
24 | "Ragdoll",
25 | "Siamese Cat",
26 | "British Shorthair",
27 | "Sphynx Cat",
28 | "Scottish Fold",
29 | "Abyssinian",
30 | "Bengal Cat",
31 | "Russian Blue"
32 | ]
33 |
34 |
35 | @app.route('/random-pets', methods=['GET'])
36 | def get_random_pet():
37 | if random.choice([True, False]):
38 | random_breed = random.choice(dog_breeds)
39 | animal_type = "dog"
40 | else:
41 | random_breed = random.choice(cat_breeds)
42 | animal_type = "cat"
43 | return jsonify({"animal": animal_type, "breed": random_breed})
44 |
45 |
46 | if __name__ == '__main__':
47 | app.run(host='0.0.0.0', port=5000)
48 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/clouddeploy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: deploy.cloud.google.com/v1beta1
2 | kind: DeliveryPipeline
3 | metadata:
4 | name: pfe-cicd
5 | description: pfe-cicd
6 | serialPipeline:
7 | stages:
8 | - targetId: dev
9 | - targetId: prod
10 | ---
11 | apiVersion: deploy.cloud.google.com/v1beta1
12 | kind: Target
13 | metadata:
14 | name: dev
15 | description: Dev Environment
16 | gke:
17 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/dev-cluster
18 | ---
19 | apiVersion: deploy.cloud.google.com/v1beta1
20 | kind: Target
21 | metadata:
22 | name: prod
23 | description: Production Environment
24 | gke:
25 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/prod-cluster
26 | ---
27 | apiVersion: deploy.cloud.google.com/v1
28 | kind: Automation
29 | metadata:
30 | name: pfe-cicd/promote
31 | description: promotes a release
32 | suspended: false
33 | serviceAccount: ${PROJECT_NUMBER}-compute@developer.gserviceaccount.com
34 | selector:
35 | targets:
36 | - id: dev
37 | rules:
38 | - promoteReleaseRule:
39 | name: "promote-release"
40 | wait: 1m
41 | toTargetId: "@next"
42 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/kubernetes-manifests/allow-myrepo.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: constraints.gatekeeper.sh/v1beta1
2 | kind: K8sAllowedRepos
3 | metadata:
4 | name: allow-my-app-repo
5 | spec:
6 | match:
7 | kinds:
8 | - apiGroups:
9 | - ""
10 | kinds:
11 | - Pod
12 | namespaces:
13 | - default
14 | parameters:
15 | repos:
16 | - asia-northeast1-docker.pkg.dev/PROJECT_ID/app-repo/
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/kubernetes-manifests/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: pets-deployment
5 | labels:
6 | app: pets
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | app: pets
12 | template:
13 | metadata:
14 | labels:
15 | app: pets
16 | spec:
17 | containers:
18 | - name: pets
19 | image: pets
20 | ports:
21 | - containerPort: 5000
22 | env:
23 | - name: FLASK_ENV
24 | value: production
25 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/kubernetes-manifests/maven-vulns.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Pod
3 | metadata:
4 | name: maven-vulns
5 | spec:
6 | containers:
7 | - name: maven-vulns-app
8 | image: us-docker.pkg.dev/google-samples/containers/gke/security/maven-vulns
9 | # This app listens on port 8080 for web traffic by default.
10 | ports:
11 | - containerPort: 8080
12 | env:
13 | - name: PORT
14 | value: "8080"
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/kubernetes-manifests/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: pets-service
5 | spec:
6 | selector:
7 | app: pets
8 | ports:
9 | - protocol: TCP
10 | port: 80
11 | targetPort: 5000
12 | type: LoadBalancer
13 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | flake8
3 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/skaffold.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: skaffold/v2beta28
2 | kind: Config
3 | metadata:
4 | name: pets-app
5 | deploy:
6 | kubectl:
7 | manifests: ["kubernetes-manifests/*.yaml"]
8 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-02/test_app.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from app import app
3 |
4 |
5 | class FlaskTestCase(unittest.TestCase):
6 |
7 | def setUp(self):
8 | self.app = app.test_client()
9 | self.app.testing = True
10 |
11 | def test_random_dog(self):
12 | response = self.app.get('/random-pets')
13 | self.assertEqual(response.status_code, 200)
14 | self.assertIn('breed', response.json)
15 |
16 |
17 | if __name__ == '__main__':
18 | unittest.main()
19 |
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-03/allow-hostpath.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: constraints.gatekeeper.sh/v1beta1
2 | kind: K8sPSPHostFilesystem
3 | metadata:
4 | name: allow-hostpath
5 | spec:
6 | match:
7 | kinds:
8 | - apiGroups:
9 | - ""
10 | kinds:
11 | - Pod
12 | parameters:
13 | allowedHostPaths:
14 | - pathPrefix: /var/log
15 | readOnly: true
--------------------------------------------------------------------------------
/pfe-adv-sep/lab-03/bad-pod.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Pod
3 | metadata:
4 | labels:
5 | app: bad-pod
6 | name: bad-pod
7 | spec:
8 | volumes:
9 | - name: host-fs
10 | hostPath:
11 | path: /
12 | containers:
13 | - image: ubuntu
14 | imagePullPolicy: Always
15 | name: bad-pod
16 | command: ["/bin/sh", "-c", "sleep infinity"]
17 | volumeMounts:
18 | - name: host-fs
19 | mountPath: /root
20 | restartPolicy: Never
--------------------------------------------------------------------------------
/pfe-basic-sep/lab-02/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'gcr.io/cloud-builders/docker'
3 | args: ['build', '-t', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/spring-gs:v1', '.']
4 |
5 | - name: 'gcr.io/cloud-builders/docker'
6 | args: ['push', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/spring-gs:v1']
7 |
8 | - name: 'gcr.io/cloud-builders/gcloud'
9 | entrypoint: 'bash'
10 | args:
11 | - '-c'
12 | - |
13 | sed -i "s|image: .*|image: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/spring-gs:v1|g" kubernetes-manifests/deployment.yaml
14 |
15 | - name: "gcr.io/cloud-builders/gke-deploy"
16 | args:
17 | - run
18 | - --filename=kubernetes-manifests/
19 | - --image=asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/spring-gs:v1
20 | - --location=asia-northeast1
21 | - --cluster=dev-cluster
--------------------------------------------------------------------------------
/pfe-basic-sep/lab-03/unschedulable-hello.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: unschedulable-hello
5 | labels:
6 | app: hello
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | app: hello
12 | template:
13 | metadata:
14 | labels:
15 | app: hello
16 | spec:
17 | containers:
18 | - name: unschedulable-hello
19 | image: us-docker.pkg.dev/google-samples/containers/gke/hello-app:1.0
20 | ports:
21 | - containerPort: 8080
22 | resources:
23 | requests:
24 | cpu: 5000m
25 | memory: 16Gi
26 | limits:
27 | cpu: 5000m
28 | memory: 16Gi
--------------------------------------------------------------------------------
/pfe-cicd/.gitignore:
--------------------------------------------------------------------------------
1 | venv
--------------------------------------------------------------------------------
/pfe-cicd/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.12-alpine
2 |
3 | WORKDIR /app
4 |
5 | COPY requirements.txt requirements.txt
6 | RUN pip install -r requirements.txt
7 |
8 | COPY app.py app.py
9 |
10 | EXPOSE 5000
11 | CMD ["python", "app.py"]
--------------------------------------------------------------------------------
/pfe-cicd/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask, jsonify
2 | import random
3 |
4 | app = Flask(__name__)
5 |
6 | breeds = [
7 | "Labrador Retriever",
8 | "German Shepherd",
9 | "Golden Retriever",
10 | "French Bulldog",
11 | "Bulldog",
12 | "Poodle",
13 | "Beagle",
14 | "Rottweiler",
15 | "German Shorthaired Pointer",
16 | "Yorkshire Terrier"
17 | ]
18 |
19 |
20 | @app.route('/random-pets', methods=['GET'])
21 | def get_random_dog():
22 | random_breed = random.choice(breeds)
23 | return jsonify({'breed': random_breed})
24 |
25 |
26 | if __name__ == '__main__':
27 | app.run(host='0.0.0.0', port=5000)
28 |
--------------------------------------------------------------------------------
/pfe-cicd/app.txt:
--------------------------------------------------------------------------------
1 | from flask import Flask, jsonify
2 | import random
3 |
4 | app = Flask(__name__)
5 |
6 |
7 | dog_breeds = [
8 | "Labrador Retriever",
9 | "German Shepherd",
10 | "Golden Retriever",
11 | "French Bulldog",
12 | "Bulldog",
13 | "Poodle",
14 | "Beagle",
15 | "Rottweiler",
16 | "German Shorthaired Pointer",
17 | "Yorkshire Terrier"
18 | ]
19 |
20 |
21 | cat_breeds = [
22 | "Persian Cat",
23 | "Maine Coon",
24 | "Ragdoll",
25 | "Siamese Cat",
26 | "British Shorthair",
27 | "Sphynx Cat",
28 | "Scottish Fold",
29 | "Abyssinian",
30 | "Bengal Cat",
31 | "Russian Blue"
32 | ]
33 |
34 |
35 | @app.route('/random-pets', methods=['GET'])
36 | def get_random_pet():
37 | if random.choice([True, False]):
38 | random_breed = random.choice(dog_breeds)
39 | animal_type = "dog"
40 | else:
41 | random_breed = random.choice(cat_breeds)
42 | animal_type = "cat"
43 | return jsonify({"animal": animal_type, "breed": random_breed})
44 |
45 |
46 | if __name__ == '__main__':
47 | app.run(host='0.0.0.0', port=5000)
48 |
--------------------------------------------------------------------------------
/pfe-cicd/cloudbuild-2.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'python:3.12-alpine'
3 | entrypoint: 'sh'
4 | args:
5 | - '-c'
6 | - |
7 | pip install -r requirements.txt
8 | pip install flake8
9 | flake8 .
10 |
11 | - name: 'python:3.12-alpine'
12 | entrypoint: 'sh'
13 | args:
14 | - '-c'
15 | - |
16 | pip install -r requirements.txt
17 | python -m unittest discover
18 |
19 | - name: 'gcr.io/cloud-builders/docker'
20 | args: ['build', '-t', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2', '.']
21 |
22 | - name: 'gcr.io/cloud-builders/docker'
23 | args: ['push', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2']
24 |
25 | - name: 'gcr.io/cloud-builders/gcloud'
26 | entrypoint: 'bash'
27 | args:
28 | - '-c'
29 | - |
30 | sed -i "s|image: .*|image: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2|g" kubernetes-manifests/deployment.yaml
31 |
32 | - name: 'gcr.io/cloud-builders/gcloud'
33 | entrypoint: 'bash'
34 | args:
35 | - '-c'
36 | - |
37 | gcloud deploy releases create release-$(date +%Y%m%d%H%M%S) --delivery-pipeline=pfe-cicd --region=asia-northeast1 --source=./ --project=$PROJECT_ID
38 |
39 | images:
40 | - 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v2'
41 |
--------------------------------------------------------------------------------
/pfe-cicd/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | steps:
2 | - name: 'python:3.12-alpine'
3 | entrypoint: 'sh'
4 | args:
5 | - '-c'
6 | - |
7 | pip install -r requirements.txt
8 | pip install flake8
9 | flake8 .
10 |
11 | - name: 'python:3.12-alpine'
12 | entrypoint: 'sh'
13 | args:
14 | - '-c'
15 | - |
16 | pip install -r requirements.txt
17 | python -m unittest discover
18 |
19 | - name: 'gcr.io/cloud-builders/docker'
20 | args: ['build', '-t', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1', '.']
21 |
22 | - name: 'gcr.io/cloud-builders/docker'
23 | args: ['push', 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1']
24 |
25 | images:
26 | - 'asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets:v1'
27 |
28 |
--------------------------------------------------------------------------------
/pfe-cicd/clouddeploy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: deploy.cloud.google.com/v1beta1
2 | kind: DeliveryPipeline
3 | metadata:
4 | name: pfe-cicd
5 | description: pfe-cicd
6 | serialPipeline:
7 | stages:
8 | - targetId: dev
9 | - targetId: prod
10 | ---
11 | apiVersion: deploy.cloud.google.com/v1beta1
12 | kind: Target
13 | metadata:
14 | name: dev
15 | description: Dev Environment
16 | gke:
17 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/dev-cluster
18 | ---
19 | apiVersion: deploy.cloud.google.com/v1beta1
20 | kind: Target
21 | metadata:
22 | name: prod
23 | description: Production Environment
24 | gke:
25 | cluster: projects/${PROJECT_ID}/locations/asia-northeast1/clusters/prod-cluster
26 |
--------------------------------------------------------------------------------
/pfe-cicd/kubernetes-manifests/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | name: pets-deployment
5 | labels:
6 | app: pets
7 | spec:
8 | replicas: 1
9 | selector:
10 | matchLabels:
11 | app: pets
12 | template:
13 | metadata:
14 | labels:
15 | app: pets
16 | spec:
17 | containers:
18 | - name: pets
19 | image: pets
20 | ports:
21 | - containerPort: 5000
22 | env:
23 | - name: FLASK_ENV
24 | value: production
25 |
--------------------------------------------------------------------------------
/pfe-cicd/kubernetes-manifests/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: pets-service
5 | spec:
6 | selector:
7 | app: pets
8 | ports:
9 | - protocol: TCP
10 | port: 80
11 | targetPort: 5000
12 | type: LoadBalancer
13 |
--------------------------------------------------------------------------------
/pfe-cicd/kustomization.yaml:
--------------------------------------------------------------------------------
1 | resources:
2 | - deployment.yaml
3 | - service.yaml
4 |
5 | images:
6 | - name: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets
7 | newName: asia-northeast1-docker.pkg.dev/$PROJECT_ID/app-repo/pets
8 | newTag: ${SHORT_SHA}
9 |
--------------------------------------------------------------------------------
/pfe-cicd/requirements.txt:
--------------------------------------------------------------------------------
1 | Flask
2 | flake8
3 |
--------------------------------------------------------------------------------
/pfe-cicd/skaffold.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: skaffold/v2beta28
2 | kind: Config
3 | metadata:
4 | name: pets-app
5 | deploy:
6 | kubectl:
7 | manifests: ["kubernetes-manifests/*.yaml"]
8 |
--------------------------------------------------------------------------------
/pfe-cicd/test_app.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from app import app
3 |
4 |
5 | class FlaskTestCase(unittest.TestCase):
6 |
7 | def setUp(self):
8 | self.app = app.test_client()
9 | self.app.testing = True
10 |
11 | def test_random_dog(self):
12 | response = self.app.get('/random-pets')
13 | self.assertEqual(response.status_code, 200)
14 | self.assertIn('breed', response.json)
15 |
16 |
17 | if __name__ == '__main__':
18 | unittest.main()
19 |
--------------------------------------------------------------------------------
/workstations_with_generative_ai/images/reconnect_cloudshell.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/gcp-getting-started-lab-jp/c5def029445a6474e5412c1bad1218c61573188c/workstations_with_generative_ai/images/reconnect_cloudshell.png
--------------------------------------------------------------------------------