├── GA4 └── bqstreaming │ ├── README.md │ ├── ga4bqschema.go │ └── ingestuserevents.go ├── GCF ├── demo.html ├── demo2.html └── python │ ├── autocomplete.py │ ├── recommendation.py │ ├── requirements.txt │ ├── search.py │ ├── testautocomplete.py │ ├── testrecs.py │ └── testsearch.py ├── GTM └── sGTM.js ├── JSON ├── README.md └── all-events-schema.json ├── LICENSE ├── README.md ├── beta ├── README.md ├── curl │ ├── catalog_import.json │ ├── create_item.sh │ ├── delete_events.sh │ ├── delete_item.sh │ ├── delete_key.sh │ ├── event_import.json │ ├── gcs_catalog_import.sh │ ├── gcs_catalog_sample.txt │ ├── gcs_event_import.sh │ ├── gcs_import_status.sh │ ├── gcs_user_event_sample.txt │ ├── get_catalog_item.sh │ ├── importEvent.sh │ ├── inlineEvent.json │ ├── list_catalog_items.sh │ ├── list_events.sh │ ├── list_keys.sh │ ├── register_key.sh │ └── testitem.json ├── php │ └── predict.php └── python │ ├── bq_catalog_export.py │ ├── copy_events.py │ ├── curl_predict.py │ ├── delete_catalog_items.py │ ├── gcs_import_catalog.py │ ├── gcs_import_events.py │ ├── import_catalog.py │ ├── import_events.py │ ├── list_catalog.py │ ├── list_events.py │ ├── list_keys.py │ ├── lro_operations.py │ ├── predict.py │ ├── register_key.py │ ├── timestamp_to_zulu.py │ └── unregister_key.py ├── php ├── README.md ├── autocomplete │ ├── complete.php │ ├── searchform.html │ └── style.css └── predict.php └── python ├── README.md ├── import_catalog_inline.py ├── import_event_inline.py ├── predict.py └── search_test.py /GA4/bqstreaming/README.md: -------------------------------------------------------------------------------- 1 | # GA4 BigQuery Streaming Ingestion Tool 2 | 3 | ## Overview 4 | 5 | You can export all of your raw events from [Google Analytics 4 properties to BigQuery](https://support.google.com/analytics/answer/9823238#step3&zippy=%2Cin-this-article) 6 | , and ingest the data into the retail API using the [CollectUserEvent method](https://cloud.google.com/retail/docs/reference/rest/v2/projects.locations.catalogs.userEvents/collect). 7 | This is a Cloud Function example that collects GA4 schema data stored in 8 | BigQuery tables and ingests the data into the retail API. 9 | The required parameters needed by this tool will be passed through the 10 | environment variables and request bodies. 11 | Please refer to the comments of `userEventIngester` and `envVariables` in 12 | "ingestuserevents.go" file to learn the parameters and variables. 13 | 14 | ## Before you begin 15 | 16 | Contact cips-dh@google.com to get allowlisted to use the data harmonization 17 | features. 18 | 19 | ## Create a Cloud Function 20 | 21 | You may follow the toturial 22 | [Your First Function: Go](https://cloud.google.com/functions/docs/first-go) to 23 | learn how to create a Cloud Function. We list the core commands needed below: 24 | 25 | 1. Enable services: 26 | 27 | ``` 28 | gcloud services enable cloudbuild.googleapis.com 29 | gcloud services enable cloudfunctions.googleapis.com 30 | ``` 31 | 32 | 1. Grant permissions: 33 | 34 | ``` 35 | BQProjectID= 36 | CFProjectID= 37 | gcloud projects add-iam-policy-binding ${BQProjectID} \ 38 | --member=serviceAccount:${CFProjectID}@appspot.gserviceaccount.com --role=roles/bigquery.jobUser 39 | gcloud projects add-iam-policy-binding ${BQProjectID} \ 40 | --member=serviceAccount:${CFProjectID}@appspot.gserviceaccount.com --role=roles/bigquery.dataViewer 41 | ``` 42 | 43 | 1. Create an API key to access the retail API 44 | 45 | You may follow the instructions in 46 | [Recommendations AI Before you begin](https://cloud.google.com/retail/recommendations-ai/docs/setting-up#create-key) 47 | to create the API key. 48 | 49 | 1. Create a directory on your local system for the function code: 50 | 51 | ``` 52 | mkdir ~/ingestuserevents 53 | cd ~/ingestuserevents 54 | ``` 55 | 56 | 1. Copy the go files to the directory "~/ingestuserevents". 57 | 58 | 1. Specify the dependencies: 59 | 60 | ``` 61 | go mod init ingestuserevents 62 | go mod tidy 63 | ``` 64 | 65 | 1. Deploy the function 66 | 67 | ``` 68 | gcloud functions deploy IngestUserEvents \ 69 | --runtime go116 --trigger-http --allow-unauthenticated \ 70 | --set-env-vars=\ 71 | MetadataProjectID=${CFProjectID},\ 72 | RetailProjectNumber=,\ 73 | APIKey=,\ 74 | TaskName= 75 | ``` 76 | 77 | 1. Test the function 78 | 79 | You should pass some parameters like below in the request body: 80 | 81 | ``` 82 | { 83 | "BQProjectID":"", 84 | "BQDatasetID":"", 85 | "DurationInSeconds":70 86 | } 87 | ``` 88 | 89 | ## Create a scheduled job 90 | 91 | We want to use the Cloud Scheduler to trigger the Cloud Function: 92 | 93 | ``` 94 | gcloud scheduler jobs create http ingest-user-events \ 95 | --description "Ingest user events hourly" \ 96 | --schedule "*/1 * * * *" \ 97 | --time-zone "Canada/Toronto" \ 98 | --uri "https://${REGION}-${PROJECT_ID}.cloudfunctions.net/IngestUserEvents" \ 99 | --http-method GET \ 100 | --message-body '{"BQProjectID":"","BQDatasetID":"","DurationInSeconds":70}' 101 | ``` 102 | -------------------------------------------------------------------------------- /GA4/bqstreaming/ga4bqschema.go: -------------------------------------------------------------------------------- 1 | package ingestuserevents 2 | 3 | import "cloud.google.com/go/bigquery" 4 | 5 | // GA4BQRow represents GA4 BQ schema. 6 | type GA4BQRow struct { 7 | EventDate bigquery.NullString `bigquery:"event_date,nullable" json:"event_date,omitempty"` 8 | EventTimestamp bigquery.NullInt64 `bigquery:"event_timestamp,nullable" json:"event_timestamp,omitempty"` 9 | EventName bigquery.NullString `bigquery:"event_name,nullable" json:"event_name,omitempty"` 10 | EventParams []EventParams `bigquery:"event_params,nullable" json:"event_params,omitempty"` 11 | EventValueInUsd bigquery.NullFloat64 `bigquery:"event_value_in_usd,nullable" json:"event_value_in_usd,omitempty"` 12 | EventBundleSequenceID bigquery.NullInt64 `bigquery:"event_bundle_sequence_id" json:"event_bundle_sequence_id,omitempty"` 13 | UserPseudoID bigquery.NullString `bigquery:"user_pseudo_id" json:"user_pseudo_id,omitempty"` 14 | PrivacyInfo *PrivacyInfo `bigquery:"privacy_info" json:"privacy_info,omitempty"` 15 | UserFirstTouchTimestamp bigquery.NullInt64 `bigquery:"user_first_touch_timestamp" json:"user_first_touch_timestamp,omitempty"` 16 | UserLtv *UserLtv `bigquery:"user_ltv" json:"user_ltv,omitempty"` 17 | Device *Device `bigquery:"device" json:"device,omitempty"` 18 | Geo *Geo `bigquery:"geo" json:"geo,omitempty"` 19 | TrafficSource *TrafficSource `bigquery:"traffic_source" json:"traffic_source,omitempty"` 20 | StreamID bigquery.NullString `bigquery:"stream_id" json:"stream_id,omitempty"` 21 | Platform bigquery.NullString `bigquery:"platform" json:"platform,omitempty"` 22 | Ecommerce *Ecommerce `bigquery:"ecommerce" json:"ecommerce,omitempty"` 23 | Items []Items `bigquery:"items" json:"items,omitempty"` 24 | } 25 | 26 | // Value is a struct in GA4 schema. 27 | type Value struct { 28 | StringValue bigquery.NullString `bigquery:"string_value" json:"string_value,omitempty"` 29 | IntValue bigquery.NullInt64 `bigquery:"int_value" json:"int_value,omitempty"` 30 | FloatValue bigquery.NullFloat64 `bigquery:"float_value" json:"float_value,omitempty"` 31 | DoubleValue bigquery.NullFloat64 `bigquery:"double_value" json:"double_value,omitempty"` 32 | } 33 | 34 | // EventParams is a struct in GA4 schema. 35 | type EventParams struct { 36 | Key bigquery.NullString `bigquery:"key" json:"key,omitempty"` 37 | Value *Value `bigquery:"value" json:"value,omitempty"` 38 | } 39 | 40 | // PrivacyInfo is a struct in GA4 schema. 41 | type PrivacyInfo struct { 42 | UsesTransientToken bigquery.NullString `bigquery:"uses_transient_token" json:"uses_transient_token,omitempty"` 43 | } 44 | 45 | // UserLtv is a struct in GA4 schema. 46 | type UserLtv struct { 47 | Revenue bigquery.NullFloat64 `bigquery:"revenue" json:"revenue,omitempty"` 48 | Currency bigquery.NullString `bigquery:"currency" json:"currency,omitempty"` 49 | } 50 | 51 | // WebInfo is a struct in GA4 schema. 52 | type WebInfo struct { 53 | Browser bigquery.NullString `bigquery:"browser" json:"browser,omitempty"` 54 | BrowserVersion bigquery.NullString `bigquery:"browser_version" json:"browser_version,omitempty"` 55 | Hostname bigquery.NullString `bigquery:"hostname" json:"hostname,omitempty"` 56 | } 57 | 58 | // Device is a struct in GA4 schema. 59 | type Device struct { 60 | Category bigquery.NullString `bigquery:"category" json:"category,omitempty"` 61 | MobileBrandName bigquery.NullString `bigquery:"mobile_brand_name" json:"mobile_brand_name,omitempty"` 62 | MobileModelName bigquery.NullString `bigquery:"mobile_model_name" json:"mobile_model_name,omitempty"` 63 | OperatingSystem bigquery.NullString `bigquery:"operating_system" json:"operating_system,omitempty"` 64 | OperatingSystemVersion bigquery.NullString `bigquery:"operating_system_version" json:"operating_system_version,omitempty"` 65 | Language bigquery.NullString `bigquery:"language" json:"language,omitempty"` 66 | IsLimitedAdTracking bigquery.NullString `bigquery:"is_limited_ad_tracking" json:"is_limited_ad_tracking,omitempty"` 67 | WebInfo *WebInfo `bigquery:"web_info" json:"web_info,omitempty"` 68 | } 69 | 70 | // Geo is a struct in GA4 schema. 71 | type Geo struct { 72 | Continent bigquery.NullString `bigquery:"continent" json:"continent,omitempty"` 73 | Country bigquery.NullString `bigquery:"country" json:"country,omitempty"` 74 | Region bigquery.NullString `bigquery:"region" json:"region,omitempty"` 75 | City bigquery.NullString `bigquery:"city" json:"city,omitempty"` 76 | SubContinent bigquery.NullString `bigquery:"sub_continent" json:"sub_continent,omitempty"` 77 | Metro bigquery.NullString `bigquery:"metro" json:"metro,omitempty"` 78 | } 79 | 80 | // TrafficSource is a struct in GA4 schema. 81 | type TrafficSource struct { 82 | Name bigquery.NullString `bigquery:"name" json:"name,omitempty"` 83 | Medium bigquery.NullString `bigquery:"medium" json:"medium,omitempty"` 84 | Source bigquery.NullString `bigquery:"source" json:"source,omitempty"` 85 | } 86 | 87 | // Ecommerce is a struct in GA4 schema. 88 | type Ecommerce struct { 89 | TotalItemQuantity bigquery.NullInt64 `bigquery:"total_item_quantity" json:"total_item_quantity,omitempty"` 90 | PurchaseRevenueInUsd bigquery.NullFloat64 `bigquery:"purchase_revenue_in_usd" json:"purchase_revenue_in_usd,omitempty"` 91 | PurchaseRevenue bigquery.NullFloat64 `bigquery:"purchase_revenue" json:"purchase_revenue,omitempty"` 92 | ShippingValueInUsd bigquery.NullFloat64 `bigquery:"shipping_value_in_usd" json:"shipping_value_in_usd,omitempty"` 93 | ShippingValue bigquery.NullFloat64 `bigquery:"shipping_value" json:"shipping_value,omitempty"` 94 | TaxValueInUsd bigquery.NullFloat64 `bigquery:"tax_value_in_usd" json:"tax_value_in_usd,omitempty"` 95 | TaxValue bigquery.NullFloat64 `bigquery:"tax_value" json:"tax_value,omitempty"` 96 | UniqueItems bigquery.NullInt64 `bigquery:"unique_items" json:"unique_items,omitempty"` 97 | TransactionID bigquery.NullString `bigquery:"transaction_id" json:"transaction_id,omitempty"` 98 | } 99 | 100 | // Items is a struct in GA4 schema. 101 | type Items struct { 102 | ItemID bigquery.NullString `bigquery:"item_id" json:"item_id,omitempty"` 103 | ItemName bigquery.NullString `bigquery:"item_name" json:"item_name,omitempty"` 104 | ItemBrand bigquery.NullString `bigquery:"item_brand" json:"item_brand,omitempty"` 105 | ItemVariant bigquery.NullString `bigquery:"item_variant" json:"item_variant,omitempty"` 106 | ItemCategory bigquery.NullString `bigquery:"item_category" json:"item_category,omitempty"` 107 | ItemCategory2 bigquery.NullString `bigquery:"item_category2" json:"item_category2,omitempty"` 108 | ItemCategory3 bigquery.NullString `bigquery:"item_category3" json:"item_category3,omitempty"` 109 | ItemCategory4 bigquery.NullString `bigquery:"item_category4" json:"item_category4,omitempty"` 110 | ItemCategory5 bigquery.NullString `bigquery:"item_category5" json:"item_category5,omitempty"` 111 | PriceInUsd bigquery.NullFloat64 `bigquery:"price_in_usd" json:"price_in_usd,omitempty"` 112 | Price bigquery.NullFloat64 `bigquery:"price" json:"price,omitempty"` 113 | Quantity bigquery.NullInt64 `bigquery:"quantity" json:"quantity,omitempty"` 114 | ItemRevenueInUsd bigquery.NullFloat64 `bigquery:"item_revenue_in_usd" json:"item_revenue_in_usd,omitempty"` 115 | ItemRevenue bigquery.NullFloat64 `bigquery:"item_revenue" json:"item_revenue,omitempty"` 116 | Coupon bigquery.NullString `bigquery:"coupon" json:"coupon,omitempty"` 117 | Affiliation bigquery.NullString `bigquery:"affiliation" json:"affiliation,omitempty"` 118 | LocationID bigquery.NullString `bigquery:"location_id" json:"location_id,omitempty"` 119 | ItemListID bigquery.NullString `bigquery:"item_list_id" json:"item_list_id,omitempty"` 120 | ItemListName bigquery.NullString `bigquery:"item_list_name" json:"item_list_name,omitempty"` 121 | ItemListIndex bigquery.NullString `bigquery:"item_list_index" json:"item_list_index,omitempty"` 122 | PromotionID bigquery.NullString `bigquery:"promotion_id" json:"promotion_id,omitempty"` 123 | PromotionName bigquery.NullString `bigquery:"promotion_name" json:"promotion_name,omitempty"` 124 | CreativeName bigquery.NullString `bigquery:"creative_name" json:"creative_name,omitempty"` 125 | CreativeSlot bigquery.NullString `bigquery:"creative_slot" json:"creative_slot,omitempty"` 126 | } 127 | -------------------------------------------------------------------------------- /GA4/bqstreaming/ingestuserevents.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package ingestuserevents contains an HTTP Cloud Function. 16 | package ingestuserevents 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "log" 24 | "net/http" 25 | "net/url" 26 | "os" 27 | "strconv" 28 | "strings" 29 | "time" 30 | 31 | "cloud.google.com/go/bigquery" 32 | "cloud.google.com/go/logging" 33 | "google.golang.org/api/iterator" 34 | ) 35 | 36 | // userEventIngester is a tool to ingest user events. 37 | // The cloud function's request body should includes the following public 38 | // fields. 39 | type userEventIngester struct { 40 | // ID of the project that the GA4 BigQuery belongs to. (required) 41 | BQProjectID string `json:"BQProjectID"` 42 | // ID of the dataset that the GA4 Dataset belongs to. (required) 43 | BQDatasetID string `json:"BQDatasetID"` 44 | // A duration in seconds that specifies how long ago the collection should 45 | // start from comparing to now. It should be a little bit larger than the 46 | // duration of the Cloud Scheduler that triggers the Cloud Function. 47 | // For example, if the Cloud Scheduler runs every minutes, its value could 48 | // be between 65 and 75. (required) 49 | DurationInSeconds int `json:"DurationInSeconds"` 50 | // Optional parameters 51 | // Used for debugging purpose only. This time will overwrite the current time. 52 | // Example, 20210724070849 53 | DebugCollectTime string `json:"DebugCollectTime"` 54 | // request body parameters end. 55 | 56 | // Other intermediate variables. 57 | writer io.Writer 58 | queryParameter queryParameter 59 | } 60 | 61 | func (ingester *userEventIngester) debugMode() bool { 62 | return len(ingester.DebugCollectTime) > 0 63 | } 64 | 65 | func (ingester *userEventIngester) valid() error { 66 | if len(ingester.BQProjectID) == 0 { 67 | return fmt.Errorf("Argument BQProjectID is not specified") 68 | } 69 | if len(ingester.BQDatasetID) == 0 { 70 | return fmt.Errorf("Argument BQDatasetID is not specified") 71 | } 72 | if ingester.DurationInSeconds == 0 { 73 | return fmt.Errorf("Argument DurationInSeconds is not specified") 74 | } 75 | if ingester.DurationInSeconds > int(time.Hour.Seconds()*48) { 76 | return fmt.Errorf( 77 | "DurationInSeconds must be a number of seconds smaller than 2 days") 78 | } 79 | return nil 80 | } 81 | 82 | // ingestResults prints results from a query to the Stack Overflow public dataset. 83 | func (ingester *userEventIngester) ingestResults(ctx context.Context, iter *bigquery.RowIterator) (total int, success int, err error) { 84 | if err != nil { 85 | return 0, 0, fmt.Errorf("failed to create retail user event client: %v", err) 86 | } 87 | totalNumber := 0 88 | successNumber := 0 89 | var ingestErrors []error 90 | for { 91 | var row GA4BQRow 92 | // var row []bigquery.Value 93 | err := iter.Next(&row) 94 | if err == iterator.Done { 95 | break 96 | } 97 | if err != nil { 98 | ingestErrors = append( 99 | ingestErrors, fmt.Errorf("error iterating through results: %v", err)) 100 | break 101 | } 102 | totalNumber++ 103 | 104 | b, err := json.Marshal(row) 105 | if err != nil { 106 | ingestErrors = append( 107 | ingestErrors, fmt.Errorf("failed to marshal row: %v", err)) 108 | continue 109 | } 110 | err = ingestRetailUserEventRawText( 111 | ctx, ingester.writer, 112 | environmentVariables.RetailProjectNumber, 113 | environmentVariables.APIKey, 114 | string(b), "ga4_bq") 115 | if err != nil { 116 | ingestErrors = append( 117 | ingestErrors, fmt.Errorf("failed to ingest: %v", err)) 118 | continue 119 | } 120 | successNumber++ 121 | } 122 | if len(ingestErrors) != 0 { 123 | return totalNumber, successNumber, fmt.Errorf("failed to ingest: %v", ingestErrors) 124 | } 125 | return totalNumber, successNumber, nil 126 | } 127 | 128 | // envVariables stores all the arguments passed through environment variables. 129 | // All the public variables should be specified in the environment variables. 130 | type envVariables struct { 131 | // Specified by environment variable MetadataProjectID. 132 | // ID of a project that the Cloud Function print logs to. You 133 | // may use the ID of the project that is running the Cloud Functions. 134 | MetadataProjectID string 135 | // Specified by environment variable RetailProjectNumber. 136 | // Number of a project that enables the retail API. 137 | RetailProjectNumber int64 138 | // Specified by environment variable APIKey. 139 | // Key of a service account that has access to the retail API. 140 | APIKey string 141 | // Specified by environment variable TaskName. 142 | // Name of the task. It is an arbitrary string to distinguish between multiple 143 | // Cloud Functions. 144 | // It affects the name of the GCS object contains the metadata and the logs. 145 | TaskName string 146 | } 147 | 148 | func (ev *envVariables) validate() error { 149 | if len(ev.MetadataProjectID) == 0 { 150 | return fmt.Errorf("Environment variable MetadataProjectID is not specified") 151 | } 152 | if ev.RetailProjectNumber == 0 { 153 | return fmt.Errorf("Environment variable RetailProjectNumber is not specified") 154 | } 155 | if len(ev.APIKey) == 0 { 156 | return fmt.Errorf("Environment variable APIKey is not specified") 157 | } 158 | if len(ev.TaskName) == 0 { 159 | return fmt.Errorf("Environment TaskName is not specified") 160 | } 161 | return nil 162 | } 163 | 164 | var environmentVariables = envVariables{ 165 | MetadataProjectID: os.Getenv("MetadataProjectID"), 166 | APIKey: os.Getenv("APIKey"), 167 | TaskName: os.Getenv("TaskName")} 168 | 169 | func init() { 170 | retailProjectNumber := os.Getenv("RetailProjectNumber") 171 | var err error 172 | environmentVariables.RetailProjectNumber, err = strconv.ParseInt( 173 | retailProjectNumber, 10, 64) 174 | if err != nil { 175 | log.Fatalf("Invalid project number: %q, %v", retailProjectNumber, err) 176 | } 177 | if err := environmentVariables.validate(); err != nil { 178 | log.Fatalf("Environments have not been set properly: %v", err) 179 | } 180 | } 181 | 182 | // IngestUserEvents collects GA4 data from BQ and ingests it to the retail API. 183 | func IngestUserEvents(w http.ResponseWriter, r *http.Request) { 184 | ingestUserEvents(w, r) 185 | } 186 | 187 | func ingestUserEvents(w io.Writer, r *http.Request) { 188 | ctx := context.Background() 189 | // Init log writer. 190 | var lw logWriter 191 | lw.Writer = w 192 | // Initializes log. 193 | var err error 194 | lw.Client, err = logging.NewClient( 195 | ctx, environmentVariables.MetadataProjectID) 196 | 197 | if err != nil { 198 | lw.logErrors(fmt.Sprintf("failed to create log client: %v", err)) 199 | } else { 200 | defer lw.Client.Close() 201 | lw.ErrLogger = lw.Client.Logger( 202 | environmentVariables.TaskName).StandardLogger(logging.Error) 203 | lw.WarningLogger = lw.Client.Logger( 204 | environmentVariables.TaskName).StandardLogger(logging.Warning) 205 | } 206 | 207 | var ingester userEventIngester 208 | ingester.writer = w 209 | if err := json.NewDecoder(r.Body).Decode(&ingester); err != nil { 210 | lw.logErrors(fmt.Sprintf("failed to parse body: %v", err)) 211 | return 212 | } 213 | 214 | // Checks parameters. 215 | if err := ingester.valid(); err != nil { 216 | lw.logErrors(fmt.Sprintf("invalid arguments: %v", err)) 217 | return 218 | } 219 | // Converts parameters to query parameters. 220 | ingester.generateQueryParameter() 221 | fmt.Fprintf(w, "Parameters are OK: %v.\n", ingester) 222 | statements := ingester.generateSQLStatements() 223 | 224 | // Queries BQ tables. 225 | rowCollections, err := query(ctx, ingester.BQProjectID, statements) 226 | if err != nil { 227 | lw.logErrors(fmt.Sprintf("failed to query database: %v", err)) 228 | return 229 | } 230 | var lastSuccessNumbers []int 231 | var lastRetrievedNumbers []int 232 | // Ingests user events. 233 | for _, rows := range rowCollections { 234 | totalNumber, ingestedNumber, err := ingester.ingestResults(ctx, rows) 235 | if err != nil { 236 | lw.logErrors(fmt.Sprintf("failed to print results: %v", err)) 237 | return 238 | } 239 | lastSuccessNumbers = append(lastSuccessNumbers, ingestedNumber) 240 | lastRetrievedNumbers = append(lastRetrievedNumbers, totalNumber) 241 | } 242 | lw.logWarnings(fmt.Sprintf( 243 | "Ingest success numbers:%v, total:%v, statements:%v", 244 | lastSuccessNumbers, lastRetrievedNumbers, statements)) 245 | } 246 | 247 | // queryParameter defines the parameters needed when querying BigQuery 248 | // databases. 249 | type queryParameter struct { 250 | // Tables that contains user events. In general cases, TableNames has a single 251 | // entity, while it has 2 entities if the time is close to 00:00:00. 252 | TableNames []string 253 | // StartTimestamp specifies the earliest time of user events. 254 | StartTimestamp int64 `json:"StartTimestamp"` 255 | // EndTimestamp specifies the latest time of user events. 256 | EndTimestamp int64 `json:"EndTimestamp"` 257 | } 258 | 259 | type logWriter struct { 260 | Client *logging.Client 261 | ErrLogger *log.Logger 262 | WarningLogger *log.Logger 263 | Writer io.Writer 264 | } 265 | 266 | func (lw *logWriter) logErrors(errorMessage string) { 267 | if lw.ErrLogger != nil { 268 | lw.ErrLogger.Println(errorMessage) 269 | } 270 | fmt.Fprintln(lw.Writer, errorMessage) 271 | } 272 | 273 | func (lw *logWriter) logWarnings(errorMessage string) { 274 | if lw.WarningLogger != nil { 275 | lw.WarningLogger.Println(errorMessage) 276 | } 277 | fmt.Fprintln(lw.Writer, errorMessage) 278 | } 279 | 280 | func (ingester *userEventIngester) generateQueryParameter() error { 281 | // The end time of a collection. 282 | ct := time.Now() 283 | tables := []string{"events_intraday_*"} 284 | duration := time.Second * time.Duration(ingester.DurationInSeconds) 285 | // There may be multiple tables if the DebugCollectTime parameter is 286 | // specified. Historical data is stored in different tables named by date. 287 | if ingester.debugMode() { 288 | var err error 289 | ct, err = time.Parse("20060102150405", ingester.DebugCollectTime) 290 | if err != nil { 291 | return fmt.Errorf( 292 | "Failed to parse DebugCollectTime %q: %v", 293 | ingester.DebugCollectTime, err) 294 | } 295 | tables = generateTableName(ct, duration) 296 | } 297 | for _, t := range tables { 298 | fullname := fmt.Sprintf( 299 | "%s.%s.%s", ingester.BQProjectID, ingester.BQDatasetID, t) 300 | ingester.queryParameter.TableNames = append( 301 | ingester.queryParameter.TableNames, fullname) 302 | } 303 | ingester.queryParameter.StartTimestamp = ct.Add( 304 | -(duration + time.Second)).UnixNano() / 1000 305 | ingester.queryParameter.EndTimestamp = ct.UnixNano() / 1000 306 | return nil 307 | } 308 | 309 | func generateTableName(t time.Time, duration time.Duration) []string { 310 | start := t.Add(-duration) 311 | // Mon Jan 2 15:04:05 MST 2006 312 | ret := []string{fmt.Sprintf("events_%s", t.Format("20060102"))} 313 | if start.YearDay() != t.YearDay() { 314 | ret = append(ret, fmt.Sprintf("events_%s", start.Format("20060102"))) 315 | } 316 | return ret 317 | } 318 | 319 | var supportedEventNames = []string{ 320 | "add_to_cart", "purchase", "view_search_results", "view_item"} 321 | 322 | func generateEventNamesBlock() string { 323 | var ret strings.Builder 324 | ret.WriteString("(") 325 | for i, name := range supportedEventNames { 326 | if i != 0 { 327 | ret.WriteString(" or ") 328 | } 329 | ret.WriteString(fmt.Sprintf("event_name='%s'", name)) 330 | } 331 | ret.WriteString(")") 332 | return ret.String() 333 | } 334 | 335 | func (ingester *userEventIngester) generateSQLStatements() []string { 336 | var res []string 337 | for _, t := range ingester.queryParameter.TableNames { 338 | sqlStatement := fmt.Sprintf( 339 | "SELECT * FROM `%s` where event_timestamp >= %d and event_timestamp < %d and %s ", 340 | t, ingester.queryParameter.StartTimestamp, 341 | ingester.queryParameter.EndTimestamp, generateEventNamesBlock()) 342 | res = append(res, sqlStatement) 343 | } 344 | return res 345 | } 346 | 347 | // query returns a row iterator suitable for reading query results. 348 | func query(ctx context.Context, BQProjectID string, statements []string) ( 349 | []*bigquery.RowIterator, error) { 350 | client, err := bigquery.NewClient(ctx, BQProjectID) 351 | if err != nil { 352 | return nil, fmt.Errorf("failed to create bigquery client: %v", err) 353 | } 354 | defer client.Close() 355 | ret := []*bigquery.RowIterator{} 356 | for _, s := range statements { 357 | query := client.Query(s) 358 | ri, err := query.Read(ctx) 359 | if err != nil { 360 | return nil, err 361 | } 362 | ret = append(ret, ri) 363 | } 364 | return ret, nil 365 | } 366 | 367 | const userEventRequestFormat = "https://retail.googleapis.com/v2alpha/projects/%d/locations/global/catalogs/default_catalog/userEvents:collect?key=%s&userEvent=&raw_json=%s&prebuilt_rule=%s" 368 | 369 | func ingestRetailUserEventRawText(ctx context.Context, w io.Writer, projectNumber int64, key string, rawText string, rule string) error { 370 | fmt.Fprintf(w, "ingesting:\n%v\n", rawText) 371 | 372 | request := fmt.Sprintf(userEventRequestFormat, projectNumber, key, url.QueryEscape(rawText), rule) 373 | 374 | resp, err := http.Get(request) 375 | if err != nil { 376 | return fmt.Errorf("failed to send CollectUserEvent request %s: %v", request, err) 377 | } 378 | if resp.StatusCode != 200 { 379 | return fmt.Errorf("failed to ingest through CollectUserEvent %s: %v", request, resp) 380 | } 381 | return nil 382 | } 383 | -------------------------------------------------------------------------------- /GCF/demo.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Product ID:
9 | 10 |

11 | 12 |
13 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /GCF/demo2.html: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 |
12 | 13 |
14 | -------------------------------------------------------------------------------- /GCF/python/autocomplete.py: -------------------------------------------------------------------------------- 1 | """ Google Cloud Function to return JSON or HTML from autocomplete 2 | This sample uses the Retail API with client libraries 3 | 4 | This can be used for doing AJAX/client side calls to get autocomplete results 5 | and render in a div below search box. 6 | 7 | Configure the GCF to use a service account which has Retail Viewer Role 8 | """ 9 | 10 | import google.auth 11 | import json 12 | 13 | from google.cloud import retail 14 | from google.oauth2 import service_account 15 | from google.protobuf.json_format import MessageToDict 16 | 17 | PROJECT_NUMBER='' 18 | 19 | credentials, project = google.auth.default() 20 | 21 | # For local testing you may want to do something like this if 22 | # your default credentials don't have Retail/Recommendations Viewer Role 23 | # SERVICE_ACCOUNT_FILE = '' 24 | # credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) 25 | # You can also just set an environment variable: 26 | # export GOOGLE_APPLICATION_CREDENTIALS=/path/to/local/SA-key-file 27 | 28 | client = retail.CompletionServiceClient(credentials=credentials) 29 | 30 | def complete(request): 31 | 32 | if request.args and 'q' in request.args: 33 | query_string = request.args.get('q') 34 | else: 35 | query_string = '' 36 | 37 | request = retail.CompleteQueryRequest( 38 | catalog='projects/' + PROJECT_NUMBER + '/locations/global/catalogs/default_catalog', 39 | dataset='cloud-retail', 40 | query=query_string, 41 | ) 42 | 43 | # Make the request 44 | response = client.complete_query(request=request) 45 | 46 | # Handle the response 47 | res = MessageToDict(response._pb) 48 | 49 | result = 'displayAC(' + json.dumps(res) + ')' 50 | 51 | # Set as needed for your CORS policy: 52 | headers = { 53 | 'Access-Control-Allow-Origin': '*' 54 | } 55 | 56 | return (result,200,headers) 57 | -------------------------------------------------------------------------------- /GCF/python/recommendation.py: -------------------------------------------------------------------------------- 1 | """ Google Cloud Function to return JSON or HTML from predict response 2 | This sample uses the Retail API with client libraries 3 | 4 | This can be used for doing AJAX/client side calls to get prediction results 5 | and render in a div. 6 | 7 | Configure the GCF to use a service account which has Retail Editor Role 8 | """ 9 | 10 | import google.auth 11 | import json 12 | 13 | from google.cloud import retail 14 | from google.oauth2 import service_account 15 | 16 | PROJECT_NUMBER='' 17 | 18 | credentials, project = google.auth.default( 19 | scopes=[ 20 | 'https://www.googleapis.com/auth/cloud-platform' 21 | ] 22 | ) 23 | 24 | # For local testing you may want to do something like this if 25 | # your default credentials don't have Retail/Recommendations Viewer Role 26 | # SERVICE_ACCOUNT_FILE = '' 27 | # credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) 28 | # You can also just set an environment variable: 29 | # export GOOGLE_APPLICATION_CREDENTIALS=/path/to/local/SA-key-file 30 | 31 | client = retail.PredictionServiceClient(credentials=credentials) 32 | 33 | def recommend(request): 34 | 35 | if request.args and 'visitorid' in request.args: 36 | visitorid = request.args.get('visitorid') 37 | else: 38 | visitorid = "" 39 | 40 | if request.args and 'productid' in request.args: 41 | productid = request.args.get('productid') 42 | else: 43 | productid = "" 44 | 45 | if request.args and 'num' in request.args: 46 | pageSize = request.args.get('num') 47 | else: 48 | pageSize = 4 49 | 50 | if request.args and 'placement' in request.args: 51 | placement = request.args.get('placement') 52 | else: 53 | placement = 'product_detail' 54 | 55 | user_event = { 56 | 'event_type': 'detail-page-view', 57 | 'visitor_id': visitorid, 58 | 'product_details': [{ 59 | 'product': { 60 | 'id': productid, 61 | } 62 | }] 63 | } 64 | 65 | predict_request = { 66 | "placement": 67 | 'projects/' + PROJECT_NUMBER + 68 | '/locations/global/catalogs/default_catalog/placements/' + placement, 69 | 70 | "user_event": user_event, 71 | 72 | "page_size": pageSize, 73 | 74 | "filter": 'filterOutOfStockItems', 75 | 76 | "params": { 77 | "returnProduct": True, 78 | "returnScore": True 79 | } 80 | } 81 | 82 | response = client.predict(predict_request) 83 | 84 | # Configure as necessary - here we just return a few fields 85 | # that we need for rendering the results 86 | items = [] 87 | for rec in response.results: 88 | product = rec.metadata.get('product') 89 | images = product.get('images') 90 | 91 | # Customize as needed: 92 | item = { 93 | "id": rec.id, 94 | "title": product.get('title'), 95 | "uri": product.get('uri'), 96 | "img": images[1]['uri'] 97 | } 98 | 99 | items.append(item) 100 | 101 | # option to return full div or JSON 102 | if request.args and 'html' in request.args: 103 | result = '
' 104 | 105 | for item in items: 106 | result = (result + '' + 107 | '') 108 | 109 | result = result + '
' 110 | 111 | else: 112 | result = 'displayRecs(' + json.dumps(items) + ')' 113 | 114 | # Set as needed for your CORS policy: 115 | headers = { 116 | 'Access-Control-Allow-Origin': '*' 117 | } 118 | 119 | return (result,200,headers) 120 | -------------------------------------------------------------------------------- /GCF/python/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-retail 2 | -------------------------------------------------------------------------------- /GCF/python/search.py: -------------------------------------------------------------------------------- 1 | """ Google Cloud Function to return JSON or HTML from search response 2 | This sample uses the Retail API with client libraries 3 | 4 | This can be used for doing AJAX/client side calls to get search results 5 | and render in a div. 6 | 7 | Configure the GCF to use a service account which has Retail Editor Role 8 | """ 9 | 10 | import google.auth 11 | import json 12 | 13 | from google.cloud import retail 14 | from google.oauth2 import service_account 15 | from google.protobuf.json_format import MessageToDict 16 | 17 | PROJECT_NUMBER='' 18 | 19 | credentials, project = google.auth.default( 20 | scopes=[ 21 | 'https://www.googleapis.com/auth/cloud-platform' 22 | ] 23 | ) 24 | 25 | # For local testing you may want to do something like this if 26 | # your default credentials don't have Retail/Recommendations Viewer Role 27 | # SERVICE_ACCOUNT_FILE = '' 28 | # credentials = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE) 29 | # You can also just set an environment variable: 30 | # export GOOGLE_APPLICATION_CREDENTIALS=/path/to/local/SA-key-file 31 | 32 | client = retail.SearchServiceClient(credentials=credentials) 33 | 34 | def search(request): 35 | 36 | if request.args and 'visitorid' in request.args: 37 | visitorid = request.args.get('visitorid') 38 | else: 39 | visitorid = '' 40 | 41 | if request.args and 'query' in request.args: 42 | query = request.args.get('query') 43 | else: 44 | query = '' 45 | 46 | if request.args and 'placement' in request.args: 47 | placement = request.args.get('placement') 48 | else: 49 | placement = 'default_search' 50 | 51 | body = { 52 | 'query': query, 53 | 'visitor_id': visitorid 54 | } 55 | 56 | search_request = { 57 | 'placement': 58 | 'projects/' + PROJECT_NUMBER + 59 | '/locations/global/catalogs/default_catalog/placements/' + placement, 60 | 'query': query, 61 | 'visitor_id': visitorid 62 | } 63 | 64 | response = client.search(search_request) 65 | 66 | res = MessageToDict(response._pb) 67 | 68 | result = 'displaySearch(' + json.dumps(res) + ')' 69 | 70 | # Set as needed for your CORS policy: 71 | headers = { 72 | 'Access-Control-Allow-Origin': '*' 73 | } 74 | 75 | return (result,200,headers) 76 | -------------------------------------------------------------------------------- /GCF/python/testautocomplete.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # command line test for search.py GCF 4 | 5 | from unittest.mock import Mock 6 | 7 | import autocomplete 8 | 9 | data = {"q": "cat"} 10 | req = Mock(get_json=Mock(return_value=data), args=data) 11 | 12 | result = autocomplete.complete(req) 13 | print(result) 14 | -------------------------------------------------------------------------------- /GCF/python/testrecs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # Command line test for recommendion.py GCF 4 | 5 | from unittest.mock import Mock 6 | 7 | import recommendation 8 | 9 | def test_recommendation(): 10 | data = {"visitorid": "goo", "productid": "12345"} 11 | req = Mock(get_json=Mock(return_value=data), args=data) 12 | 13 | result = recommendation.recommend(req) 14 | print(result) 15 | 16 | test_recommendation() 17 | -------------------------------------------------------------------------------- /GCF/python/testsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # command line test for search.py GCF 4 | 5 | from unittest.mock import Mock 6 | 7 | import search 8 | 9 | data = {"visitorid": "fakevisitorid", "productid": "12345"} 10 | req = Mock(get_json=Mock(return_value=data), args=data) 11 | 12 | result = search.search(req) 13 | print(result) 14 | -------------------------------------------------------------------------------- /GTM/sGTM.js: -------------------------------------------------------------------------------- 1 | const encodeUriComponent = require('encodeUriComponent'); 2 | const generateRandom = require('generateRandom'); 3 | const getCookieValues = require('getCookieValues'); 4 | const getEventData = require('getEventData'); 5 | const logToConsole = require('logToConsole'); 6 | const makeString = require('makeString'); 7 | const sendHttpGet = require('sendHttpGet'); 8 | const setCookie = require('setCookie'); 9 | const json = require('JSON'); 10 | const makeNumber = require('makeNumber'); 11 | const getType = require('getType'); 12 | const VISITOR_ID_COOKIE = '_ga'; 13 | const MAX_USER_ID = 1000000000; 14 | 15 | /** 16 | * Transform an GA4 event to Recommendations AI public beta event 17 | * proto. The enhanced ecommerce user event is represented via a list of 18 | * ecommerce products and top level metadata such as revenue. 19 | * 20 | * @param {?string} eventType event type of the user event. eg: 21 | * 'purchase-complete', 'add-to-cart' 22 | * @param {!Object} ecommerceData An object of action fields related with 23 | * the user event. 24 | * @param {?string} currencyCode A three-character ISO-4217 code. If this is 25 | * not set, the currency code is set as USD by default. 26 | * @return {!Object} An object representation of Recommendations AI public 27 | * proto. 28 | */ 29 | function transformGA4EventToCloudRetail( 30 | eventType, currencyCode) { 31 | // Set event type. 32 | let userEvent = {'eventType': eventType}; 33 | let products = getEventData('items'); 34 | if (getType(products) !== 'array') { 35 | return userEvent; 36 | } 37 | logToConsole(userEvent); 38 | let productDetails = []; 39 | 40 | /** 41 | * Helper function to add product features into categoricalFeatures or 42 | * NumericalFeatures depends on feature value type. Currently only string 43 | * and number are supported. 44 | * 45 | * @param {!Array>} productList A list of products, each 46 | * is an Object. 47 | */ 48 | const addProductToProductDetails = (productList) => { 49 | for (let j = 0; j < productList.length; j++) { 50 | let ecommerceProduct = productList[j]; 51 | logToConsole(ecommerceProduct); 52 | let productDetail = {'product': {}}; 53 | for (let key in ecommerceProduct) { 54 | if (key === 'item_id') { 55 | productDetail.product.id = makeString(ecommerceProduct[key]); 56 | } else if (key === 'quantity') { 57 | if (ecommerceProduct[key] !== '') { 58 | productDetail.quantity = ecommerceProduct[key]; 59 | } 60 | } 61 | } 62 | // Push back product if the product id is defined. 63 | if (productDetail.product.id !== undefined) { 64 | if (productDetail.quantity === undefined) { 65 | productDetail.quantity = 1; 66 | } 67 | logToConsole(productDetail); 68 | productDetails.push(productDetail); 69 | } 70 | } 71 | }; 72 | 73 | const createTransactionInfo = () => { 74 | const purchaseTransaction = {}; 75 | var keys = ['transaction_id', 'value', 'tax', 'shipping']; 76 | for (let key of keys) { 77 | const value = getEventData(key); 78 | if (key === 'items') { 79 | return; 80 | } 81 | if (key === 'transaction_id') { 82 | purchaseTransaction.id = value; 83 | } else if (key === 'value') { 84 | if (value !== '') { 85 | purchaseTransaction.revenue = makeNumber(value); 86 | } 87 | } else if (key === 'tax') { 88 | if (value !== '') { 89 | purchaseTransaction.tax = makeNumber(value); 90 | } 91 | } else if (key === 'shipping') { 92 | if (value !== '') { 93 | purchaseTransaction.cost = makeNumber(value); 94 | } 95 | } 96 | } 97 | 98 | if (purchaseTransaction.revenue === undefined) { 99 | purchaseTransaction.revenue = 0; 100 | } 101 | if (purchaseTransaction.currencyCode === undefined) { 102 | purchaseTransaction.currencyCode = currencyCode || 'USD'; 103 | } 104 | return purchaseTransaction; 105 | }; 106 | 107 | 108 | logToConsole(userEvent); 109 | logToConsole(products); 110 | addProductToProductDetails(products); 111 | logToConsole(productDetails); 112 | userEvent.productDetails = productDetails; 113 | if (eventType === 'purchase-complete') { 114 | userEvent.purchaseTransaction = createTransactionInfo(); 115 | } 116 | return userEvent; 117 | } 118 | 119 | function ingestData(event) { 120 | logToConsole(event); 121 | const pageLocation = getEventData('page_location'); 122 | const automlUrl = 'https://retail.googleapis.com/v2' + 123 | '/projects/' + encodeUriComponent(data.projectNumber) + 124 | '/locations/global/catalogs/default_catalog' + 125 | '/userEvents:collect?key=' + encodeUriComponent(data.apiKey) + 126 | '&uri=' + encodeUriComponent(pageLocation.substring(0, 1500)) + 127 | '&user_event=' + encodeUriComponent(json.stringify(event)); 128 | 129 | sendHttpGet(automlUrl, (statusCode) => { 130 | if (statusCode >= 200 && statusCode < 300) { 131 | data.gtmOnSuccess(); 132 | } else { 133 | data.gtmOnFailure(); 134 | } 135 | }); 136 | } 137 | 138 | 139 | function ingestGA4Data(ecommerceData, eventType) { 140 | const currencyCode = getEventData('currency'); 141 | let ga4ToRetailMap = { 142 | 'view_item': 'detail-page-view', 143 | 'purchase': 'purchase-complete', 144 | 'add_to_cart': 'add-to-cart' 145 | }; 146 | if (ga4ToRetailMap[eventType] || 147 | ((eventType == 'view_item_list' || 148 | eventType === 'view_search_results') && 149 | data.searchQuery !== undefined && 150 | data.searchQuery !== "")) { 151 | const ga4EventType = (eventType == 'view_item_list' || 152 | eventType === 'view_search_results') ? 'search' : ga4ToRetailMap[eventType]; 153 | const cloudRetail = transformGA4EventToCloudRetail(ga4EventType, currencyCode); 154 | const gaCookiePrefix = data.gaCookiePrefix || ""; 155 | cloudRetail.visitorId = gaCookiePrefix + data.visitorId; 156 | cloudRetail.attributes = {}; 157 | cloudRetail.attributes.tag = {}; 158 | cloudRetail.attributes.tag.text = ["SGTM"]; 159 | const experimentIds = getCookieValues('_gaexp'); 160 | if (experimentIds.length) { 161 | cloudRetail.experimentIds = experimentIds; 162 | } 163 | if (data.searchQuery !== undefined && data.searchQuery !== "") { 164 | cloudRetail.searchQuery = data.searchQuery; 165 | } 166 | ingestData(cloudRetail); 167 | } else { 168 | data.gtmOnSuccess(); 169 | } 170 | return; 171 | } 172 | 173 | // The event name is taken from either the tag's configuration or from the 174 | // event. Configuration data comes into the sandboxed code as a predefined 175 | // variable called 'data'. 176 | const eventName = data.eventName || getEventData('event_name'); 177 | ingestGA4Data(data, eventName); 178 | -------------------------------------------------------------------------------- /JSON/README.md: -------------------------------------------------------------------------------- 1 | all-events-schema.json - BigQuery Schema for User Events -------------------------------------------------------------------------------- /JSON/all-events-schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "eventTime", 4 | "type": "TIMESTAMP", 5 | "mode": "REQUIRED" 6 | }, 7 | { 8 | "fields" : [ 9 | { 10 | "fields" : [ 11 | { 12 | "fields" : [ 13 | { 14 | "mode" : "NULLABLE", 15 | "fields" : [ 16 | { 17 | "mode" : "REPEATED", 18 | "name" : "value", 19 | "type" : "STRING" 20 | } 21 | ], 22 | "name" : "placement_id", 23 | "type" : "RECORD" 24 | }, 25 | { 26 | "mode" : "NULLABLE", 27 | "fields" : [ 28 | { 29 | "mode" : "REPEATED", 30 | "name" : "value", 31 | "type" : "STRING" 32 | } 33 | ], 34 | "type" : "RECORD", 35 | "name" : "context_event_type" 36 | }, 37 | { 38 | "name" : "context_item_ids", 39 | "fields" : [ 40 | { 41 | "type" : "INTEGER", 42 | "name" : "value", 43 | "mode" : "REPEATED" 44 | } 45 | ], 46 | "type" : "RECORD", 47 | "mode" : "NULLABLE" 48 | } 49 | ], 50 | "name" : "categoricalFeatures", 51 | "type" : "RECORD", 52 | "mode" : "NULLABLE" 53 | } 54 | ], 55 | "name" : "eventAttributes", 56 | "type" : "RECORD", 57 | "mode" : "NULLABLE" 58 | }, 59 | { 60 | "mode": "NULLABLE", 61 | "name": "recommendationToken", 62 | "type": "STRING" 63 | }, 64 | { 65 | "name": "uri", 66 | "type": "STRING", 67 | "mode": "NULLABLE" 68 | }, 69 | { 70 | "name": "experimentIds", 71 | "mode": "REPEATED", 72 | "type": "STRING" 73 | } 74 | ], 75 | "mode": "NULLABLE", 76 | "name": "eventDetail", 77 | "type": "RECORD" 78 | }, 79 | { 80 | "fields": [ 81 | { 82 | "mode": "NULLABLE", 83 | "name": "userId", 84 | "type": "STRING" 85 | }, 86 | { 87 | "mode": "REQUIRED", 88 | "name": "visitorId", 89 | "type": "STRING" 90 | }, 91 | { 92 | "mode": "NULLABLE", 93 | "name": "directUserRequest", 94 | "type": "BOOLEAN" 95 | } 96 | ], 97 | "mode": "REQUIRED", 98 | "name": "userInfo", 99 | "type": "RECORD" 100 | }, 101 | { 102 | "fields": [ 103 | { 104 | "fields": [ 105 | { 106 | "fields": [ 107 | { 108 | "mode": "NULLABLE", 109 | "name": "cost", 110 | "type": "FLOAT" 111 | }, 112 | { 113 | "mode": "NULLABLE", 114 | "name": "manufacturing", 115 | "type": "FLOAT" 116 | } 117 | ], 118 | "mode": "NULLABLE", 119 | "name": "costs", 120 | "type": "RECORD" 121 | }, 122 | { 123 | "fields": [ 124 | { 125 | "mode": "NULLABLE", 126 | "name": "local", 127 | "type": "FLOAT" 128 | }, 129 | { 130 | "mode": "NULLABLE", 131 | "name": "state", 132 | "type": "FLOAT" 133 | } 134 | ], 135 | "mode": "NULLABLE", 136 | "name": "taxes", 137 | "type": "RECORD" 138 | }, 139 | { 140 | "mode": "NULLABLE", 141 | "name": "currencyCode", 142 | "type": "STRING" 143 | }, 144 | { 145 | "mode": "NULLABLE", 146 | "name": "revenue", 147 | "type": "FLOAT" 148 | }, 149 | { 150 | "mode": "NULLABLE", 151 | "name": "id", 152 | "type": "INTEGER" 153 | } 154 | ], 155 | "mode": "NULLABLE", 156 | "name": "purchaseTransaction", 157 | "type": "RECORD" 158 | }, 159 | { 160 | "fields": [ 161 | { 162 | "mode": "NULLABLE", 163 | "name": "quantity", 164 | "type": "INTEGER" 165 | }, 166 | { 167 | "mode": "NULLABLE", 168 | "name": "availableQuantity", 169 | "type": "INTEGER" 170 | }, 171 | { 172 | "mode": "NULLABLE", 173 | "name": "displayPrice", 174 | "type": "FLOAT" 175 | }, 176 | { 177 | "mode": "NULLABLE", 178 | "name": "stockState", 179 | "type": "STRING" 180 | }, 181 | { 182 | "mode": "NULLABLE", 183 | "name": "originalPrice", 184 | "type": "FLOAT" 185 | }, 186 | { 187 | "mode": "NULLABLE", 188 | "name": "id", 189 | "type": "INTEGER" 190 | }, 191 | { 192 | "mode": "NULLABLE", 193 | "name": "currencyCode", 194 | "type": "STRING" 195 | } 196 | ], 197 | "mode": "REPEATED", 198 | "name": "productDetails", 199 | "type": "RECORD" 200 | }, 201 | { 202 | "mode": "NULLABLE", 203 | "name": "cartId", 204 | "type": "INTEGER" 205 | }, 206 | { 207 | "mode": "NULLABLE", 208 | "name": "searchQuery", 209 | "type": "STRING" 210 | }, 211 | { 212 | "fields": [ 213 | { 214 | "mode": "REPEATED", 215 | "name": "categories", 216 | "type": "STRING" 217 | } 218 | ], 219 | "mode": "REPEATED", 220 | "name": "pageCategories", 221 | "type": "RECORD" 222 | }, 223 | { 224 | "mode": "NULLABLE", 225 | "name": "listId", 226 | "type": "STRING" 227 | } 228 | ], 229 | "mode": "NULLABLE", 230 | "name": "productEventDetail", 231 | "type": "RECORD" 232 | }, 233 | { 234 | "mode": "REQUIRED", 235 | "name": "eventType", 236 | "type": "STRING" 237 | } 238 | ] 239 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Retail AI 2 | 3 | Samples and Examples for Google Retail AI (Recommendations AI and Retail Search) 4 | 5 | #### All samples provided as-is without warranty 6 | 7 | Please refer to the official documentation for more examples and up-to-date API specifics: 8 | 9 | * [Google Retail Documentation](https://cloud.google.com/retail/docs) 10 | * [Google Retail API Reference](https://cloud.google.com/retail/docs/reference/rest) 11 | 12 | *** 13 | ### Directories: 14 | - [GCF](/GCF/) - Sample AJAX implementation for client-side predictions using Google Cloud Functions 15 | - [JSON](/JSON/) - Example JSON for Retail API 16 | - [beta](/beta/) - Old examples using the old, beta Recommendations AI API 17 | - [curl](/curl/) - Examples for doing common tasks with curl commands (REST API) 18 | - [php](/php/) - Example predict request in php 19 | - [python](/python/) - Python examples using retail API client libraries 20 | -------------------------------------------------------------------------------- /beta/README.md: -------------------------------------------------------------------------------- 1 | ## Recommendations AI Beta API Examples 2 | 3 | Samples and Examples for Google Recommendations AI 4 | 5 | Examples in this directory use the old beta Recommendations API, which will be deprecated eventually: 6 | * [Old Google Recommendations AI Beta Documentation] (https://cloud.google.com/recommendations-ai/docs/) 7 | 8 | The Recommendations API has migrated to the Retail API, please see the latest GA docs here: 9 | * [Retail API] (https://cloud.google.com/retail/recommendations-ai/docs) 10 | 11 | Also, while the examples here using the [discovery API](https://developers.google.com/discovery/) will continue to work, the Retail Client Libraries are the preferred method: 12 | * [Retail Client Libraries] (https://cloud.google.com/retail/docs/reference/libraries) 13 | -------------------------------------------------------------------------------- /beta/curl/catalog_import.json: -------------------------------------------------------------------------------- 1 | { 2 | "errorsConfig": { 3 | "gcsPrefix": "gs://recommendations-sandbox-test/errors/" 4 | }, 5 | "inputConfig": { 6 | "gcsSource": { 7 | "inputUris": ["gs://recommendations-sandbox-test/gcs_catalog_sample.txt"] 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /beta/curl/create_item.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Add a catalog item from testitem.json file 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -X POST \ 9 | -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 10 | -H "Content-Type: application/json; charset=utf-8" \ 11 | --data @./testitem.json \ 12 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/catalogItems" 13 | -------------------------------------------------------------------------------- /beta/curl/delete_events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Delete events using the purge method: 4 | # https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/projects.locations.catalogs.eventStores.userEvents/purge 5 | # The filter field specifies a time range (eventTime) and/or eventType 6 | # Note the encoding for the filter parameter, and times are in "zulu" format. In this case we've encoded: 7 | # filter=eventTime > "2000-04-23T18:25:43.511Z" eventTime < "2001-04-23T18:30:43.511Z" 8 | # force:true will cause an actual delete, force:false (the default) will return only return a list of events that match 9 | 10 | export GOOGLE_APPLICATION_CREDENTIALS= 11 | export PROJECT_NUM= 12 | 13 | curl -X POST -H "Authorization: Bearer "$(gcloud auth application-default print-access-token)"" \ 14 | -H "Content-Type: application/json; charset=utf-8" \ 15 | --data '{ 16 | "filter":"eventTime > \"2019-12-23T18:25:43.511Z\" eventTime < \"2019-12-23T18:30:43.511Z\"", 17 | "force":"true" 18 | }' \ 19 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT_NUM/locations/global/catalogs/default_catalog/eventStores/default_event_store/userEvents:purge" 20 | -------------------------------------------------------------------------------- /beta/curl/delete_item.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Delete an item from the catalog 4 | # Usage: ./delete_item.sh itemid 5 | 6 | export GOOGLE_APPLICATION_CREDENTIALS= 7 | export PROJECT= 8 | 9 | curl -X DELETE \ 10 | -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 11 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/catalogItems/$1" 12 | -------------------------------------------------------------------------------- /beta/curl/delete_key.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Delete a registered predict key 4 | # usage: ./delete_key 5 | 6 | export GOOGLE_APPLICATION_CREDENTIALS= 7 | export PROJECT= 8 | 9 | curl -X DELETE \ 10 | -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 11 | -H "Content-Type: application/json; charset=utf-8" \ 12 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/eventStores/default_event_store/predictionApiKeyRegistrations/$1" 13 | -------------------------------------------------------------------------------- /beta/curl/event_import.json: -------------------------------------------------------------------------------- 1 | { 2 | "errorsConfig": { 3 | "gcsPrefix": "gs://recommendations-sandbox-test/errors/" 4 | }, 5 | "inputConfig": { 6 | "gcsSource": { 7 | "inputUris": ["gs://recommendations-sandbox-test/1-user_event.txt"] 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /beta/curl/gcs_catalog_import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Import catalog items from GCS bucket, as specified in catalog_import.json 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -X POST \ 9 | -H "Content-Type: application/json; charset=utf-8" -d @./catalog_import.json \ 10 | -H "Authorization: Bearer "$(gcloud auth application-default print-access-token)"" \ 11 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/catalogItems:import" 12 | -------------------------------------------------------------------------------- /beta/curl/gcs_catalog_sample.txt: -------------------------------------------------------------------------------- 1 | {'id': '68955', 'categoryHierarchies': [{'categories': ['Postcards', 'US State & Town Views', 'California', 'Mountain View']}], 'title': 'Castro Street View', 'description': 'A view of downtown Mountain View', 'languageCode': 'en', 'productMetadata': {'exactPrice': {'displayPrice': 5.95}, 'currencyCode': 'USD', 'availableQuantity': '1', 'canonicalProductUri': 'https://www.example.com/6895/'}} 2 | {'id': '68956', 'categoryHierarchies': [{'categories': ['Postcards', 'US State & Town Views', 'California', 'Santa Catalina Island']}], 'title': 'Avalon Bay', 'description': 'A view of Avalon from the surrounding hills that overlook the beautiful bay, with hundreds of pleasure craft quietly rocking at anchor, is a thrilling and unforgettable sight.\n', 'itemAttributes': {'categoricalFeatures': {'condition': {'value': ['unused']}, 'addDate': {'value': ['2007-10-27']}, 'county': {'value': ['Los Angeles']}, 'city': {'value': ['Santa Catalina Island']}, 'era': {'value': ['Chrome']}, 'state': {'value': ['CA']}, 'publisher': {'value': ['Western Publishing & Novelty Co.']}, 'type': {'value': ['Postcard']}}, 'numericalFeatures': {'width_in': {'value': [5.5]}, 'height_in': {'value': [3.5]}}}, 'languageCode': 'en', 'tags': ['onsale'], 'productMetadata': {'exactPrice': {'displayPrice': 5.95}, 'currencyCode': 'USD', 'availableQuantity': '1', 'canonicalProductUri': 'https://www.example.com/68956/', 'images': [{'uri': 'https://www.example.com/images/test/card00258_fr.jpg', 'height': 379, 'width': 600}]}} 3 | -------------------------------------------------------------------------------- /beta/curl/gcs_event_import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Import events from GCS bucket, as specified in event_import.json 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -X POST \ 9 | -H "Content-Type: application/json; charset=utf-8" -d @/tmp/event_import.json \ 10 | -H "Authorization: Bearer "$(gcloud auth application-default print-access-token)"" \ 11 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/eventStores/default_event_store/userEvents:import" 12 | -------------------------------------------------------------------------------- /beta/curl/gcs_import_status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Display status of GCS catalog or event import job 4 | # Will usually show "done" - errors written to errorsConfig bucket 5 | 6 | export GOOGLE_APPLICATION_CREDENTIALS= 7 | export PROJECT= 8 | 9 | curl -H "Authorization: Bearer "$(gcloud auth application-default print-access-token)"" \ 10 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/operations/$1" 11 | -------------------------------------------------------------------------------- /beta/curl/gcs_user_event_sample.txt: -------------------------------------------------------------------------------- 1 | {"eventType": "detail-page-view", "userInfo": {"visitorId": "GOvboX872ktws"}, "eventDetail": {"uri": "https://www.example.com/68956/"}, "productEventDetail": {"productDetails": [{"id": "68956", "displayPrice": 4.95, "availableQuantity": 1, "currencyCode": "USD"}]}, "eventTime": "2019-05-01T19:12:24.402017Z"} 2 | {"eventType": "add-to-cart", "userInfo": {"visitorId": "GOvboX872ktws"}, "eventDetail": {"uri": "https://www.example.com/cart.php?add"}, "productEventDetail": {"productDetails": [{"id": "68956", "displayPrice": 4.95, "quantity": 1, "availableQuantity": 1, "currencyCode": "USD"}]}, "eventTime": "2019-05-01T19:12:31.625228Z"} 3 | {"eventType": "purchase-complete", "userInfo": {"visitorId": "GOvboX872ktws"}, "eventDetail": {"uri": "https://www.example.com/cart.php?mode=order_message&orderids=12345"}, "productEventDetail": {"productDetails": [{"id": "68956", "displayPrice": 4.95, "quantity": 1, "availableQuantity": 1, "currencyCode": "USD"}, {"id": "106736", "displayPrice": 2.95, "quantity": 1, "availableQuantity": 1, "currencyCode": "USD"}, {"id": "251403", "displayPrice": 8.95, "quantity": 1, "availableQuantity": 1, "currencyCode": "USD"}], "purchaseTransaction": {"id": "101103", "revenue": 19.35, "currencyCode": "USD"}}, "eventTime": "2019-05-01T19:20:47.226666Z"} 4 | -------------------------------------------------------------------------------- /beta/curl/get_catalog_item.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # usage: ./get_catalog_item.sh 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 9 | -H "Content-Type: application/json; charset=utf-8" \ 10 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/catalogItems/$1" 11 | -------------------------------------------------------------------------------- /beta/curl/importEvent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Inline event Import from inlineEvent.json 4 | # Inline event import can be used for importing small numbers of events per API call. 5 | # GCS import should be used for larger batch uploads 6 | 7 | export GOOGLE_APPLICATION_CREDENTIALS= 8 | export PROJECT= 9 | 10 | curl -X POST \ 11 | -H "Authorization: Bearer "$(gcloud auth application-default print-access-token) \ 12 | -H "Content-Type: application/json; charset=utf-8" \ 13 | --data @./inlineEvent.json \ 14 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/eventStores/default_event_store/userEvents:import" 15 | -------------------------------------------------------------------------------- /beta/curl/inlineEvent.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputConfig": { 3 | "userEventInlineSource" : { 4 | "userEvents": [ 5 | {"eventType": "detail-page-view", "userInfo": {"visitorId": "GO8YdFzqZ2bnE", "userId": "13745"}, "eventDetail": {"uri": "https://www.example.com/62705/jan-1-happy-new-year-holidays-years/"}, "productEventDetail": {"productDetails": [{"id": "62705", "displayPrice": 4.95, "availableQuantity": 1, "currencyCode": "USD"}]}, "eventTime": "2019-01-01T11:33:52.139174Z"} 6 | ] 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /beta/curl/list_catalog_items.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # List Catalog Items 4 | # Returns one page (up to 100) of items. Use nextPageToken parameter to get subsequent pages 5 | 6 | export GOOGLE_APPLICATION_CREDENTIALS= 7 | export PROJECT= 8 | 9 | curl -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 10 | -H "Content-Type: application/json; charset=utf-8" \ 11 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/catalogItems?pageSize=100" 12 | -------------------------------------------------------------------------------- /beta/curl/list_events.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Example of Listing Events 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 9 | -H "Content-Type: application/json; charset=utf-8" \ 10 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECT/locations/global/catalogs/default_catalog/eventStores/default_event_store/userEvents?pageSize=100" 11 | -------------------------------------------------------------------------------- /beta/curl/list_keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get list of registered predict keys 4 | 5 | export GOOGLE_APPLICATION_CREDENTIALS= 6 | export PROJECT= 7 | 8 | curl -X GET \ 9 | -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 10 | -H "Content-Type: application/json; charset=utf-8" \ 11 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECTID/locations/global/catalogs/default_catalog/eventStores/default_event_store/predictionApiKeyRegistrations" 12 | -------------------------------------------------------------------------------- /beta/curl/register_key.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Register a key for use with predict API 4 | # usage: ./register_key.sh 5 | 6 | export GOOGLE_APPLICATION_CREDENTIALS= 7 | export PROJECTID= 8 | 9 | curl -X POST \ 10 | -H "Authorization: Bearer $(gcloud auth application-default print-access-token)" \ 11 | -H "Content-Type: application/json; charset=utf-8" \ 12 | --data '{ 13 | "predictionApiKeyRegistration": { 14 | "apiKey": "'"$1"'" 15 | } 16 | }'\ 17 | "https://recommendationengine.googleapis.com/v1beta1/projects/$PROJECTID/locations/global/catalogs/default_catalog/eventStores/default_event_store/predictionApiKeyRegistrations" 18 | -------------------------------------------------------------------------------- /beta/curl/testitem.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "1234", 3 | "category_hierarchies": [ { "categories": [ "athletic wear", "shoes" ] } ], 4 | "title": "ABC sneakers", 5 | "description": "Sneakers for the rest of us", 6 | "language_code": "en", 7 | "tags": [ ], 8 | "product_metadata": { 9 | "exact_price": { 10 | "display_price": 99.98, 11 | "original_price": 111.99 12 | }, 13 | "costs": { 14 | "manufacturing": 35.990002, 15 | "other": 20 16 | }, 17 | "currency_code": "USD", 18 | "canonical_product_uri": "https://www.example.com/products/1234", 19 | "images": [ 20 | { 21 | "uri": "https://www.example.com/images/image1234.jpg" 22 | } 23 | ] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /beta/php/predict.php: -------------------------------------------------------------------------------- 1 | recommendationToken; 95 | 96 | $ids = ""; 97 | 98 | $idcount = 0; 99 | 100 | foreach ((array)$json->results as $item) { 101 | $id = $item->id; 102 | print "Recommendation: $id
\n"; 103 | } 104 | 105 | ?> 106 | -------------------------------------------------------------------------------- /beta/python/bq_catalog_export.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Export Catalog from BQ to json import txt file. 3 | 4 | Currently just outputs to stdout, so pipe to a file. 5 | Import files should be <500MB, so may require splitting the resulting file. 6 | """ 7 | 8 | import argparse 9 | import json 10 | import os 11 | import re 12 | from google.cloud import bigquery 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | '--service_account', 17 | required=True, 18 | type=str, 19 | help='Path to service account .json key file') 20 | parser.add_argument( 21 | '--bqtable', 22 | required=True, 23 | type=str, 24 | help='Table Name for MC Data') 25 | 26 | args = parser.parse_args() 27 | 28 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 29 | 30 | client = bigquery.Client() 31 | query = ('SELECT * FROM `' + args.bqtable + '` ' 32 | 'WHERE _PARTITIONTIME IN (' 33 | 'SELECT MAX(_PARTITIONTIME) ' 34 | 'FROM `' + args.bqtable + '`)') 35 | query_job = client.query(query) 36 | 37 | for row in query_job: 38 | # Build a CatalogItem: 39 | # https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/projects.locations.catalogs.catalogItems#CatalogItem 40 | 41 | try: # This will skip any records with missing data 42 | rec_json = {} 43 | rec_json['id'] = row.get('offer_id') 44 | rec_json['categoryHierarchies'] = { 45 | 'categories': re.split(' > |,', row.get('product_type')) 46 | } 47 | rec_json['title'] = row.get('title') 48 | rec_json['description'] = row.get('description') 49 | rec_json['languageCode'] = row.get('content_language') 50 | rec_json['productMetadata'] = ({ 51 | 'currencyCode': row.get('price')['currency'], 52 | 'canonicalProductUri': row.get('link'), 53 | 'exactPrice': { 54 | 'displayPrice': row.get('price')['value'] 55 | }, 56 | 'availableQuantity': '1', 57 | }) 58 | 59 | # Customize for extra attributes as necessary 60 | if (row.get('mpm') or row.get('gtin')): 61 | rec_json['itemAttributes'] = ({ 62 | 'categoricalFeatures': { 63 | } 64 | }) 65 | 66 | if row.get('mpm'): 67 | rec_json['itemAttributes']['categoricalFeatures']['mpm'] = ({ 68 | 'value': [row.get('mpm')] 69 | }) 70 | 71 | if row.get('gtin'): 72 | rec_json['itemAttributes']['categoricalFeatures']['gtin'] = { 73 | 'value': [row.get('gtin')] 74 | } 75 | 76 | print(json.dumps(rec_json)) 77 | except KeyError: 78 | pass 79 | -------------------------------------------------------------------------------- /beta/python/copy_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Copy events from one project to another. 3 | 4 | Service Account must have Recommendations Reader access in the source 5 | project, and Recommendations Editor access in the destination 6 | """ 7 | 8 | import argparse 9 | import os 10 | from apiclient.discovery import build 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument( 14 | '--service_account', 15 | required=True, 16 | type=str, 17 | help='path to service account .json key file') 18 | parser.add_argument( 19 | '--apikey', 20 | required=True, 21 | type=str, 22 | help='API key') 23 | parser.add_argument( 24 | '--src_project', 25 | required=True, 26 | type=str, 27 | help='Source cloud project #') 28 | parser.add_argument( 29 | '--dest_project', 30 | required=True, 31 | type=str, 32 | help='Destination cloud project #') 33 | parser.add_argument( 34 | '--start_date', 35 | required=False, 36 | type=str, 37 | help='Start Date: YYYY-MM-DD') 38 | parser.add_argument( 39 | '--end_date', 40 | required=False, 41 | type=str, 42 | help='End Date: YYYY-MM-DD') 43 | parser.add_argument( 44 | '--event_type', 45 | required=False, 46 | type=str, 47 | help='deteail-page-view, add-to-cart, purchase-complete, etc') 48 | parser.add_argument( 49 | '--events_missing_catalog_items', 50 | required=False, 51 | action='store_true', 52 | help='Return only unjoined events') 53 | 54 | args = parser.parse_args() 55 | 56 | filter_string = '' 57 | if args.start_date is not None: 58 | filter_string = (' eventTime > "' + args.start_date + 'T00:00:00.00Z" ') 59 | if args.end_date is not None: 60 | filter_string = (filter_string + ' eventTime < "' + 61 | args.end_date + 'T23:59:59.99Z" ') 62 | if args.event_type is not None: 63 | filter_string = (filter_string + ' eventType = '+ args.event_type) 64 | if args.events_missing_catalog_items: 65 | filter_string = (filter_string + ' eventsMissingCatalogItems') 66 | 67 | 68 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 69 | 70 | service = build('recommendationengine', 71 | 'v1beta1', 72 | discoveryServiceUrl= 73 | 'https://recommendationengine.googleapis.com/' 74 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 75 | 76 | next_page = '' 77 | total = 0 78 | while next_page is not None: 79 | 80 | request = service.projects().locations().catalogs().eventStores()\ 81 | .userEvents().list( 82 | parent='projects/'+ args.src_project + 83 | '/locations/global/catalogs/default_catalog'+ 84 | '/eventStores/default_event_store', 85 | filter=filter_string, 86 | pageSize=5000, pageToken=next_page) 87 | 88 | response = request.execute() 89 | 90 | if 'nextPageToken' in response: 91 | next_page = response['nextPageToken'] 92 | else: 93 | next_page = None 94 | 95 | import_body = {} 96 | import_body['inputConfig'] = {} 97 | import_body['inputConfig']['userEventInlineSource'] = {} 98 | import_body['inputConfig']['userEventInlineSource']['userEvents'] = [] 99 | 100 | for event in response['userEvents']: 101 | try: # Add currencyCode since it's not returned currently (bug #130748472) 102 | for i in range(len(event['productEventDetail']['productDetails'])): 103 | event['productEventDetail']['productDetails'][i]['currencyCode'] = 'USD' 104 | except KeyError: 105 | pass 106 | 107 | (import_body['inputConfig']['userEventInlineSource']['userEvents'] 108 | .append(event)) 109 | 110 | total = total +\ 111 | len(import_body['inputConfig']['userEventInlineSource']['userEvents']) 112 | 113 | # TODO(elarson): Add exponential backoff on the import_request 114 | # Sometimes import breaks (quota, etc) so we need to recontinue 115 | # if (total <= 635000): 116 | # continue 117 | 118 | print('Importing 5000') 119 | import_request = service.projects().locations().catalogs().eventStores()\ 120 | .userEvents().import_( 121 | parent='projects/'+ args.dest_project + 122 | '/locations/global/catalogs/default_catalog'+ 123 | '/eventStores/default_event_store', 124 | body=import_body 125 | ) 126 | 127 | response = import_request.execute() 128 | if 'done' not in response: 129 | print(response) 130 | 131 | print('Imported ' + str(total) + ' events') 132 | -------------------------------------------------------------------------------- /beta/python/curl_predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | # This sample constructs and executes a simple curl command to call the 4 | # predict API, 5 | 6 | import argparse 7 | import os 8 | import json 9 | 10 | APIKEY='' 11 | PROJECT='' 12 | PLACEMENT='' 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('itemids', 16 | help='comma-seperated list of item ids') 17 | 18 | args = parser.parse_args() 19 | 20 | data = { 21 | 'dryRun': False, 22 | 'pageSize': 4, 23 | 'params': { 24 | 'returnCatalogItem': True, 25 | 'returnItemScore': True 26 | }, 27 | 'userEvent': { 28 | 'eventType': 'detail-page-view', 29 | 'userInfo': { 30 | 'visitorId': 'visitor1', 31 | 'userId': 'user1', 32 | 'ipAddress': '0.0.0.0', 33 | 'userAgent': 'Mozilla/5.0 (Windows NT 6.1)' 34 | }, 35 | 'productEventDetail': { 36 | 'productDetails': [ 37 | ] 38 | } 39 | } 40 | } 41 | 42 | for id in args.itemids.split(','): 43 | data['userEvent']['productEventDetail']['productDetails'].append( 44 | {'id': str(id)} 45 | ) 46 | 47 | 48 | CMD='curl -X POST -H "Content-Type: application/json; charset=utf-8" ' + \ 49 | '--data \'' + json.dumps(data) + \ 50 | '\' https://recommendationengine.googleapis.com/v1beta1/projects/' + \ 51 | PROJECT + \ 52 | '/locations/global/catalogs/default_catalog/eventStores' + \ 53 | '/default_event_store/placements/' + \ 54 | PLACEMENT + ':predict?key=' + APIKEY 55 | 56 | print(CMD) 57 | os.system(CMD) 58 | 59 | -------------------------------------------------------------------------------- /beta/python/delete_catalog_items.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Script to delete items in Recommendations AI catalog. 3 | 4 | """ 5 | 6 | import argparse 7 | import os 8 | from apiclient.discovery import build 9 | from googleapiclient.errors import HttpError 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument( 13 | '--service_account', 14 | required=True, 15 | type=str, 16 | help='path to service account .json key file') 17 | parser.add_argument( 18 | '--apikey', 19 | required=True, 20 | type=str, 21 | help='API key') 22 | parser.add_argument( 23 | '--project', 24 | required=True, 25 | type=str, 26 | help='cloud project #') 27 | 28 | group = parser.add_mutually_exclusive_group(required=True) 29 | group.add_argument( 30 | '--delete_all_items', 31 | required=False, 32 | action='store_true', 33 | help='Delete ALL items in catalog') 34 | group.add_argument( 35 | '--items', 36 | required=False, 37 | type=str, 38 | help='item, or list of items to delete. ex: 12345,12346,12347') 39 | 40 | args = parser.parse_args() 41 | 42 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 43 | 44 | service = build('recommendationengine', 45 | 'v1beta1', 46 | discoveryServiceUrl= 47 | 'https://recommendationengine.googleapis.com/' 48 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 49 | 50 | next_page = '' 51 | while next_page is not None: 52 | if args.delete_all_items: 53 | request = service.projects().locations().catalogs().catalogItems().list( 54 | parent='projects/'+ args.project + 55 | '/locations/global/catalogs/default_catalog', 56 | pageSize=1000, pageToken=next_page) 57 | response = request.execute() 58 | 59 | if 'nextPageToken' in response: 60 | next_page = response['nextPageToken'] 61 | else: 62 | next_page = None 63 | else: # Create our own 'response' with the item ids 64 | next_page = None 65 | response = { 66 | 'catalogItems': [] 67 | } 68 | for item in args.items.split(','): 69 | i = {'id': item} 70 | response['catalogItems'].append(i) 71 | 72 | for item in response['catalogItems']: 73 | print('Deleting item id ' + item['id']) 74 | request = service.projects().locations().catalogs().catalogItems().delete( 75 | name='projects/' + args.project + 76 | '/locations/global/catalogs/default_catalog/catalogItems/' + item['id']) 77 | try: 78 | response = request.execute() 79 | except HttpError as err: 80 | print(err) 81 | -------------------------------------------------------------------------------- /beta/python/gcs_import_catalog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Import catalog from a GCS bucket 3 | https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/projects.locations.catalogs.catalogItems/import 4 | 5 | You will need to customize the json_body below for your GCS bucket paths 6 | 7 | """ 8 | 9 | import argparse 10 | import json 11 | import os 12 | import sys 13 | from apiclient.discovery import build 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument( 17 | '--service_account', 18 | required=True, 19 | type=str, 20 | help='path to service account .json key file') 21 | parser.add_argument( 22 | '--apikey', 23 | required=True, 24 | type=str, 25 | help='API key') 26 | parser.add_argument( 27 | '--project', 28 | required=True, 29 | type=str, 30 | help='Cloud project #') 31 | 32 | args = parser.parse_args() 33 | 34 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 35 | 36 | service = build('recommendationengine', 37 | 'v1beta1', 38 | discoveryServiceUrl= 39 | 'https://recommendationengine.googleapis.com/' 40 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 41 | 42 | # Customize to your environment 43 | #https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/InputConfig#GcsSource 44 | json_body = { 45 | "requestId": 'import12345', # Optional 46 | "inputConfig": { 47 | "gcsSource": { 48 | # Note you can use * for all files in bucket, or pass a list of bucket/files 49 | "inputUris": ['gs://mybucket/*'] 50 | }, 51 | }, 52 | "errorsConfig": { 53 | "gcsPrefix": 'gs://mybucket/errors' 54 | } 55 | } 56 | 57 | import_request = service.projects().locations().catalogs().eventStores()\ 58 | .catalogItems().import_( 59 | parent='projects/'+ args.project + 60 | '/locations/global/catalogs/default_catalog'+ 61 | '/eventStores/default_event_store', 62 | body=json_body 63 | ) 64 | 65 | response = import_request.execute() 66 | print(response) 67 | 68 | -------------------------------------------------------------------------------- /beta/python/gcs_import_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Import events from a GCS bucket 3 | https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/projects.locations.catalogs.eventStores.userEvents/import 4 | 5 | You will need to customize the json_body below for your GCS bucket paths 6 | 7 | """ 8 | 9 | import argparse 10 | import json 11 | import os 12 | import sys 13 | from apiclient.discovery import build 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument( 17 | '--service_account', 18 | required=True, 19 | type=str, 20 | help='path to service account .json key file') 21 | parser.add_argument( 22 | '--apikey', 23 | required=True, 24 | type=str, 25 | help='API key') 26 | parser.add_argument( 27 | '--project', 28 | required=True, 29 | type=str, 30 | help='Cloud project #') 31 | 32 | args = parser.parse_args() 33 | 34 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 35 | 36 | service = build('recommendationengine', 37 | 'v1beta1', 38 | discoveryServiceUrl= 39 | 'https://recommendationengine.googleapis.com/' 40 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 41 | 42 | # Customize to your environment 43 | #https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/InputConfig#GcsSource 44 | json_body = { 45 | "requestId": 'import12345', # Optional 46 | "inputConfig": { 47 | "gcsSource": { 48 | # Note you can use * for all files in bucket, or pass a list of bucket/files 49 | "inputUris": ['gs://mybucket/*'] 50 | }, 51 | }, 52 | "errorsConfig": { 53 | "gcsPrefix": 'gs://mybucket/errors' 54 | } 55 | } 56 | 57 | import_request = service.projects().locations().catalogs().eventStores()\ 58 | .userEvents().import_( 59 | parent='projects/'+ args.project + 60 | '/locations/global/catalogs/default_catalog'+ 61 | '/eventStores/default_event_store', 62 | body=json_body 63 | ) 64 | 65 | response = import_request.execute() 66 | print(response) 67 | 68 | -------------------------------------------------------------------------------- /beta/python/import_catalog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Import catalog from file to Recommendations AI. 3 | 4 | Currently supports Merchant Center file format (TSV, tab-delimited only): 5 | https://support.google.com/merchants/answer/160567?hl=en 6 | Uses only the following fields from the MC TSV file: 7 | id, link, title, description, product_type, availability, price 8 | """ 9 | 10 | import argparse 11 | import csv 12 | import os 13 | import re 14 | import sys 15 | from apiclient.discovery import build 16 | 17 | LANGUAGE_CODE = 'en' 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument( 21 | '--service_account', 22 | required=True, 23 | type=str, 24 | help='path to service account .json key file') 25 | parser.add_argument( 26 | '--apikey', 27 | required=True, 28 | type=str, 29 | help='API key') 30 | parser.add_argument( 31 | '--project', 32 | required=True, 33 | type=str, 34 | help='Cloud project #') 35 | parser.add_argument( 36 | '--infile', 37 | required=False, 38 | type=str, 39 | help='Input File, use stdinn if omitted') 40 | 41 | args = parser.parse_args() 42 | 43 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 44 | 45 | service = build('recommendationengine', 46 | 'v1beta1', 47 | discoveryServiceUrl= 48 | 'https://recommendationengine.googleapis.com/' 49 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 50 | 51 | 52 | def main(): 53 | if args.infile: 54 | infile = open(args.infile, 'r') 55 | else: 56 | infile = sys.stdin 57 | 58 | count = 0 59 | reader = csv.DictReader(infile, dialect='excel-tab') 60 | for row in reader: 61 | if count == 0: 62 | import_body = { 63 | 'inputConfig': { 64 | 'catalogInlineSource': { 65 | 'catalogItems': [] 66 | } 67 | } 68 | } 69 | 70 | item = { 71 | 'id': row['id'], 72 | 'languageCode': LANGUAGE_CODE, 73 | 'title': row['title'], 74 | 'description': row['description'], 75 | 'categoryHierarchies': { 76 | # May need to tweak this based on your categories: 77 | 'categories': re.split(' > |,', row['product_type']) 78 | }, 79 | 'productMetadata': { 80 | 'currencyCode': 'USD', 81 | 'canonicalProductUri': row['link'], 82 | 'exactPrice': { 83 | 'originalPrice': row['price'] 84 | }, 85 | 'images': [] 86 | }, 87 | 'itemAttributes': { 88 | 'categoricalFeatures': { 89 | 90 | } 91 | }, 92 | 'tags': [] 93 | } 94 | 95 | if row['availability'] == 'in stock': 96 | item['productMetadata']['stockState'] = 'IN_STOCK' 97 | else: 98 | item['productMetadata']['stockState'] = 'OUT_OF_STOCK' 99 | 100 | # Optional Fields 101 | if row['weight']: 102 | item['itemAttributes']['categoricalFeatures']['weight'] = { 103 | 'value': [row['weight']] 104 | } 105 | if row['brand']: 106 | item['itemAttributes']['categoricalFeatures']['brand'] = { 107 | 'value': [row['brand']] 108 | } 109 | if row['condition']: 110 | item['itemAttributes']['categoricalFeatures']['condition'] = { 111 | 'value': [row['condition']] 112 | } 113 | 114 | (import_body['inputConfig']['catalogInlineSource']['catalogItems'] 115 | .append(item)) 116 | 117 | count = count + 1 118 | 119 | if count == 5000: 120 | import_catalog(import_body) 121 | count = 0 122 | import_body = {} 123 | 124 | import_catalog(import_body) 125 | 126 | 127 | def import_catalog(json_body): 128 | """Upload up to 5000 events using catalogItems.import().""" 129 | 130 | event_count = \ 131 | len(json_body['inputConfig']['catalogInlineSource']['catalogItems']) 132 | 133 | print('Importing ' + str(event_count)) 134 | 135 | import_request = service.projects().locations().catalogs().catalogItems()\ 136 | .import_( 137 | parent='projects/{}/locations/global/catalogs/default_catalog' 138 | .format(args.project), 139 | body=json_body 140 | ) 141 | 142 | response = import_request.execute() 143 | if 'done' not in response: 144 | print(response) 145 | 146 | 147 | if __name__ == '__main__': 148 | main() 149 | -------------------------------------------------------------------------------- /beta/python/import_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Read one JSON struct per line from file and import to Recommendations AI. 3 | 4 | This shouldn't be used for very large imports, use gcsSource instead: 5 | https://cloud.google.com/recommendations-ai/docs/reference/rest/v1beta1/InputConfig#GcsSource 6 | """ 7 | 8 | import argparse 9 | import json 10 | import os 11 | import sys 12 | from apiclient.discovery import build 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | '--service_account', 17 | required=True, 18 | type=str, 19 | help='path to service account .json key file') 20 | parser.add_argument( 21 | '--apikey', 22 | required=True, 23 | type=str, 24 | help='API key') 25 | parser.add_argument( 26 | '--project', 27 | required=True, 28 | type=str, 29 | help='Cloud project #') 30 | parser.add_argument( 31 | '--infile', 32 | required=False, 33 | type=str, 34 | help='Input File, use stdinn if omitted') 35 | 36 | args = parser.parse_args() 37 | 38 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 39 | 40 | service = build('recommendationengine', 41 | 'v1beta1', 42 | discoveryServiceUrl= 43 | 'https://recommendationengine.googleapis.com/' 44 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 45 | 46 | 47 | def main(): 48 | count = 0 49 | if args.infile: 50 | infile = open(args.infile, 'r') 51 | else: 52 | infile = sys.stdin 53 | 54 | for line in infile: 55 | data = json.loads(line) 56 | count = count + 1 57 | 58 | if count == 1: 59 | import_body = {} 60 | import_body['inputConfig'] = {} 61 | import_body['inputConfig']['userEventInlineSource'] = {} 62 | import_body['inputConfig']['userEventInlineSource']['userEvents'] = [] 63 | 64 | (import_body['inputConfig']['userEventInlineSource']['userEvents'] 65 | .append(data)) 66 | 67 | if count == 5000: 68 | import_event(import_body) 69 | count = 0 70 | 71 | if count > 0: 72 | import_event(import_body) 73 | 74 | 75 | def import_event(json_body): 76 | """Upload up to 5000 events using userEvents.import().""" 77 | 78 | event_count = \ 79 | len(json_body['inputConfig']['userEventInlineSource']['userEvents']) 80 | 81 | print('Importing ' + str(event_count)) 82 | 83 | import_request = service.projects().locations().catalogs().eventStores()\ 84 | .userEvents().import_( 85 | parent='projects/'+ args.project + 86 | '/locations/global/catalogs/default_catalog'+ 87 | '/eventStores/default_event_store', 88 | body=json_body 89 | ) 90 | 91 | response = import_request.execute() 92 | if 'done' not in response: 93 | print(response) 94 | 95 | if __name__ == '__main__': 96 | main() 97 | -------------------------------------------------------------------------------- /beta/python/list_catalog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Script to list items in Recommendations AI catalog. 3 | 4 | """ 5 | 6 | import argparse 7 | import os 8 | from apiclient.discovery import build 9 | 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument( 12 | '--service_account', 13 | required=True, 14 | type=str, 15 | help='path to service account .json key file') 16 | parser.add_argument( 17 | '--apikey', 18 | required=True, 19 | type=str, 20 | help='API key') 21 | parser.add_argument( 22 | '--project', 23 | required=True, 24 | type=str, 25 | help='cloud project #') 26 | parser.add_argument( 27 | '--simple', 28 | required=False, 29 | action='store_true', 30 | help='Display itemId & Title Only') 31 | 32 | 33 | args = parser.parse_args() 34 | 35 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 36 | 37 | service = build('recommendationengine', 38 | 'v1beta1', 39 | discoveryServiceUrl= 40 | 'https://recommendationengine.googleapis.com/' 41 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 42 | 43 | next_page = '' 44 | while next_page is not None: 45 | request = service.projects().locations().catalogs().catalogItems().list( 46 | parent='projects/'+ args.project + 47 | '/locations/global/catalogs/default_catalog', 48 | pageSize=1000, pageToken=next_page) 49 | response = request.execute() 50 | 51 | if 'nextPageToken' in response: 52 | next_page = response['nextPageToken'] 53 | else: 54 | next_page = None 55 | 56 | for item in response['catalogItems']: 57 | if args.simple: 58 | print('{:s}:{:s}'.format(item['id'], item['title'])) 59 | else: 60 | print(item) 61 | -------------------------------------------------------------------------------- /beta/python/list_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """List all Recommendations AI User Events for a given time period. 3 | 4 | Returns one json response per line 5 | This is useful for importing into BigQuery or other offline analysis: 6 | https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json 7 | 8 | Can also import directly to BigQuery (create a table with all-events-schema.json) 9 | Using GCS import is usualy faster than inline, 10 | which only streams up to 5000 events per call 11 | """ 12 | 13 | import argparse 14 | import json 15 | from apiclient.discovery import build 16 | from google.oauth2 import service_account 17 | from google.cloud import bigquery 18 | from google.cloud import storage 19 | 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument( 22 | '--service_account', 23 | required=True, 24 | type=str, 25 | help='path to service account .json key file') 26 | parser.add_argument( 27 | '--apikey', 28 | required=True, 29 | type=str, 30 | help='API key') 31 | parser.add_argument( 32 | '--project', 33 | required=True, 34 | type=str, 35 | help='cloud project #') 36 | parser.add_argument( 37 | '--start_date', 38 | required=False, 39 | type=str, 40 | help='Start Date: YYYY-MM-DD') 41 | parser.add_argument( 42 | '--end_date', 43 | required=False, 44 | type=str, 45 | help='End Date: YYYY-MM-DD') 46 | parser.add_argument( 47 | '--event_type', 48 | required=False, 49 | type=str, 50 | help='detail-page-view, add-to-cart, purchase-complete, etc') 51 | parser.add_argument( 52 | '--events_missing_catalog_items', 53 | required=False, 54 | action='store_true', 55 | help='Return only unjoined events') 56 | parser.add_argument( 57 | '--bq_table', 58 | required=False, 59 | type=str, 60 | help='Stream events to a BigQuery table') 61 | parser.add_argument( 62 | '--gcs_bucket', 63 | required=False, 64 | type=str, 65 | help='temp bucket for writing event files') 66 | parser.add_argument( 67 | '-v', '--verbose', 68 | required=False, 69 | action='store_true', 70 | help='Verbose output') 71 | 72 | args = parser.parse_args() 73 | 74 | filter_string = '' 75 | if args.start_date is not None: 76 | filter_string = (' eventTime > "' + args.start_date + 'T00:00:00.00Z" ') 77 | if args.end_date is not None: 78 | filter_string = (filter_string + ' eventTime < "' + 79 | args.end_date + 'T23:59:59.99Z" ') 80 | if args.event_type is not None: 81 | filter_string = (filter_string + ' eventType = '+ args.event_type) 82 | if args.events_missing_catalog_items: 83 | filter_string = (filter_string + ' eventsMissingCatalogItems') 84 | 85 | SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 86 | SERVICE_ACCOUNT_FILE = args.service_account 87 | credentials = service_account.Credentials.from_service_account_file( 88 | SERVICE_ACCOUNT_FILE, scopes=SCOPES) 89 | 90 | # Can also set GOOGLE_APPLICATION_CREDENTIALS environment variable, 91 | # But the above creditals code may be somewhat more "correct" 92 | # os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 93 | 94 | if args.bq_table: 95 | bq = bigquery.Client(credentials=credentials) 96 | 97 | if args.gcs_bucket: 98 | gcs = storage.Client(credentials=credentials) 99 | gcs_data = '' 100 | file_num = 1; 101 | 102 | service = build('recommendationengine', 103 | 'v1beta1', 104 | discoveryServiceUrl= 105 | 'https://recommendationengine.googleapis.com/' 106 | '$discovery/rest?version=v1beta1&key=' + args.apikey, 107 | credentials=credentials 108 | ) 109 | 110 | next_page = '' 111 | while next_page is not None: 112 | 113 | if args.verbose: 114 | print('Getting Recommendations Events') 115 | request = service.projects().locations().catalogs().eventStores()\ 116 | .userEvents().list( 117 | parent='projects/'+ args.project + 118 | '/locations/global/catalogs/default_catalog'+ 119 | '/eventStores/default_event_store', 120 | filter=filter_string, 121 | pageSize=5000, pageToken=next_page) 122 | 123 | response = request.execute() 124 | 125 | if 'nextPageToken' in response: 126 | next_page = response['nextPageToken'] 127 | else: 128 | next_page = None 129 | 130 | try: 131 | bq_list = [] 132 | for event in response['userEvents']: 133 | try: # Add currencyCode since it's not returned currently (b/130748472) 134 | for i in range(len(event['productEventDetail']['productDetails'])): 135 | event['productEventDetail']['productDetails'][i]['currencyCode'] = ( 136 | 'USD') 137 | except KeyError as err: 138 | pass 139 | 140 | if args.bq_table: 141 | # Remove some attributes when importing into BQ 142 | 143 | if 'eventDetail' in event: 144 | if 'eventAttributes' in event['eventDetail']: 145 | if 'categoricalFeatures' in event['eventDetail']['eventAttributes']: 146 | if 'ecommerce.actionField.affiliation' in event['eventDetail']['eventAttributes']['categoricalFeatures']: 147 | del(event['eventDetail']['eventAttributes']['categoricalFeatures']['ecommerce.actionField.affiliation']) 148 | 149 | if 'productEventDetail' in event: 150 | if 'productDetails' in event['productEventDetail']: 151 | for item in event['productEventDetail']['productDetails']: 152 | if 'itemAttributes' in item: 153 | del(item['itemAttributes']) 154 | if 'purchaseTransaction' in event['productEventDetail']: 155 | if 'costs' in event['productEventDetail']['purchaseTransaction']: 156 | del(event['productEventDetail']['purchaseTransaction']['costs']) 157 | if 'taxes' in event['productEventDetail']['purchaseTransaction']: 158 | del(event['productEventDetail']['purchaseTransaction']['taxes']) 159 | 160 | 161 | if args.gcs_bucket: 162 | gcs_data += json.dumps(event) + "\n" 163 | else: 164 | bq_list.append(event) 165 | 166 | else: 167 | print(json.dumps(event)) 168 | 169 | # Upload to BQ 170 | if args.bq_table: 171 | # Use GCS (faster) 172 | if args.gcs_bucket: 173 | if (len(gcs_data) > 500000000 or next_page is None): 174 | if args.verbose: 175 | print('Writing to GCS') 176 | print('Data Size: ' + str(len(gcs_data))) 177 | bucket = gcs.bucket(args.gcs_bucket) 178 | blob = bucket.blob('events-' + str(file_num) + '.txt') 179 | gcs_file = blob.upload_from_string(gcs_data) 180 | 181 | if args.verbose: 182 | print ('Doing BigQuery Import') 183 | job_config = bigquery.LoadJobConfig( 184 | source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, 185 | ) 186 | uri = 'gs://' + args.gcs_bucket + '/' + 'events-' + str(file_num) + '.txt' 187 | 188 | load_job = bq.load_table_from_uri( 189 | uri, 190 | args.bq_table, 191 | location="US", 192 | job_config=job_config 193 | ) 194 | 195 | gcs_data = '' 196 | file_num += 1 197 | 198 | # inline import (slow) 199 | else: 200 | if args.verbose: 201 | print ('Writing to BigQuery') 202 | errors = bq.insert_rows_json(args.bq_table, bq_list) 203 | if errors: 204 | print(json.dumps(errors)) 205 | exit() 206 | else: 207 | if args.verbose: 208 | print('Wrote ' + str(len(bq_list)) + ' events to BiqQuery') 209 | bq_list = [] 210 | 211 | except KeyError as err: 212 | print(err) 213 | pass 214 | -------------------------------------------------------------------------------- /beta/python/list_keys.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Script to list registered API keys 3 | 4 | """ 5 | 6 | import argparse 7 | import os 8 | from apiclient.discovery import build 9 | 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument( 12 | '--service_account', 13 | required=True, 14 | type=str, 15 | help='path to service account .json key file') 16 | parser.add_argument( 17 | '--apikey', 18 | required=True, 19 | type=str, 20 | help='API key to remove') 21 | parser.add_argument( 22 | '--project', 23 | required=True, 24 | type=str, 25 | help='cloud project #') 26 | 27 | args = parser.parse_args() 28 | 29 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 30 | 31 | service = build('recommendationengine', 32 | 'v1beta1', 33 | discoveryServiceUrl= 34 | 'https://recommendationengine.googleapis.com/' 35 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 36 | 37 | request = service.projects().locations().catalogs().eventStores()\ 38 | .predictionApiKeyRegistrations().list( 39 | parent='projects/'+ args.project + 40 | '/locations/global/catalogs/default_catalog'+ 41 | '/eventStores/default_event_store' 42 | ) 43 | 44 | response = request.execute() 45 | for key in response['predictionApiKeyRegistrations']: 46 | print(key) 47 | -------------------------------------------------------------------------------- /beta/python/lro_operations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """list Long Running Operations (LRO) 3 | Usually GCS catalog & event imports or purge operations 4 | 5 | """ 6 | 7 | import argparse 8 | import json 9 | import os 10 | import sys 11 | from apiclient.discovery import build 12 | from pprint import pprint 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | '--service_account', 17 | required=True, 18 | type=str, 19 | help='path to service account .json key file') 20 | parser.add_argument( 21 | '--apikey', 22 | required=True, 23 | type=str, 24 | help='API key') 25 | parser.add_argument( 26 | '--project', 27 | required=True, 28 | type=str, 29 | help='Cloud project #') 30 | parser.add_argument( 31 | '--id', 32 | required=False, 33 | type=str, 34 | help='operationName, to list only a specific operation') 35 | 36 | args = parser.parse_args() 37 | 38 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 39 | 40 | service = build('recommendationengine', 41 | 'v1beta1', 42 | discoveryServiceUrl= 43 | 'https://recommendationengine.googleapis.com/' 44 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 45 | 46 | if args.id: 47 | request = service.projects().locations().catalogs().eventStores()\ 48 | .operations().get( 49 | name='projects/'+ args.project + 50 | '/locations/global/catalogs/default_catalog'+ 51 | '/eventStores/default_event_store/operations/' + args.id 52 | ) 53 | else: 54 | request = service.projects().locations().catalogs().eventStores()\ 55 | .operations().list( 56 | name='projects/'+ args.project + 57 | '/locations/global/catalogs/default_catalog'+ 58 | '/eventStores/default_event_store', 59 | ) 60 | 61 | response = request.execute() 62 | 63 | if args.id: 64 | pprint(response) 65 | else: 66 | for operation in response['operations']: 67 | pprint(operation) 68 | -------------------------------------------------------------------------------- /beta/python/predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Call prediction API, return recommendations. 3 | 4 | There are a lot of options for passing in different events to predict. 5 | This is just a very basic sample that can easily be extended. 6 | """ 7 | 8 | import argparse 9 | import pprint 10 | from apiclient.discovery import build 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument( 14 | '--apikey', 15 | required=True, 16 | type=str, 17 | help='API key') 18 | parser.add_argument( 19 | '--project', 20 | required=True, 21 | type=str, 22 | help='cloud project #') 23 | parser.add_argument( 24 | '--placementid', 25 | required=True, 26 | type=str, 27 | help='placement id, ie: product_detail') 28 | parser.add_argument( 29 | '--items', 30 | required=False, 31 | type=str, 32 | help='item, or list of items for prediction. ex: 12345,12346,12347') 33 | parser.add_argument( 34 | '--visitorid', 35 | type=str, 36 | default='visitor1') 37 | parser.add_argument( 38 | '--userid', 39 | type=str, 40 | default='user1') 41 | 42 | args = parser.parse_args() 43 | 44 | service = build('recommendationengine', 45 | 'v1beta1', 46 | discoveryServiceUrl= 47 | 'https://recommendationengine.googleapis.com/' 48 | '$discovery/rest?version=v1beta1', 49 | developerKey=args.apikey) 50 | 51 | json_body = { 52 | 'pageSize': 5, 53 | 'dryRun': False, 54 | 'userEvent': { 55 | 'userInfo': { 56 | 'visitorId': args.visitorid, 57 | 'userId': args.userid, 58 | 'ipAddress': '0.0.0.0', 59 | 'userAgent': 'Test' 60 | }, 61 | 'eventDetail': { 62 | # 'experimentIds': 'experiment-group' 63 | } 64 | } 65 | } 66 | 67 | 68 | if args.items is not None: 69 | json_body['userEvent']['eventType'] = 'detail-page-view' 70 | json_body['userEvent']['productEventDetail'] = { 71 | 'productDetails': [], 72 | } 73 | for item in args.items.split(','): 74 | (json_body['userEvent']['productEventDetail']['productDetails'] 75 | .append({'id': item})) 76 | else: 77 | json_body['userEvent']['eventType'] = 'home-page-view' 78 | 79 | pprint.pprint(json_body) 80 | 81 | request = service.projects().locations().catalogs().eventStores()\ 82 | .placements().predict( 83 | name='projects/'+ args.project + 84 | '/locations/global/catalogs/default_catalog'+ 85 | '/eventStores/default_event_store/placements/'+args.placementid, 86 | body=json_body) # body takes a python dict, converts to json 87 | 88 | response = request.execute() 89 | pprint.pprint(response) 90 | -------------------------------------------------------------------------------- /beta/python/register_key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Script to register an API key. 3 | Normaly, you should only need to do this once per project. 4 | The registered key should be used only for predict calls 5 | (not exposed publicly, lik the key that may be used for event 6 | ingestion via the collect method) 7 | 8 | """ 9 | 10 | import argparse 11 | import os 12 | from apiclient.discovery import build 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | '--service_account', 17 | required=True, 18 | type=str, 19 | help='path to service account .json key file') 20 | parser.add_argument( 21 | '--apikey', 22 | required=True, 23 | type=str, 24 | help='API key to register') 25 | parser.add_argument( 26 | '--project', 27 | required=True, 28 | type=str, 29 | help='cloud project #') 30 | 31 | args = parser.parse_args() 32 | 33 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 34 | 35 | service = build('recommendationengine', 36 | 'v1beta1', 37 | discoveryServiceUrl= 38 | 'https://recommendationengine.googleapis.com/' 39 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 40 | 41 | json_body = { 42 | "predictionApiKeyRegistration": { 43 | 'apiKey': args.apikey 44 | } 45 | } 46 | 47 | request = service.projects().locations().catalogs().eventStores()\ 48 | .predictionApiKeyRegistrations().create( 49 | parent='projects/'+ args.project + 50 | '/locations/global/catalogs/default_catalog'+ 51 | '/eventStores/default_event_store', body=json_body) 52 | 53 | response = request.execute() 54 | print(response) 55 | -------------------------------------------------------------------------------- /beta/python/timestamp_to_zulu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Take unix timestamp on command line and convert to Zulu time 3 | 4 | """ 5 | 6 | import sys 7 | from datetime import datetime 8 | 9 | d = datetime.utcfromtimestamp(int(sys.argv[1])/1000) 10 | zulutime = d.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 11 | print(zulutime) 12 | -------------------------------------------------------------------------------- /beta/python/unregister_key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Script to unregister an API key. 3 | This does not delete the key, 4 | it just makes it unusable for predict calls 5 | 6 | """ 7 | 8 | import argparse 9 | import os 10 | from apiclient.discovery import build 11 | 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument( 14 | '--service_account', 15 | required=True, 16 | type=str, 17 | help='path to service account .json key file') 18 | parser.add_argument( 19 | '--apikey', 20 | required=True, 21 | type=str, 22 | help='API key to remove') 23 | parser.add_argument( 24 | '--project', 25 | required=True, 26 | type=str, 27 | help='cloud project #') 28 | 29 | args = parser.parse_args() 30 | 31 | os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.service_account 32 | 33 | service = build('recommendationengine', 34 | 'v1beta1', 35 | discoveryServiceUrl= 36 | 'https://recommendationengine.googleapis.com/' 37 | '$discovery/rest?version=v1beta1&key=' + args.apikey) 38 | 39 | request = service.projects().locations().catalogs().eventStores()\ 40 | .predictionApiKeyRegistrations().delete( 41 | name='projects/'+ args.project + 42 | '/locations/global/catalogs/default_catalog'+ 43 | '/eventStores/default_event_store/predictionApiKeyRegistrations/' + 44 | args.apikey 45 | ) 46 | 47 | response = request.execute() 48 | print(response) 49 | -------------------------------------------------------------------------------- /php/README.md: -------------------------------------------------------------------------------- 1 | Current php client libraries for google-cloud-retail are here: 2 | 3 | - [https://github.com/googleapis/google-cloud-php/tree/master/Retail](https://github.com/googleapis/google-cloud-php/tree/master/Retail) 4 | -------------------------------------------------------------------------------- /php/autocomplete/complete.php: -------------------------------------------------------------------------------- 1 | '[YOUR JSON KEY FILE]', // or use application default credentials 21 | ]); 22 | 23 | $formattedCatalog = CompletionServiceClient::catalogName($PROJECT, 'global', 'default_catalog'); 24 | 25 | // Call the API and handle any network failures. 26 | try { 27 | /** @var CompleteQueryResponse $response */ 28 | $response = $completionServiceClient->completeQuery($formattedCatalog, $TERM, ['dataset' => $DATASET]); 29 | $suggestions = $response->getCompletionResults(); 30 | $results = count($suggestions); 31 | 32 | print "["; 33 | for ($i = 0; $i < $results; $i++) { 34 | printf("\"%s\"", $suggestions[$i]->getSuggestion()); 35 | if ($i < $results-1) { 36 | print ","; 37 | } 38 | } 39 | print "]"; 40 | //printf($response->serializeToJsonString()); 41 | } catch (ApiException $ex) { 42 | printf('Call failed with message: %s' . PHP_EOL, $ex->getMessage()); 43 | } 44 | 45 | ?> 46 | -------------------------------------------------------------------------------- /php/autocomplete/searchform.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | jQuery Autocomplete Example 9 | 10 | 11 | 12 | 13 | 24 | 25 | 26 | 27 |
28 | 29 | 30 |
31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /php/autocomplete/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Arial, Helvetica, sans-serif; 3 | } 4 | 5 | table { 6 | font-size: 1em; 7 | } 8 | 9 | .ui-draggable, .ui-droppable { 10 | background-position: top; 11 | } 12 | -------------------------------------------------------------------------------- /php/predict.php: -------------------------------------------------------------------------------- 1 | '[your service account key.json file]' 15 | ]); 16 | 17 | $placement = "projects/[YOUR PROJECT #]/locations/global/catalogs/default_catalog/placements/product_detail"; 18 | 19 | $product_detail = new ProductDetail([ 20 | 'product' => new Product(['id' => '12345']), 21 | 'quantity' => 1 22 | ]); 23 | 24 | $userEvent = new UserEvent([ 25 | 'event_type' => 'detail-page-view', 26 | 'visitor_id' => 'ABCDEFG', 27 | 'product_details' => [ 28 | new ProductDetail([ 29 | 'product' => new Product(['id' => '12345']), 30 | 'quantity' => 1 31 | ]) 32 | ] 33 | ]); 34 | 35 | $pb_true = new Google\Protobuf\Value(); 36 | $pb_true->setBoolValue(true); 37 | 38 | try { 39 | $predictions = $predictionClient->predict($placement, $userEvent, [ 40 | 'params' => ['returnProduct' => $pb_true], 41 | 'filter' => 'filterOutOfStockItems' 42 | ] 43 | ); 44 | } finally { 45 | $predictionClient->close(); 46 | } 47 | 48 | $results = $predictions->getResults(); 49 | 50 | print($results->count() . " Item Id's Returned:"); 51 | 52 | $iterator = $results->getIterator(); 53 | while($iterator->valid()) { 54 | print($iterator->current()->getId() . "\n"); 55 | 56 | // With returnProduct=true we can get all the product data 57 | print_r(json_decode($iterator->current()->serializeToJsonString())); 58 | 59 | $iterator->next(); 60 | } 61 | 62 | ?> 63 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | ## Recommendations AI Python Examples using Retail API 2 | 3 | [Python Client Library](https://pypi.org/project/google-cloud-retail/) 4 | 5 | #### All samples provided as-is without warranty 6 | -------------------------------------------------------------------------------- /python/import_catalog_inline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ Inline import catalog items using Retail API 3 | 4 | """ 5 | 6 | from google.cloud import retail 7 | from google.oauth2 import service_account 8 | 9 | SERVICE_ACCOUNT_FILE = "" 10 | PROJECT_NUM = "" 11 | 12 | SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 13 | credentials = service_account.Credentials.from_service_account_file( 14 | SERVICE_ACCOUNT_FILE, scopes=SCOPES) 15 | 16 | client = retail.ProductServiceClient(credentials=credentials) 17 | 18 | products = { 19 | "products": [ 20 | { 21 | "id": "123", 22 | "title": "Steamed Hams", 23 | "description": "Bulk Lot of Steamed Hams", 24 | "categories": ["Food","Steamed & Canned"], 25 | "price_info": { 26 | "price": 149.95, 27 | "currency_code": "USD" 28 | } 29 | }, 30 | { 31 | "id": "456", 32 | "title": "Canned Yams", 33 | "description": "Bulk Lot of Canned Yams", 34 | "categories": ["Food","Steamed & Canned"], 35 | "price_info": { 36 | "price": 89.95, 37 | "currency_code": "USD" 38 | } 39 | } 40 | ] 41 | } 42 | 43 | request = { 44 | "parent": 'projects/' + PROJECT_NUM + '/locations/global/catalogs/default_catalog/branches/0', 45 | "input_config": {"product_inline_source": products} 46 | } 47 | 48 | response = client.import_products(request) 49 | 50 | -------------------------------------------------------------------------------- /python/import_event_inline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | """ Inline import user events using Retail API 4 | 5 | This example shows how to construct a timestamp (using current time) 6 | 7 | """ 8 | 9 | import datetime 10 | 11 | from google.cloud import retail 12 | from google.oauth2 import service_account 13 | from google.protobuf.timestamp_pb2 import Timestamp 14 | from google.protobuf.wrappers_pb2 import Int32Value 15 | 16 | SERVICE_ACCOUNT_FILE = "" 17 | PROJECT_NUM = "" 18 | 19 | SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 20 | credentials = service_account.Credentials.from_service_account_file( 21 | SERVICE_ACCOUNT_FILE, scopes=SCOPES) 22 | 23 | client = retail.UserEventServiceClient(credentials=credentials) 24 | 25 | timestamp = Timestamp() 26 | timestamp.FromDatetime(dt=datetime.datetime.now()) 27 | 28 | quantity = Int32Value(value=1) 29 | 30 | user_events = { 31 | "user_events": [ 32 | { 33 | "event_type": "home-page-view", 34 | "event_time": timestamp, 35 | "visitor_id": "visitor-1", 36 | "user_info": { 37 | "user_id": "user12345" 38 | } 39 | }, 40 | { 41 | "event_type": "detail-page-view", 42 | "event_time": timestamp, 43 | "visitor_id": "visitor-1", 44 | "user_info": { 45 | "user_id": "user12345" 46 | }, 47 | "product_details": [{ 48 | "product": { 49 | "id": "12345" 50 | } 51 | }] 52 | }, 53 | { 54 | "event_type": "add-to-cart", 55 | "event_time": timestamp, 56 | "visitor_id": "visitor-1", 57 | "user_info": { 58 | "user_id": "user12345" 59 | }, 60 | "product_details": [{ 61 | "product": { 62 | "id": "12345" 63 | }, 64 | "quantity": quantity 65 | }] 66 | }, 67 | { 68 | "event_type": "purchase-complete", 69 | "event_time": timestamp, 70 | "visitor_id": "visitor-1", 71 | "user_info": { 72 | "user_id": "user12345" 73 | }, 74 | "product_details": [{ 75 | "product": { 76 | "id": "12345" 77 | }, 78 | "quantity": quantity 79 | }], 80 | "purchase_transaction": { 81 | "id": "567", 82 | "revenue": 19.95, 83 | "currency_code": "USD" 84 | } 85 | } 86 | ] 87 | } 88 | 89 | request = { 90 | "parent": 91 | 'projects/' + PROJECT_NUM + '/locations/global/catalogs/default_catalog', 92 | "input_config": {"user_event_inline_source": user_events} 93 | } 94 | 95 | response = client.import_user_events(request) 96 | -------------------------------------------------------------------------------- /python/predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | """ Sample predict call showing how to pass event 4 | 5 | Customize with placement, event type as needed 6 | 7 | """ 8 | 9 | import datetime 10 | 11 | from google.cloud import retail 12 | from google.oauth2 import service_account 13 | from google.protobuf.timestamp_pb2 import Timestamp 14 | 15 | SERVICE_ACCOUNT_FILE = "" 16 | PROJECT_NUM = "" 17 | 18 | SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 19 | credentials = service_account.Credentials.from_service_account_file( 20 | SERVICE_ACCOUNT_FILE, scopes=SCOPES) 21 | 22 | client = retail.PredictionServiceClient(credentials=credentials) 23 | 24 | timestamp = Timestamp() 25 | timestamp.FromDatetime(dt=datetime.datetime.now()) 26 | 27 | user_event = { 28 | "event_type": "detail-page-view", 29 | "event_time": timestamp, 30 | "visitor_id": "visitor-1", 31 | "user_info": { 32 | "user_id": "user12345" 33 | }, 34 | "product_details": [{ 35 | "product": { 36 | "id": "12345" 37 | } 38 | }] 39 | } 40 | 41 | request = { 42 | "placement": 43 | 'projects/' + PROJECT_NUM + 44 | '/locations/global/catalogs/default_catalog/placements/home_page', 45 | 46 | "user_event": user_event, 47 | 48 | "filter": "filterOutOfStockItems", 49 | 50 | "params": { 51 | "returnProduct": True, 52 | "returnScore": True 53 | } 54 | } 55 | 56 | response = client.predict(request) 57 | 58 | print(response) 59 | -------------------------------------------------------------------------------- /python/search_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Google Retail API performance testing script. 5 | Measures search performance by executing queries and reporting timing metrics. 6 | """ 7 | 8 | import sys 9 | import time 10 | from pathlib import Path 11 | from typing import List 12 | 13 | from google.cloud.retail_v2 import SearchRequest, SearchServiceClient 14 | from google.api_core import exceptions 15 | from google.oauth2 import service_account 16 | 17 | 18 | # --- Configuration --- 19 | PROJECT_ID = "" 20 | KEY_FILE = "path to keyfile.json" 21 | SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] 22 | 23 | LOCATION = "global" 24 | CATALOG = "default_catalog" 25 | PLACEMENT = "default_search" 26 | QUERY_FILE = "queries.txt" 27 | VISITOR_ID = "12345" 28 | PAGE_SIZE = 100 29 | FILTER = 'availability: ANY("IN_STOCK")' 30 | QUERY_EXPANSION = {'condition':'AUTO'} 31 | 32 | def validate_configuration(): 33 | """Validate that required configuration is set.""" 34 | if not PROJECT_ID: 35 | print("Error: PROJECT_ID is not set.") 36 | return False 37 | 38 | if not Path(KEY_FILE).exists(): 39 | print(f"Error: Service account key file not found at '{KEY_FILE}'") 40 | return False 41 | 42 | if not Path(QUERY_FILE).exists(): 43 | print(f"Error: Query file not found at '{QUERY_FILE}'") 44 | return False 45 | 46 | return True 47 | 48 | 49 | def create_credentials(): 50 | """Create credentials from service account key file.""" 51 | try: 52 | credentials = service_account.Credentials.from_service_account_file( 53 | KEY_FILE, 54 | scopes=SCOPES 55 | ) 56 | return credentials 57 | except Exception as e: 58 | print(f"Error loading credentials: {e}") 59 | return None 60 | 61 | 62 | def read_queries(query_file: str) -> List[str]: 63 | """Read search queries from file, one per line.""" 64 | queries = [] 65 | with open(query_file, 'r', encoding='utf-8') as f: 66 | for line in f: 67 | query = line.strip() 68 | if query: 69 | queries.append(query) 70 | return queries 71 | 72 | 73 | def perform_search(client: SearchServiceClient, placement_path: str, query: str) -> tuple: 74 | """Perform a single search request and return (result_count, duration_ms, success).""" 75 | start_time = time.time() 76 | 77 | try: 78 | request = SearchRequest( 79 | placement=placement_path, 80 | visitor_id=VISITOR_ID, 81 | query=query, 82 | page_size=PAGE_SIZE, 83 | query_expansion_spec=QUERY_EXPANSION, 84 | filter=FILTER 85 | ) 86 | 87 | response_pager = client.search(request=request) 88 | first_page = next(iter(response_pager.pages)) 89 | result_count = len(first_page.results) 90 | 91 | end_time = time.time() 92 | duration_ms = (end_time - start_time) * 1000 93 | 94 | return result_count, duration_ms, True 95 | 96 | except exceptions.GoogleAPIError as e: 97 | end_time = time.time() 98 | duration_ms = (end_time - start_time) * 1000 99 | print(f" ✗ API Error: {e}") 100 | return 0, duration_ms, False 101 | except Exception as e: 102 | end_time = time.time() 103 | duration_ms = (end_time - start_time) * 1000 104 | print(f" ✗ Error: {e}") 105 | return 0, duration_ms, False 106 | 107 | 108 | def main(): 109 | """Main function to execute the performance test.""" 110 | print("Google Retail API Performance Test") 111 | print("=" * 50) 112 | 113 | # Validate configuration 114 | if not validate_configuration(): 115 | sys.exit(1) 116 | 117 | # Read queries 118 | try: 119 | queries = read_queries(QUERY_FILE) 120 | except Exception as e: 121 | print(f"Error reading query file: {e}") 122 | sys.exit(1) 123 | 124 | if not queries: 125 | print(f"No queries found in '{QUERY_FILE}'. Exiting.") 126 | sys.exit(0) 127 | 128 | print(f"Found {len(queries)} queries to test") 129 | 130 | # Initialize client 131 | try: 132 | credentials = create_credentials() 133 | if not credentials: 134 | sys.exit(1) 135 | 136 | client = SearchServiceClient(credentials=credentials) 137 | print("✓ Client initialized successfully") 138 | except Exception as e: 139 | print(f"Error initializing client: {e}") 140 | sys.exit(1) 141 | 142 | # Build placement path 143 | placement_path = f"projects/{PROJECT_ID}/locations/{LOCATION}/catalogs/{CATALOG}/placements/{PLACEMENT}" 144 | print(f"Using placement: {placement_path}") 145 | print("-" * 50) 146 | 147 | # Execute searches and collect metrics 148 | total_start_time = time.time() 149 | successful_searches = 0 150 | failed_searches = 0 151 | total_results = 0 152 | durations = [] 153 | 154 | for i, query in enumerate(queries, 1): 155 | print(f"[{i:3d}/{len(queries)}] Testing: '{query[:50]}{'...' if len(query) > 50 else ''}'") 156 | 157 | result_count, duration_ms, success = perform_search(client, placement_path, query) 158 | durations.append(duration_ms) 159 | 160 | if success: 161 | successful_searches += 1 162 | total_results += result_count 163 | print(f" ✓ {result_count} results in {duration_ms:.1f}ms") 164 | else: 165 | failed_searches += 1 166 | print(f" ✗ Failed in {duration_ms:.1f}ms") 167 | 168 | total_end_time = time.time() 169 | total_duration_seconds = total_end_time - total_start_time 170 | 171 | # Print performance summary 172 | print("\n" + "=" * 50) 173 | print("PERFORMANCE SUMMARY") 174 | print("=" * 50) 175 | print(f"Total queries processed: {len(queries)}") 176 | print(f"Successful searches: {successful_searches}") 177 | print(f"Failed searches: {failed_searches}") 178 | print(f"Total results found: {total_results}") 179 | print(f"Average results per query: {total_results / successful_searches if successful_searches > 0 else 0:.1f}") 180 | print() 181 | print(f"Total execution time: {total_duration_seconds:.2f} seconds") 182 | print(f"Average time per request: {sum(durations) / len(durations):.1f} ms") 183 | print(f"Fastest request: {min(durations):.1f} ms") 184 | print(f"Slowest request: {max(durations):.1f} ms") 185 | print(f"Requests per second: {len(queries) / total_duration_seconds:.2f}") 186 | 187 | if successful_searches > 0: 188 | successful_durations = [durations[i] for i, query in enumerate(queries) 189 | if perform_search(client, placement_path, query)[2]] 190 | print(f"Avg time (successful only): {sum(d for d in durations if d > 0) / successful_searches:.1f} ms") 191 | 192 | 193 | if __name__ == "__main__": 194 | main() 195 | 196 | --------------------------------------------------------------------------------