recordKinds = new HashSet<>();
31 |
32 | @Getter @Setter
33 | private boolean recordAll;
34 |
35 | @Inject
36 | public Recorder(BucketFactory bucketFactory, Collector collector, Codepointer codepointer) {
37 | this.bucketFactory = bucketFactory;
38 | this.collector = collector;
39 | this.codepointer = codepointer;
40 | }
41 |
42 | /**
43 | * Add a kind to the list of kinds that get recorded. Unless recordAll is set,
44 | * only these kinds will be recorded.
45 | *
46 | * This method is not thread-safe; register all kinds at application startup,
47 | * before you begin using the InsightAsyncDatastoreService.
48 | */
49 | public void recordKind(String kind) {
50 | recordKinds.add(kind);
51 | }
52 |
53 | /** */
54 | private boolean shouldRecord(String kind) {
55 | return recordAll || recordKinds.contains(kind);
56 | }
57 |
58 | /** Create a new batch for recording */
59 | public Batch batch() {
60 | return new Batch();
61 | }
62 |
63 | /** Create a new batch for recording query data */
64 | public QueryBatch query(Query query) {
65 | return new QueryBatch(query);
66 | }
67 |
68 | /**
69 | * A session of recording associated with a particular code point.
70 | */
71 | public class Batch {
72 |
73 | protected final String codePoint;
74 |
75 | Batch() {
76 | codePoint = codepointer.getCodepoint();
77 | }
78 |
79 | /**
80 | */
81 | public void get(Key key) {
82 | if (shouldRecord(key.getKind()))
83 | collector.collect(bucketFactory.forGet(codePoint, key.getNamespace(), key.getKind(), 1));
84 | }
85 |
86 | /**
87 | */
88 | public void put(Entity entity) {
89 | if (shouldRecord(entity.getKind()))
90 | collector.collect(bucketFactory.forPut(codePoint, entity.getNamespace(), entity.getKind(), !entity.getKey().isComplete(), 1));
91 | }
92 |
93 | /**
94 | */
95 | public void delete(Key key) {
96 | if (shouldRecord(key.getKind()))
97 | collector.collect(bucketFactory.forDelete(codePoint, key.getNamespace(), key.getKind(), 1));
98 | }
99 | }
100 |
101 | public class QueryBatch extends Batch {
102 | protected final String query;
103 |
104 | /** */
105 | QueryBatch(Query q) {
106 | query = q.toString();
107 | }
108 |
109 | /**
110 | */
111 | public void query(Entity entity) {
112 | if (shouldRecord(entity.getKind()))
113 | collector.collect(bucketFactory.forQuery(codePoint, entity.getNamespace(), entity.getKind(), query, 1));
114 | }
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/puller/BigUploader.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | import com.google.api.services.bigquery.Bigquery;
4 | import com.google.api.services.bigquery.model.TableDataInsertAllRequest;
5 | import com.google.api.services.bigquery.model.TableDataInsertAllRequest.Rows;
6 | import com.google.api.services.bigquery.model.TableDataInsertAllResponse;
7 | import com.google.api.services.bigquery.model.TableRow;
8 | import com.google.common.collect.Lists;
9 | import com.googlecode.objectify.insight.Bucket;
10 | import lombok.extern.java.Log;
11 | import javax.inject.Inject;
12 | import java.io.IOException;
13 | import java.util.ArrayList;
14 | import java.util.Collection;
15 | import java.util.List;
16 |
17 | /**
18 | * Does the work of leasing tasks from the task queue, aggregating again, and pushing
19 | * the result to BigQuery.
20 | */
21 | @Log
22 | public class BigUploader {
23 |
24 | private final Bigquery bigquery;
25 | private final InsightDataset insightDataset;
26 | private final TablePicker tablePicker;
27 |
28 | @Inject
29 | public BigUploader(Bigquery bigquery, InsightDataset insightDataset, TablePicker tablePicker) {
30 | this.bigquery = bigquery;
31 | this.insightDataset = insightDataset;
32 | this.tablePicker = tablePicker;
33 | }
34 |
35 | public void upload(Collection buckets) {
36 | log.finer("Uploading " + buckets.size() + " buckets to bigquery");
37 |
38 | // Seriously, Google, you f'd up the naming of 'Rows'. The date handling is atrocious.
39 | // And what's with the whole new JSON layer, including annotations? This library is crap.
40 |
41 | List rows = new ArrayList();
42 |
43 | for (Bucket bucket : buckets) {
44 | TableRow row = new TableRow();
45 | row.set("uploaded", System.currentTimeMillis() / 1000f); // unix timestamp
46 | row.set("codepoint", bucket.getKey().getCodepoint());
47 | row.set("namespace", bucket.getKey().getNamespace());
48 | row.set("module", bucket.getKey().getModule());
49 | row.set("version", bucket.getKey().getVersion());
50 | row.set("kind", bucket.getKey().getKind());
51 | row.set("op", bucket.getKey().getOp());
52 | row.set("query", bucket.getKey().getQuery());
53 | row.set("time", bucket.getKey().getTime() / 1000f); // unix timestamp
54 | row.set("reads", bucket.getReads());
55 | row.set("writes", bucket.getWrites());
56 |
57 | TableDataInsertAllRequest.Rows rowWrapper = new TableDataInsertAllRequest.Rows();
58 |
59 | // As much as we would like to do this there isn't really any kind of stable hash because we
60 | // are constantly aggregating. If we really want this, we will have to stop aggregating at the
61 | // task level (the thing that retries).
62 | //rowWrapper.setInsertId(timestamp);
63 |
64 | rowWrapper.setJson(row);
65 |
66 | rows.add(rowWrapper);
67 | }
68 |
69 | // BQ can handle maximum 10000 rows per request
70 | // https://cloud.google.com/bigquery/quotas#streaming_inserts
71 | // The suggested batch size is 500 but I would like to avoid partitioning the rows if it is possible
72 | // so I don't have to worry about buckets that are partially written to BQ
73 | for (List partition: Lists.partition(rows, 10000)) {
74 | TableDataInsertAllRequest request = new TableDataInsertAllRequest().setRows(partition);
75 | request.setIgnoreUnknownValues(true);
76 |
77 | String tableId = tablePicker.pick();
78 |
79 | try {
80 | TableDataInsertAllResponse response = bigquery
81 | .tabledata()
82 | .insertAll(insightDataset.projectId(), insightDataset.datasetId(), tableId, request)
83 | .execute();
84 |
85 | if (response.getInsertErrors() != null && !response.getInsertErrors().isEmpty()) {
86 | throw new RuntimeException("There were errors! " + response.getInsertErrors());
87 | }
88 | } catch (IOException e) {
89 | throw new RuntimeException(e);
90 | }
91 | }
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/puller/BucketAggregator.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | import com.googlecode.objectify.insight.Bucket;
4 | import com.googlecode.objectify.insight.BucketKey;
5 | import java.util.Collection;
6 | import java.util.HashMap;
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | /**
11 | * Aggregates buckets together.
12 | */
13 | public class BucketAggregator {
14 | /** */
15 | Map bucketMap = new HashMap<>();
16 |
17 | /** */
18 | public void aggregate(List buckets) {
19 | for (Bucket bucket : buckets) {
20 | Bucket alreadyHere = bucketMap.get(bucket.getKey());
21 |
22 | if (alreadyHere == null) {
23 | alreadyHere = new Bucket(bucket.getKey());
24 | bucketMap.put(bucket.getKey(), alreadyHere);
25 | }
26 |
27 | alreadyHere.merge(bucket);
28 | }
29 | }
30 |
31 | /** */
32 | public Collection getBuckets() {
33 | return bucketMap.values();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/puller/InsightDataset.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | /**
4 | * The "address" of the dataset used to store insight data. If you're using guice, you must
5 | * bind this to a real implementation.
6 | */
7 | public interface InsightDataset {
8 | String projectId();
9 | String datasetId();
10 | }
11 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/puller/Puller.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | import com.google.appengine.api.taskqueue.Queue;
4 | import com.googlecode.objectify.insight.BucketList;
5 | import com.googlecode.objectify.insight.util.QueueHelper;
6 | import com.googlecode.objectify.insight.util.TaskHandleHelper;
7 | import lombok.Getter;
8 | import lombok.Setter;
9 | import lombok.extern.java.Log;
10 | import javax.inject.Inject;
11 | import javax.inject.Named;
12 | import javax.inject.Singleton;
13 | import java.util.List;
14 | import java.util.concurrent.TimeUnit;
15 | import java.util.logging.Level;
16 |
17 | /**
18 | * Does the work of leasing tasks from the task queue, aggregating again, and pushing
19 | * the result to BigQuery.
20 | */
21 | @Log
22 | @Singleton
23 | public class Puller {
24 |
25 | /** */
26 | public static final int DEFAULT_BATCH_SIZE = 20;
27 |
28 | /** Something long enough to be safe */
29 | public static final int DEFAULT_LEASE_DURATION_SECONDS = 60 * 10;
30 |
31 | /** */
32 | private final QueueHelper queue;
33 |
34 | /** */
35 | private final BigUploader bigUploader;
36 |
37 | /** Number of tasks to lease and aggregate per write to BQ */
38 | @Getter @Setter
39 | private int batchSize = DEFAULT_BATCH_SIZE;
40 |
41 | /** How long to maintain task leases; short values risk duplicates */
42 | @Getter @Setter
43 | private int leaseDurationSeconds = DEFAULT_LEASE_DURATION_SECONDS;
44 |
45 | /**
46 | */
47 | @Inject
48 | public Puller(@Named("insight") Queue queue, BigUploader bigUploader) {
49 | this.queue = new QueueHelper<>(queue, BucketList.class);
50 | this.bigUploader = bigUploader;
51 | }
52 |
53 | /**
54 | * Repeatedly leases batches of tasks, aggregates them, pushes the result to BQ, and deletes the tasks.
55 | * Continues until there are no more tasks to lease or some sort of error is encountered. This method
56 | * should be called regularly on a cron schedule (say, once per minute).
57 | */
58 | public void execute() {
59 | log.finest("Pulling");
60 |
61 | while (true) {
62 | try {
63 | if (processOneBatch() < batchSize)
64 | return;
65 |
66 | } catch (RuntimeException ex) {
67 | log.log(Level.WARNING, "Exception while processing insight data; aborting for now", ex);
68 | return;
69 | }
70 | }
71 | }
72 |
73 | /**
74 | * @return the # of tasks actually processed
75 | */
76 | private int processOneBatch() {
77 | List> handles = queue.lease(leaseDurationSeconds, TimeUnit.SECONDS, batchSize);
78 |
79 | if (!handles.isEmpty()) {
80 | log.finer("Leased " + handles.size() + " bucketlist tasks");
81 |
82 | BucketAggregator aggregator = new BucketAggregator();
83 |
84 | for (TaskHandleHelper handle : handles) {
85 | aggregator.aggregate(handle.getPayload().getBuckets());
86 | }
87 |
88 | bigUploader.upload(aggregator.getBuckets());
89 |
90 | queue.delete(handles);
91 | }
92 |
93 | return handles.size();
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/puller/TablePicker.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | import com.google.api.client.googleapis.json.GoogleJsonResponseException;
4 | import com.google.api.services.bigquery.Bigquery;
5 | import com.google.api.services.bigquery.model.Table;
6 | import com.google.api.services.bigquery.model.TableFieldSchema;
7 | import com.google.api.services.bigquery.model.TableReference;
8 | import com.google.api.services.bigquery.model.TableSchema;
9 | import com.google.common.annotations.VisibleForTesting;
10 |
11 | import lombok.Getter;
12 | import lombok.RequiredArgsConstructor;
13 | import lombok.Setter;
14 | import lombok.extern.java.Log;
15 | import javax.inject.Inject;
16 | import java.io.IOException;
17 | import java.text.DateFormat;
18 | import java.text.SimpleDateFormat;
19 | import java.util.ArrayList;
20 | import java.util.Date;
21 | import java.util.List;
22 |
23 | /**
24 | * Manages tables on our behalf. Makes sure we have enough tables to pick from.
25 | */
26 | @Log
27 | public class TablePicker {
28 | /** Number of days ahead to create tables. */
29 | private static final int DAYS_AHEAD = 7;
30 |
31 | /** */
32 | private static final long MILLIS_PER_DAY = 1000 * 60 * 60 * 24;
33 |
34 | private final BigqueryHandler bigqueryHandler;
35 | private final InsightDataset insightDataset;
36 |
37 | /**
38 | * Default format is SimpleDateFormat("'OBJSTATS_'yyyyMMdd") which will produce
39 | * values like OBJSTATS_20150114. If you set the format, be sure to include any
40 | * desired table name prefix.
41 | */
42 | @Getter @Setter
43 | private DateFormat format = new SimpleDateFormat("'OBJSTATS_'yyyyMMdd");
44 |
45 | @Inject
46 | public TablePicker(Bigquery bigquery, InsightDataset insightDataset) {
47 | this(new DefaultBigqueryHandler(bigquery), insightDataset);
48 | }
49 |
50 | @VisibleForTesting
51 | TablePicker(BigqueryHandler bigqueryHandler, InsightDataset insightDataset) {
52 | this.bigqueryHandler = bigqueryHandler;
53 | this.insightDataset = insightDataset;
54 | }
55 |
56 | /** */
57 | public String pick() {
58 | return tableIdFor(new Date());
59 | }
60 |
61 | /**
62 | * Make sure we have a week's worth of tables. This should be called periodically via cron - more than once a week.
63 | */
64 | public void ensureEnoughTables() throws IOException {
65 | log.finer("Ensuring sufficient tables for " + DAYS_AHEAD + " days");
66 |
67 | long now = System.currentTimeMillis();
68 |
69 | for (int i = 0; i < DAYS_AHEAD; i++) {
70 | String tableId = tableIdFor(new Date(now + MILLIS_PER_DAY * i));
71 | ensureTable(tableId);
72 | }
73 | }
74 |
75 | /** */
76 | private String tableIdFor(Date date) {
77 | return format.format(date);
78 | }
79 |
80 | /** */
81 | private void ensureTable(String tableId) throws IOException {
82 | log.finest("Ensuring table exists: " + tableId);
83 |
84 | TableReference reference = new TableReference();
85 | reference.setProjectId(insightDataset.projectId());
86 | reference.setDatasetId(insightDataset.datasetId());
87 | reference.setTableId(tableId);
88 |
89 | Table table = new Table();
90 | table.setTableReference(reference);
91 | table.setSchema(schema());
92 |
93 | if (getTable(tableId) == null) {
94 | bigqueryHandler.tablesInsert(insightDataset, table);
95 | } else {
96 | bigqueryHandler.tablesUpdate(insightDataset, tableId, table);
97 | }
98 | }
99 |
100 | private TableSchema schema() {
101 | List fields = new ArrayList<>();
102 |
103 | fields.add(tableFieldSchema("uploaded", "TIMESTAMP"));
104 | fields.add(tableFieldSchema("codepoint", "STRING"));
105 | fields.add(tableFieldSchema("namespace", "STRING"));
106 | fields.add(tableFieldSchema("module", "STRING"));
107 | fields.add(tableFieldSchema("version", "STRING"));
108 | fields.add(tableFieldSchema("kind", "STRING"));
109 | fields.add(tableFieldSchema("op", "STRING"));
110 | fields.add(tableFieldSchema("query", "STRING"));
111 | fields.add(tableFieldSchema("time", "TIMESTAMP"));
112 |
113 | fields.add(tableFieldSchema("reads", "INTEGER"));
114 | fields.add(tableFieldSchema("writes", "INTEGER"));
115 |
116 | TableSchema schema = new TableSchema();
117 | schema.setFields(fields);
118 |
119 | return schema;
120 | }
121 |
122 | private TableFieldSchema tableFieldSchema(String name, String type) {
123 | TableFieldSchema field = new TableFieldSchema();
124 | field.setName(name);
125 | field.setType(type);
126 | return field;
127 | }
128 |
129 | private Table getTable(String tableId) throws IOException {
130 | try {
131 | return bigqueryHandler.tablesGet(insightDataset, tableId);
132 | } catch (GoogleJsonResponseException e) {
133 | if (e.getStatusCode() == 404) {
134 | log.finest("Table " + tableId + " is missing.");
135 | return null;
136 | } else {
137 | throw e;
138 | }
139 | }
140 | }
141 |
142 | @VisibleForTesting
143 | interface BigqueryHandler {
144 | Table tablesGet(InsightDataset insightDataset, String tableId) throws IOException;
145 | Table tablesInsert(InsightDataset insightDataset, Table table) throws IOException;
146 | Table tablesUpdate(InsightDataset insightDataset, String tableId, Table table) throws IOException;
147 | }
148 |
149 | @RequiredArgsConstructor
150 | private static class DefaultBigqueryHandler implements BigqueryHandler {
151 |
152 | private final Bigquery bigquery;
153 |
154 | @Override
155 | public Table tablesGet(InsightDataset insightDataset, String tableId) throws IOException {
156 | return bigquery.tables().get(insightDataset.projectId(), insightDataset.datasetId(), tableId).execute();
157 | }
158 |
159 | @Override
160 | public Table tablesInsert(InsightDataset insightDataset, Table table) throws IOException {
161 | return bigquery.tables().insert(insightDataset.projectId(), insightDataset.datasetId(), table).execute();
162 | }
163 |
164 | @Override
165 | public Table tablesUpdate(InsightDataset insightDataset, String tableId, Table table) throws IOException {
166 | return bigquery.tables().update(insightDataset.projectId(), insightDataset.datasetId(), tableId, table).execute();
167 | }
168 | }
169 | }
170 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/servlet/AbstractBigQueryServlet.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.servlet;
2 |
3 | import com.google.api.services.bigquery.Bigquery;
4 | import com.googlecode.objectify.insight.puller.InsightDataset;
5 | import javax.servlet.http.HttpServlet;
6 |
7 | /**
8 | * Base servlet for nonguice servlets that use bigquery. Users must extend this class and implement
9 | * the methods that provide Bigquery information.
10 | */
11 | abstract public class AbstractBigQueryServlet extends HttpServlet {
12 |
13 | private static final long serialVersionUID = 1;
14 |
15 | /**
16 | * Implement this to provide the projectId and datasetId where we will store data.
17 | */
18 | abstract protected InsightDataset insightDataset();
19 |
20 | /**
21 | * Implement this to provide the authenticated bigquery object.
22 | */
23 | abstract protected Bigquery bigquery();
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/servlet/AbstractPullerServlet.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.servlet;
2 |
3 | import com.google.api.services.bigquery.Bigquery;
4 | import com.google.appengine.api.taskqueue.Queue;
5 | import com.google.appengine.api.taskqueue.QueueFactory;
6 | import com.googlecode.objectify.insight.Flusher;
7 | import com.googlecode.objectify.insight.puller.BigUploader;
8 | import com.googlecode.objectify.insight.puller.InsightDataset;
9 | import com.googlecode.objectify.insight.puller.Puller;
10 | import com.googlecode.objectify.insight.puller.TablePicker;
11 | import javax.servlet.ServletException;
12 | import javax.servlet.http.HttpServletRequest;
13 | import javax.servlet.http.HttpServletResponse;
14 | import java.io.IOException;
15 |
16 | /**
17 | * Call this servlet from cron once per minute. It will empty the pull queue and then go back to sleep.
18 | * Tasks are pulled off the pull queue, aggregated, and then pushed to BigQuery.
19 | *
20 | * Extend this if you do not use guice.
21 | */
22 | abstract public class AbstractPullerServlet extends AbstractBigQueryServlet {
23 |
24 | private static final long serialVersionUID = 1;
25 |
26 | /** */
27 | private Puller puller;
28 |
29 | /** This is what guice is for */
30 | @Override
31 | public void init() throws ServletException {
32 | InsightDataset insightDataset = insightDataset();
33 | Queue queue = QueueFactory.getQueue(queueName());
34 | Bigquery bigquery = bigquery();
35 | TablePicker tablePicker = new TablePicker(bigquery, insightDataset);
36 | BigUploader bigUploader = new BigUploader(bigquery, insightDataset, tablePicker);
37 |
38 | puller = new Puller(queue, bigUploader);
39 | }
40 |
41 | /**
42 | * Override this to change the name of the queue we pull from. Be sure to change the value in the Flusher as well.
43 | */
44 | protected String queueName() {
45 | return Flusher.DEFAULT_QUEUE;
46 | }
47 |
48 | @Override
49 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
50 | puller.execute();
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/servlet/AbstractTableMakerServlet.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.servlet;
2 |
3 | import com.googlecode.objectify.insight.puller.TablePicker;
4 | import javax.servlet.ServletException;
5 | import javax.servlet.http.HttpServletRequest;
6 | import javax.servlet.http.HttpServletResponse;
7 | import java.io.IOException;
8 |
9 | /**
10 | * Call this servlet from cron once per day. It will make sure there are an appropriate set of tables in bigquery.
11 | * Make sure this servlet starts before the PullerServlet.
12 | *
13 | * Extend this if you do not use guice.
14 | */
15 | abstract public class AbstractTableMakerServlet extends AbstractBigQueryServlet {
16 |
17 | private static final long serialVersionUID = 1;
18 |
19 | /** */
20 | private TablePicker tablePicker;
21 |
22 | /** Sets up the picker based on the abstract methods */
23 | @Override
24 | public void init() throws ServletException {
25 | tablePicker = new TablePicker(bigquery(), insightDataset());
26 | }
27 |
28 | @Override
29 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
30 | tablePicker.ensureEnoughTables();
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/servlet/GuicePullerServlet.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.servlet;
2 |
3 | import com.googlecode.objectify.insight.puller.Puller;
4 | import javax.inject.Inject;
5 | import javax.inject.Singleton;
6 | import javax.servlet.ServletException;
7 | import javax.servlet.http.HttpServlet;
8 | import javax.servlet.http.HttpServletRequest;
9 | import javax.servlet.http.HttpServletResponse;
10 | import java.io.IOException;
11 |
12 | /**
13 | * Call this servlet from cron once per minute. It will empty the pull queue and then go back to sleep.
14 | * Tasks are pulled off the pull queue, aggregated, and then pushed to BigQuery.
15 | */
16 | @Singleton
17 | public class GuicePullerServlet extends HttpServlet {
18 |
19 | private static final long serialVersionUID = 1;
20 |
21 | /** */
22 | private final Puller puller;
23 |
24 | /** */
25 | @Inject
26 | public GuicePullerServlet(Puller puller) {
27 | this.puller = puller;
28 | }
29 |
30 | @Override
31 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
32 | puller.execute();
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/servlet/GuiceTableMakerServlet.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.servlet;
2 |
3 | import com.googlecode.objectify.insight.puller.TablePicker;
4 | import javax.inject.Inject;
5 | import javax.inject.Singleton;
6 | import javax.servlet.ServletException;
7 | import javax.servlet.http.HttpServlet;
8 | import javax.servlet.http.HttpServletRequest;
9 | import javax.servlet.http.HttpServletResponse;
10 | import java.io.IOException;
11 |
12 | /**
13 | * Call this servlet from cron once per day. It will make sure there are an appropriate set of tables in bigquery.
14 | * Make sure this servlet starts before the PullerServlet.
15 | */
16 | @Singleton
17 | public class GuiceTableMakerServlet extends HttpServlet {
18 |
19 | private static final long serialVersionUID = 1;
20 |
21 | /** */
22 | private final TablePicker tablePicker;
23 |
24 | /** */
25 | @Inject
26 | public GuiceTableMakerServlet(TablePicker tablePicker) {
27 | this.tablePicker = tablePicker;
28 | }
29 |
30 | @Override
31 | protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
32 | tablePicker.ensureEnoughTables();
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/util/QueueHelper.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.util;
2 |
3 | import com.fasterxml.jackson.core.JsonProcessingException;
4 | import com.fasterxml.jackson.databind.ObjectMapper;
5 | import com.google.appengine.api.taskqueue.Queue;
6 | import com.google.appengine.api.taskqueue.QueueConstants;
7 | import com.google.appengine.api.taskqueue.TaskHandle;
8 | import com.google.appengine.api.taskqueue.TaskOptions;
9 | import com.google.common.base.Function;
10 | import com.google.common.collect.Iterables;
11 | import com.google.common.collect.Lists;
12 | import lombok.Data;
13 | import java.util.List;
14 | import java.util.concurrent.TimeUnit;
15 |
16 | /** Just a slightly more convenient interface for our purposes */
17 | @Data
18 | public class QueueHelper {
19 | static final ObjectMapper MAPPER = new ObjectMapper();
20 |
21 | /** */
22 | private final Queue queue;
23 | private final Class clazz;
24 |
25 | /** */
26 | public void add(T jsonifyMe) {
27 | queue.addAsync(null, makeTask(jsonifyMe));
28 | }
29 |
30 | /** Allows any number of tasks; automatically partitions as necessary */
31 | public void add(Iterable payloads) {
32 | Iterable opts = Iterables.transform(payloads, new Function() {
33 | @Override
34 | public TaskOptions apply(T thing) {
35 | return makeTask(thing);
36 | }
37 | });
38 |
39 | Iterable> partitioned = Iterables.partition(opts, QueueConstants.maxTasksPerAdd());
40 |
41 | for (List piece: partitioned)
42 | queue.addAsync(null, piece);
43 | }
44 |
45 | /** */
46 | private TaskOptions makeTask(T jsonifyMe) {
47 | try {
48 | byte[] payload = MAPPER.writeValueAsBytes(jsonifyMe);
49 | return TaskOptions.Builder.withMethod(TaskOptions.Method.PULL).payload(payload);
50 | } catch (JsonProcessingException e) {
51 | throw new RuntimeException(e);
52 | }
53 | }
54 |
55 | /** */
56 | public List> lease(int duration, TimeUnit units, int count) {
57 | List handles = queue.leaseTasks(duration, units, count);
58 |
59 | return Lists.transform(handles, new Function>() {
60 | @Override
61 | public TaskHandleHelper apply(TaskHandle taskHandle) {
62 | return new TaskHandleHelper(taskHandle, clazz);
63 | }
64 | });
65 | }
66 |
67 | /** */
68 | public void delete(List> handles) {
69 | List raw = Lists.transform(handles, new Function, TaskHandle>() {
70 | @Override
71 | public TaskHandle apply(TaskHandleHelper input) {
72 | return input.getRaw();
73 | }
74 | });
75 |
76 | queue.deleteTaskAsync(raw);
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/util/ReflectionUtils.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.util;
2 |
3 | import java.lang.reflect.Method;
4 |
5 | /**
6 | * Working around more checked exception brain damage
7 | */
8 | public class ReflectionUtils {
9 | public static Method getMethod(Class clazz, String methodName, Class... parameterTypes) {
10 | try {
11 | return clazz.getMethod(methodName, parameterTypes);
12 | } catch (NoSuchMethodException e) {
13 | throw new RuntimeException(e);
14 | }
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/util/StackTraceUtils.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.util;
2 |
3 | import java.util.regex.Matcher;
4 | import java.util.regex.Pattern;
5 |
6 | public class StackTraceUtils {
7 |
8 | private static final String ENHANCER_BY = "EnhancerBy";
9 | private static final String FAST_CLASS_BY = "FastClassBy";
10 |
11 | // http://regex101.com/r/aC1pS0/5
12 | private static final Pattern pattern = Pattern.compile("((?:" + ENHANCER_BY + "|" + FAST_CLASS_BY + ")\\w+)(\\${2})(\\w+)(\\${0,2}\\.?)");
13 |
14 | public static String removeMutableEnhancements(String oldStack) {
15 | // much faster than pattern.matcher
16 | if (!containsMutableEnhancements(oldStack)) {
17 | return oldStack;
18 | }
19 |
20 | String stack = oldStack;
21 | Matcher m = pattern.matcher(stack);
22 | if (m.find()) {
23 | // replace first number with "number" and second number with the first
24 | stack = m.replaceAll("$1$2$4");
25 | }
26 | return stack;
27 | }
28 |
29 | public static boolean containsMutableEnhancements(String s) {
30 | return s.contains(ENHANCER_BY) || s.contains(FAST_CLASS_BY);
31 | }
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/googlecode/objectify/insight/util/TaskHandleHelper.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.util;
2 |
3 | import com.google.appengine.api.taskqueue.TaskHandle;
4 | import lombok.RequiredArgsConstructor;
5 | import java.io.IOException;
6 |
7 | /** */
8 | @RequiredArgsConstructor
9 | public class TaskHandleHelper {
10 | private final TaskHandle handle;
11 | private final Class clazz;
12 |
13 | /** */
14 | public T getPayload() {
15 | try {
16 | return QueueHelper.MAPPER.readValue(handle.getPayload(), clazz);
17 | } catch (IOException e) {
18 | throw new RuntimeException(e);
19 | }
20 | }
21 |
22 | /** */
23 | public TaskHandle getRaw() {
24 | return handle;
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/CodepointerTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight;
2 |
3 | import static org.hamcrest.Matchers.containsString;
4 | import static org.hamcrest.Matchers.equalTo;
5 | import static org.hamcrest.Matchers.greaterThan;
6 | import static org.hamcrest.Matchers.not;
7 | import static org.junit.Assert.assertThat;
8 |
9 | import org.hamcrest.Matchers;
10 | import org.testng.annotations.Test;
11 |
12 | import com.google.common.collect.Iterables;
13 | import com.google.common.collect.Lists;
14 | import com.googlecode.objectify.insight.Codepointer.AdvancedStackProducer;
15 | import com.googlecode.objectify.insight.Codepointer.LegacyStackProducer;
16 |
17 | /**
18 | */
19 | public class CodepointerTest {
20 |
21 | @Test
22 | public void codepointingCanBeDisabled() throws Exception {
23 | Codepointer codepointer = new Codepointer();
24 | assertThat(codepointer.getCodepoint(), not(equalTo("disabled")));
25 | codepointer.setDisabled(true);
26 | assertThat(codepointer.getCodepoint(), equalTo("disabled"));
27 | }
28 |
29 | @Test
30 | public void testDefaultStackProducer() {
31 | assertThat(new Codepointer().getStackProducer(), Matchers.instanceOf(LegacyStackProducer.class));
32 | }
33 |
34 | @Test
35 | public void testCodepointForCompacterStack() {
36 | final StackTraceElement ste = new StackTraceElement("className", "methodName", "fileName", 1);
37 | final String compactedStack = Exception.class.getName() + "\r\n" + "\tat " + ste.toString() + "\r\n";
38 |
39 | final String codepoint1 = new Codepointer().setStackProducer(new LegacyStackProducer()).getCodepoint();
40 | final String codepoint2 = new Codepointer().setStackProducer(new AdvancedStackProducer()).getCodepoint();
41 | final String codepoint3 = new Codepointer().setStackProducer(new AdvancedStackProducer() {
42 | @Override
43 | protected Iterable filterStack(Iterable stack) {
44 | return stack;
45 | }
46 | }).getCodepoint();
47 | final String codepoint4 = new Codepointer().setStackProducer(new AdvancedStackProducer() {
48 | @Override
49 | protected Iterable filterStack(Iterable stack) {
50 | return Lists.newArrayList(ste);
51 | }
52 | }).getCodepoint();
53 |
54 | assertThat(codepoint3, not(equalTo(codepoint2)));
55 |
56 | assertThat(codepoint4, not(equalTo(codepoint1)));
57 | assertThat(codepoint4, not(equalTo(codepoint2)));
58 | assertThat(codepoint4, not(equalTo(codepoint3)));
59 | assertThat(codepoint4, equalTo(new Codepointer().digest(compactedStack)));
60 | }
61 |
62 | @Test
63 | public void testCompacterStack() {
64 | final String stack1 = new Codepointer().setStackProducer(new LegacyStackProducer()).stack();
65 | final String stack2 = new Codepointer().setStackProducer(new AdvancedStackProducer()).stack();
66 | final String stack3 = new Codepointer().setStackProducer(new AdvancedStackProducer() {
67 | @Override
68 | protected Iterable filterStack(Iterable stack) {
69 | return stack;
70 | }
71 | }).stack();
72 | final String stack4 = new Codepointer().setStackProducer(new AdvancedStackProducer() {
73 | @Override
74 | protected Iterable filterStack(Iterable stack) {
75 | return Iterables.limit(stack, 1);
76 | }
77 | }).stack();
78 |
79 | assertThat(getStackTraceElementCount(stack1), greaterThan(2));
80 | assertThat(getStackTraceElementCount(stack2), greaterThan(2));
81 | assertThat(getStackTraceElementCount(stack3), greaterThan(2));
82 | assertThat(getStackTraceElementCount(stack4), equalTo(1));
83 | }
84 |
85 | private int getStackTraceElementCount(final String stack) {
86 | return stack.split("\tat").length - 1;
87 | }
88 |
89 | @Test
90 | public void testAdvancedFiltering() {
91 | final String stack1 = new Codepointer().setStackProducer(new LegacyStackProducer()).stack();
92 | final String stack2 = new Codepointer().setStackProducer(new AdvancedStackProducer()).stack();
93 |
94 | assertThat(stack1, containsString("com.googlecode.objectify."));
95 | assertThat(stack2, not(containsString("com.googlecode.objectify.")));
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/puller/TablePickerTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller;
2 |
3 | import com.google.api.client.googleapis.json.GoogleJsonResponseException;
4 | import com.google.api.client.googleapis.testing.json.GoogleJsonResponseExceptionFactoryTesting;
5 | import com.google.api.client.testing.json.MockJsonFactory;
6 | import com.google.api.services.bigquery.model.Table;
7 | import com.googlecode.objectify.insight.puller.InsightDataset;
8 | import com.googlecode.objectify.insight.puller.TablePicker;
9 | import com.googlecode.objectify.insight.puller.TablePicker.BigqueryHandler;
10 | import com.googlecode.objectify.insight.test.util.TestBase;
11 | import org.mockito.Mock;
12 | import org.mockito.stubbing.OngoingStubbing;
13 | import org.testng.annotations.BeforeMethod;
14 | import org.testng.annotations.Test;
15 |
16 | import java.io.IOException;
17 | import java.text.SimpleDateFormat;
18 | import java.util.Date;
19 | import static org.hamcrest.Matchers.equalTo;
20 | import static org.junit.Assert.assertThat;
21 | import static org.mockito.Matchers.any;
22 | import static org.mockito.Mockito.when;
23 |
24 | /**
25 | */
26 | public class TablePickerTest extends TestBase {
27 |
28 | @Mock BigqueryHandler bigqueryHandler;
29 | @Mock InsightDataset insightDataset;
30 |
31 | private TablePicker picker;
32 |
33 | @BeforeMethod
34 | public void before() {
35 | when(insightDataset.projectId()).thenReturn("foo");
36 | when(insightDataset.datasetId()).thenReturn("bar");
37 |
38 | picker = new TablePicker(bigqueryHandler, insightDataset);
39 | }
40 |
41 | @Test
42 | public void pickerPicksTheRightName() throws Exception {
43 | String picked = picker.pick();
44 |
45 | String expected = "OBJSTATS_" + new SimpleDateFormat("yyyyMMdd").format(new Date());
46 |
47 | assertThat(picked, equalTo(expected));
48 | }
49 |
50 | @Test
51 | public void ensuringTablesWorksWithoutExistingTables() throws Exception {
52 | whenTablesGet().thenThrow(notFoundGoogleJsonResponseException());
53 | whenTablesInsert().thenReturn(new Table());
54 | whenTablesUpdate().thenThrow(notFoundGoogleJsonResponseException());
55 |
56 | // It's enough just to make sure this doesn't produce a higher level exception
57 | picker.ensureEnoughTables();
58 | }
59 |
60 | @Test
61 | public void ensuringTablesWorksEvenIfTablesExist() throws Exception {
62 | whenTablesGet().thenReturn(new Table());
63 | whenTablesInsert().thenThrow(duplicateGoogleJsonResponseException());
64 | whenTablesUpdate().thenReturn(new Table());
65 |
66 | // It's enough just to make sure this doesn't produce a higher level exception
67 | picker.ensureEnoughTables();
68 | }
69 |
70 | @Test(expectedExceptions = GoogleJsonResponseException.class)
71 | public void ensuringTablesFailsIfNonExistingTablesCantBeInserted() throws Exception {
72 | whenTablesGet().thenThrow(notFoundGoogleJsonResponseException());
73 | whenTablesInsert().thenThrow(unknownGoogleJsonResponseException());
74 |
75 | // It's enough just to make sure this doesn't produce a higher level exception
76 | picker.ensureEnoughTables();
77 | }
78 |
79 | @Test(expectedExceptions = GoogleJsonResponseException.class)
80 | public void ensuringTablesFailsIfExistingTablesCantBeUpdated() throws Exception {
81 | whenTablesGet().thenReturn(new Table());
82 | whenTablesUpdate().thenThrow(unknownGoogleJsonResponseException());
83 |
84 | // It's enough just to make sure this doesn't produce a higher level exception
85 | picker.ensureEnoughTables();
86 | }
87 |
88 | private OngoingStubbing whenTablesGet() throws IOException {
89 | return when(bigqueryHandler.tablesGet(any(InsightDataset.class), any(String.class)));
90 | }
91 |
92 | private OngoingStubbing whenTablesInsert() throws IOException {
93 | return when(bigqueryHandler.tablesInsert(any(InsightDataset.class), any(Table.class)));
94 | }
95 |
96 | private OngoingStubbing whenTablesUpdate() throws IOException {
97 | return when(bigqueryHandler.tablesUpdate(any(InsightDataset.class), any(String.class), any(Table.class)));
98 | }
99 |
100 | private GoogleJsonResponseException notFoundGoogleJsonResponseException() throws IOException {
101 | return googleJsonResponseException(404, "notFound");
102 | }
103 |
104 | private GoogleJsonResponseException duplicateGoogleJsonResponseException() throws IOException {
105 | return googleJsonResponseException(409, "duplicate");
106 | }
107 |
108 | private GoogleJsonResponseException unknownGoogleJsonResponseException() throws IOException {
109 | return googleJsonResponseException(500, "unknown");
110 | }
111 |
112 | private GoogleJsonResponseException googleJsonResponseException(int httpCode, String reasonPhrase) throws IOException {
113 | return GoogleJsonResponseExceptionFactoryTesting.newMock(new MockJsonFactory(), httpCode, reasonPhrase);
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/puller/test/PullerTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.puller.test;
2 |
3 | import com.google.appengine.api.taskqueue.Queue;
4 | import com.google.appengine.api.taskqueue.TaskHandle;
5 | import com.googlecode.objectify.insight.Bucket;
6 | import com.googlecode.objectify.insight.BucketFactory;
7 | import com.googlecode.objectify.insight.BucketList;
8 | import com.googlecode.objectify.insight.puller.BigUploader;
9 | import com.googlecode.objectify.insight.puller.Puller;
10 | import com.googlecode.objectify.insight.test.util.TestBase;
11 | import org.mockito.Mock;
12 | import org.testng.annotations.BeforeMethod;
13 | import org.testng.annotations.Test;
14 | import java.util.Arrays;
15 | import java.util.concurrent.TimeUnit;
16 | import static org.mockito.Mockito.verify;
17 | import static org.mockito.Mockito.when;
18 |
19 | /**
20 | */
21 | public class PullerTest extends TestBase {
22 |
23 | @Mock private Queue queue;
24 | @Mock private BigUploader bigUploader;
25 |
26 | private BucketFactory bucketFactory;
27 | private Bucket bucket1;
28 | private Bucket bucket2;
29 | private Bucket bucket3;
30 | private Bucket bucket4;
31 | private BucketList bucketList1;
32 | private BucketList bucketList2;
33 | private Puller puller;
34 |
35 | @BeforeMethod
36 | public void setUpFixture() throws Exception {
37 | bucketFactory = new BucketFactory();
38 |
39 | bucket1 = bucketFactory.forGet("here", "ns", "kindA", 11);
40 | bucket2 = bucketFactory.forGet("here", "ns", "kindB", 22);
41 | bucket3 = bucketFactory.forGet("here", "ns", "kindA", 33);
42 | bucket4 = bucketFactory.forGet("here", "ns", "kindB", 44);
43 |
44 | bucketList1 = new BucketList(Arrays.asList(bucket1, bucket2));
45 | bucketList2 = new BucketList(Arrays.asList(bucket3, bucket4));
46 |
47 | puller = new Puller(queue, bigUploader);
48 | }
49 |
50 | @Test
51 | public void uploadsAggregatedLeasedTasks() throws Exception {
52 |
53 | TaskHandle taskHandle1 = makeTaskHandle(bucketList1);
54 | TaskHandle taskHandle2 = makeTaskHandle(bucketList2);
55 |
56 | when(queue.leaseTasks(Puller.DEFAULT_LEASE_DURATION_SECONDS, TimeUnit.SECONDS, Puller.DEFAULT_BATCH_SIZE))
57 | .thenReturn(Arrays.asList(taskHandle1, taskHandle2));
58 |
59 | puller.execute();
60 |
61 | verify(queue).deleteTaskAsync(Arrays.asList(taskHandle1, taskHandle2));
62 |
63 | verify(bigUploader).upload(buckets(Arrays.asList(
64 | bucketFactory.forGet("here", "ns", "kindA", 44),
65 | bucketFactory.forGet("here", "ns", "kindB", 66)
66 | )));
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/FlusherTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.fasterxml.jackson.databind.ObjectMapper;
4 | import com.google.appengine.api.datastore.Transaction;
5 | import com.google.appengine.api.taskqueue.Queue;
6 | import com.google.appengine.api.taskqueue.TaskOptions;
7 | import com.googlecode.objectify.insight.Bucket;
8 | import com.googlecode.objectify.insight.BucketFactory;
9 | import com.googlecode.objectify.insight.BucketList;
10 | import com.googlecode.objectify.insight.Flusher;
11 | import com.googlecode.objectify.insight.test.util.TestBase;
12 | import org.mockito.ArgumentCaptor;
13 | import org.mockito.Captor;
14 | import org.mockito.Mock;
15 | import org.testng.annotations.BeforeMethod;
16 | import org.testng.annotations.Test;
17 | import java.util.ArrayList;
18 | import java.util.List;
19 | import static org.hamcrest.CoreMatchers.equalTo;
20 | import static org.junit.Assert.assertThat;
21 | import static org.mockito.Matchers.isNull;
22 | import static org.mockito.Mockito.verify;
23 |
24 | /**
25 | */
26 | public class FlusherTest extends TestBase {
27 |
28 | @Mock private Queue queue;
29 | @Captor private ArgumentCaptor taskCaptor;
30 |
31 | private BucketFactory bucketFactory;
32 | private Flusher flusher;
33 |
34 | @BeforeMethod
35 | public void setUpFixture() throws Exception {
36 | bucketFactory = constantTimeBucketFactory();
37 | flusher = new Flusher(queue);
38 | }
39 |
40 | @Test
41 | public void flushingBucketsProducesJsonPullTask() throws Exception {
42 | List buckets = new ArrayList<>();
43 | buckets.add(bucketFactory.forGet("here", "ns", "kindA", 123));
44 | buckets.add(bucketFactory.forGet("here", "ns", "kindB", 456));
45 |
46 | flusher.flush(buckets);
47 |
48 | verify(queue).addAsync(isNull(Transaction.class), taskCaptor.capture());
49 | TaskOptions parameter = taskCaptor.getValue();
50 |
51 | byte[] expectedJson = new ObjectMapper().writeValueAsBytes(new BucketList(buckets));
52 |
53 | assertThat(parameter.getPayload(), equalTo(expectedJson));
54 |
55 | // damn, no way to check headers for content type because there is no accessor on TaskOptions
56 | }
57 |
58 | }
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/InsightAsyncDatastoreServiceTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.google.appengine.api.datastore.AsyncDatastoreService;
4 | import com.google.appengine.api.datastore.Entity;
5 | import com.google.appengine.api.datastore.Key;
6 | import com.google.appengine.api.datastore.KeyFactory;
7 | import com.google.appengine.api.datastore.PreparedQuery;
8 | import com.google.appengine.api.datastore.Query;
9 | import com.google.appengine.api.datastore.Transaction;
10 | import com.googlecode.objectify.insight.BucketFactory;
11 | import com.googlecode.objectify.insight.Codepointer;
12 | import com.googlecode.objectify.insight.Collector;
13 | import com.googlecode.objectify.insight.InsightAsyncDatastoreService;
14 | import com.googlecode.objectify.insight.InsightPreparedQuery;
15 | import com.googlecode.objectify.insight.Recorder;
16 | import com.googlecode.objectify.insight.test.util.TestBase;
17 | import org.mockito.Mock;
18 | import org.testng.annotations.BeforeMethod;
19 | import org.testng.annotations.Test;
20 | import java.util.Arrays;
21 | import java.util.Collections;
22 | import static org.hamcrest.Matchers.instanceOf;
23 | import static org.hamcrest.Matchers.not;
24 | import static org.junit.Assert.assertThat;
25 | import static org.mockito.Mockito.verify;
26 | import static org.mockito.Mockito.when;
27 |
28 | /**
29 | * Tests the nonquery methods. Query methods are tested in InsightPreparedQueryTest.
30 | */
31 | public class InsightAsyncDatastoreServiceTest extends TestBase {
32 |
33 | private InsightAsyncDatastoreService service;
34 | private BucketFactory bucketFactory;
35 |
36 | @Mock private AsyncDatastoreService raw;
37 | @Mock private Collector collector;
38 | @Mock private Codepointer codepointer;
39 |
40 | @BeforeMethod
41 | public void setUpFixture() throws Exception {
42 | bucketFactory = constantTimeBucketFactory();
43 |
44 | when(codepointer.getCodepoint()).thenReturn("here");
45 |
46 | Recorder recorder = new Recorder(bucketFactory, collector, codepointer);
47 | recorder.setRecordAll(true);
48 |
49 | service = new InsightAsyncDatastoreService(raw, recorder);
50 | }
51 |
52 | @Test
53 | public void getIsCollected() throws Exception {
54 | runInNamespace("ns", new Runnable() {
55 | @Override
56 | public void run() {
57 | Key key = KeyFactory.createKey("Thing", 123L);
58 | service.get(key);
59 | }
60 | });
61 |
62 | verify(collector).collect(bucketFactory.forGet("here", "ns", "Thing", 1));
63 | }
64 |
65 | @Test
66 | public void getWithTxnIsCollected() throws Exception {
67 | runInNamespace("ns", new Runnable() {
68 | @Override
69 | public void run() {
70 | Key key = KeyFactory.createKey("Thing", 123L);
71 | service.get(null, key);
72 | }
73 | });
74 |
75 | verify(collector).collect(bucketFactory.forGet("here", "ns", "Thing", 1));
76 | }
77 |
78 | @Test
79 | public void getMultiIsCollected() throws Exception {
80 | runInNamespace("ns", new Runnable() {
81 | @Override
82 | public void run() {
83 | Key key = KeyFactory.createKey("Thing", 123L);
84 | service.get(Collections.singleton(key));
85 | }
86 | });
87 |
88 | verify(collector).collect(bucketFactory.forGet("here", "ns", "Thing", 1));
89 | }
90 |
91 | @Test
92 | public void getMultiWithTxnIsCollected() throws Exception {
93 | runInNamespace("ns", new Runnable() {
94 | @Override
95 | public void run() {
96 | Key key = KeyFactory.createKey("Thing", 123L);
97 | service.get(null, Collections.singleton(key));
98 | }
99 | });
100 |
101 | verify(collector).collect(bucketFactory.forGet("here", "ns", "Thing", 1));
102 | }
103 |
104 | @Test
105 | public void putInsertIsCollected() throws Exception {
106 | runInNamespace("ns", new Runnable() {
107 | @Override
108 | public void run() {
109 | Entity ent = new Entity("Thing"); // no id
110 | service.put(ent);
111 | }
112 | });
113 |
114 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", true, 1));
115 | }
116 |
117 | @Test
118 | public void putUpdateIsCollected() throws Exception {
119 | runInNamespace("ns", new Runnable() {
120 | @Override
121 | public void run() {
122 | Entity ent = new Entity("Thing", 123L);
123 | service.put(ent);
124 | }
125 | });
126 |
127 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", false, 1));
128 | }
129 |
130 | @Test
131 | public void putWithTxnInsertIsCollected() throws Exception {
132 | runInNamespace("ns", new Runnable() {
133 | @Override
134 | public void run() {
135 | Entity ent = new Entity("Thing"); // no id
136 | service.put(null, ent);
137 | }
138 | });
139 |
140 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", true, 1));
141 | }
142 |
143 | @Test
144 | public void putWithTxnUpdateIsCollected() throws Exception {
145 | runInNamespace("ns", new Runnable() {
146 | @Override
147 | public void run() {
148 | Entity ent = new Entity("Thing", 123L);
149 | service.put(null, ent);
150 | }
151 | });
152 |
153 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", false, 1));
154 | }
155 |
156 | @Test
157 | public void putMultiIsCollected() throws Exception {
158 | runInNamespace("ns", new Runnable() {
159 | @Override
160 | public void run() {
161 | service.put(Arrays.asList(new Entity("Thing"), new Entity("Thing", 123L)));
162 | }
163 | });
164 |
165 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", true, 1));
166 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", false, 1));
167 | }
168 |
169 | @Test
170 | public void putMultiWithTxnIsCollected() throws Exception {
171 | runInNamespace("ns", new Runnable() {
172 | @Override
173 | public void run() {
174 | service.put(null, Arrays.asList(new Entity("Thing"), new Entity("Thing", 123L)));
175 | }
176 | });
177 |
178 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", true, 1));
179 | verify(collector).collect(bucketFactory.forPut("here", "ns", "Thing", false, 1));
180 | }
181 |
182 | @Test
183 | public void deleteIsCollected() throws Exception {
184 | runInNamespace("ns", new Runnable() {
185 | @Override
186 | public void run() {
187 | Key key = KeyFactory.createKey("Thing", 123L);
188 | service.delete(key);
189 | }
190 | });
191 |
192 | verify(collector).collect(bucketFactory.forDelete("here", "ns", "Thing", 1));
193 | }
194 |
195 | @Test
196 | public void deleteWithTxnIsCollected() throws Exception {
197 | runInNamespace("ns", new Runnable() {
198 | @Override
199 | public void run() {
200 | Key key = KeyFactory.createKey("Thing", 123L);
201 | service.delete((Transaction)null, key);
202 | }
203 | });
204 |
205 | verify(collector).collect(bucketFactory.forDelete("here", "ns", "Thing", 1));
206 | }
207 |
208 | @Test
209 | public void deleteMultiIsCollected() throws Exception {
210 | runInNamespace("ns", new Runnable() {
211 | @Override
212 | public void run() {
213 | Key key = KeyFactory.createKey("Thing", 123L);
214 | service.delete(Collections.singleton(key));
215 | }
216 | });
217 |
218 | verify(collector).collect(bucketFactory.forDelete("here", "ns", "Thing", 1));
219 | }
220 |
221 | @Test
222 | public void deleteMultiWithTxnIsCollected() throws Exception {
223 | runInNamespace("ns", new Runnable() {
224 | @Override
225 | public void run() {
226 | Key key = KeyFactory.createKey("Thing", 123L);
227 | service.delete(null, Collections.singleton(key));
228 | }
229 | });
230 |
231 | verify(collector).collect(bucketFactory.forDelete("here", "ns", "Thing", 1));
232 | }
233 |
234 | @Test
235 | public void keysOnlyQueriesAreNotCollected() throws Exception {
236 | Query query = new Query();
237 | query.setKeysOnly();
238 |
239 | PreparedQuery pq = service.prepare(query);
240 |
241 | assertThat(pq, not(instanceOf(InsightPreparedQuery.class)));
242 | }
243 |
244 | @Test
245 | public void keysOnlyTxnQueriesAreNotCollected() throws Exception {
246 | Query query = new Query();
247 | query.setKeysOnly();
248 |
249 | PreparedQuery pq = service.prepare(null, query);
250 |
251 | assertThat(pq, not(instanceOf(InsightPreparedQuery.class)));
252 | }
253 | }
254 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/InsightCollectorTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.googlecode.objectify.insight.Bucket;
4 | import com.googlecode.objectify.insight.BucketFactory;
5 | import com.googlecode.objectify.insight.Collector;
6 | import com.googlecode.objectify.insight.Flusher;
7 | import com.googlecode.objectify.insight.test.util.TestBase;
8 | import org.mockito.Mock;
9 | import org.testng.annotations.BeforeMethod;
10 | import org.testng.annotations.Test;
11 | import java.util.Iterator;
12 | import java.util.LinkedHashSet;
13 | import java.util.Set;
14 | import static org.mockito.Matchers.anyCollectionOf;
15 | import static org.mockito.Mockito.never;
16 | import static org.mockito.Mockito.verify;
17 |
18 | /**
19 | */
20 | public class InsightCollectorTest extends TestBase {
21 |
22 | @Mock private Flusher flusher;
23 |
24 | private Collector collector;
25 | private BucketFactory bucketFactory;
26 |
27 | @BeforeMethod
28 | public void setUpFixture() throws Exception {
29 | bucketFactory = constantTimeBucketFactory();
30 | collector = new Collector(flusher);
31 | }
32 |
33 | @Test
34 | public void tooManyDifferentBucketsCausesFlush() throws Exception {
35 | collector.setSizeThreshold(2);
36 |
37 | Set buckets = new LinkedHashSet<>();
38 | buckets.add(bucketFactory.forGet("here", "ns", "kindA", 1));
39 | buckets.add(bucketFactory.forGet("here", "ns", "kindB", 1));
40 |
41 | Iterator it = buckets.iterator();
42 |
43 | collector.collect(it.next());
44 |
45 | verify(flusher, never()).flush(anyCollectionOf(Bucket.class));
46 |
47 | collector.collect(it.next());
48 |
49 | verify(flusher).flush(buckets(buckets));
50 | }
51 |
52 | @Test
53 | public void ageCausesFlush() throws Exception {
54 | collector.setAgeThresholdMillis(100);
55 |
56 | Bucket bucket = bucketFactory.forGet("here", "ns", "kindA", 1);
57 |
58 | collector.collect(bucket);
59 | collector.collect(bucket);
60 |
61 | verify(flusher, never()).flush(anyCollectionOf(Bucket.class));
62 |
63 | Thread.sleep(101);
64 | collector.collect(bucket);
65 |
66 | verify(flusher).flush(buckets(bucketFactory.forGet("here", "ns", "kindA", 3)));
67 | }
68 |
69 | @Test
70 | public void sameBucketOverAndOverDoesNotCauseFlush() throws Exception {
71 | collector.setSizeThreshold(2);
72 |
73 | for (int i=0; i<10; i++)
74 | collector.collect(bucketFactory.forGet("here", "ns", "kind", 1));
75 |
76 | verify(flusher, never()).flush(anyCollectionOf(Bucket.class));
77 | }
78 |
79 | @Test
80 | public void sameBucketGetsAggregated() throws Exception {
81 | collector.setSizeThreshold(2);
82 |
83 | collector.collect(bucketFactory.forGet("here", "ns", "kindA", 1));
84 | collector.collect(bucketFactory.forGet("here", "ns", "kindA", 2));
85 | collector.collect(bucketFactory.forGet("here", "ns", "kindA", 3));
86 |
87 | verify(flusher, never()).flush(anyCollectionOf(Bucket.class));
88 |
89 | collector.collect(bucketFactory.forGet("here", "ns", "kindB", 1));
90 |
91 | Set expected = new LinkedHashSet<>();
92 | expected.add(bucketFactory.forGet("here", "ns", "kindA", 6));
93 | expected.add(bucketFactory.forGet("here", "ns", "kindB", 1));
94 |
95 | verify(flusher).flush(buckets(expected));
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/InsightCollectorTimeTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.googlecode.objectify.insight.Bucket;
4 | import com.googlecode.objectify.insight.BucketFactory;
5 | import com.googlecode.objectify.insight.Clock;
6 | import com.googlecode.objectify.insight.Collector;
7 | import com.googlecode.objectify.insight.Flusher;
8 | import com.googlecode.objectify.insight.test.util.TestBase;
9 | import org.mockito.Mock;
10 | import org.testng.annotations.BeforeMethod;
11 | import org.testng.annotations.Test;
12 | import java.util.LinkedHashSet;
13 | import java.util.Set;
14 | import static org.mockito.Mockito.verify;
15 | import static org.mockito.Mockito.when;
16 |
17 | /**
18 | */
19 | public class InsightCollectorTimeTest extends TestBase {
20 |
21 | @Mock private Flusher flusher;
22 | @Mock private Clock clock;
23 |
24 | private BucketFactory bucketFactory;
25 |
26 | private Collector collector;
27 |
28 | @BeforeMethod
29 | public void setUpFixture() throws Exception {
30 | when(clock.getTime()).thenReturn(100L, 200L);
31 |
32 | bucketFactory = new BucketFactory(clock, "module", "version");
33 | collector = new Collector(flusher);
34 | }
35 |
36 | /**
37 | */
38 | @Test
39 | public void differentTimeBucketsAreSplit() throws Exception {
40 | collector.setSizeThreshold(2);
41 |
42 | Set expected = new LinkedHashSet<>();
43 | expected.add(bucketFactory.forGet("here", "ns", "kindA", 1));
44 | expected.add(bucketFactory.forGet("here", "ns", "kindA", 2));
45 |
46 | for (Bucket bucket: expected)
47 | collector.collect(bucket);
48 |
49 | verify(flusher).flush(buckets(expected));
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/InsightPreparedQueryTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.google.appengine.api.datastore.AsyncDatastoreService;
4 | import com.google.appengine.api.datastore.Entity;
5 | import com.google.appengine.api.datastore.FetchOptions;
6 | import com.google.appengine.api.datastore.FetchOptions.Builder;
7 | import com.google.appengine.api.datastore.Key;
8 | import com.google.appengine.api.datastore.KeyFactory;
9 | import com.google.appengine.api.datastore.PreparedQuery;
10 | import com.google.appengine.api.datastore.Query;
11 | import com.google.appengine.api.datastore.QueryResultIterable;
12 | import com.google.appengine.api.datastore.QueryResultList;
13 | import com.google.appengine.api.datastore.Transaction;
14 | import com.google.common.base.Supplier;
15 | import com.google.common.collect.Lists;
16 | import com.googlecode.objectify.insight.BucketFactory;
17 | import com.googlecode.objectify.insight.Codepointer;
18 | import com.googlecode.objectify.insight.Collector;
19 | import com.googlecode.objectify.insight.InsightAsyncDatastoreService;
20 | import com.googlecode.objectify.insight.Recorder;
21 | import com.googlecode.objectify.insight.test.util.FakeQueryResultList;
22 | import com.googlecode.objectify.insight.test.util.FakeQueryResultList.QueryResult;
23 | import com.googlecode.objectify.insight.test.util.TestBase;
24 | import org.mockito.Mock;
25 | import org.testng.annotations.BeforeMethod;
26 | import org.testng.annotations.Test;
27 | import java.util.ArrayList;
28 | import java.util.Iterator;
29 | import java.util.List;
30 | import static org.mockito.Matchers.any;
31 | import static org.mockito.Mockito.verify;
32 | import static org.mockito.Mockito.when;
33 |
34 | /**
35 | */
36 | public class InsightPreparedQueryTest extends TestBase {
37 |
38 | private QueryResult ENTITIES;
39 | private Query QUERY;
40 |
41 | private InsightAsyncDatastoreService service;
42 | private BucketFactory bucketFactory;
43 |
44 | @Mock private Collector collector;
45 | @Mock private AsyncDatastoreService rawService;
46 | @Mock private PreparedQuery rawPq;
47 | @Mock private Codepointer codepointer;
48 |
49 | @BeforeMethod
50 | public void setUpFixture() throws Exception {
51 | bucketFactory = constantTimeBucketFactory();
52 |
53 | when(rawService.prepare(any(Query.class))).thenReturn(rawPq);
54 | when(rawService.prepare(any(Transaction.class), any(Query.class))).thenReturn(rawPq);
55 |
56 | when(codepointer.getCodepoint()).thenReturn("here");
57 |
58 | // Constants, but need to wait until the gae apienvironment is set up
59 | final List entities = runInNamespace("ns", new Supplier>() {
60 | @Override
61 | public List get() {
62 | List entities = new ArrayList<>();
63 | entities.add(new Entity("Thing", 123L));
64 | return entities;
65 | }
66 | });
67 | ENTITIES = FakeQueryResultList.create(entities);
68 |
69 | QUERY = runInNamespace("ns", new Supplier() {
70 | @Override
71 | public Query get() {
72 | return new Query("Thing", KeyFactory.createKey("Parent", 567L));
73 | }
74 | });
75 |
76 | Recorder recorder = new Recorder(bucketFactory, collector, codepointer);
77 | recorder.setRecordAll(true);
78 | service = new InsightAsyncDatastoreService(rawService, recorder);
79 | }
80 |
81 | private void iterate(Iterable iterable) {
82 | iterate(iterable.iterator());
83 | }
84 | private void iterate(Iterator iterator) {
85 | while (iterator.hasNext())
86 | iterator.next();
87 | }
88 |
89 | /** We can get rid of a lot of boilerplate */
90 | private void runTest(Runnable work) {
91 | runInNamespace("ns", work);
92 | verify(collector).collect(bucketFactory.forQuery("here", "ns", "Thing", QUERY.toString(), 1));
93 | }
94 |
95 | @Test
96 | public void queryIsCollected() throws Exception {
97 | when(rawPq.asIterable()).thenReturn(ENTITIES);
98 |
99 | runTest(new Runnable() {
100 | @Override
101 | public void run() {
102 | Iterable it = service.prepare(QUERY).asIterable();
103 | iterate(it);
104 | iterate(it);
105 | }
106 | });
107 | }
108 |
109 | @Test
110 | public void queryWithTxnIsCollected() throws Exception {
111 | when(rawPq.asIterable()).thenReturn(ENTITIES);
112 |
113 | runTest(new Runnable() {
114 | @Override
115 | public void run() {
116 | Iterable iterable = service.prepare(null, QUERY).asIterable();
117 | iterate(iterable);
118 | iterate(iterable);
119 | }
120 | });
121 | }
122 |
123 | @Test
124 | public void asQueryResultIterableIsCollected() throws Exception {
125 | when(rawPq.asQueryResultIterable()).thenReturn(ENTITIES);
126 |
127 | runTest(new Runnable() {
128 | @Override
129 | public void run() {
130 | QueryResultIterable iterable = service.prepare(QUERY).asQueryResultIterable();
131 | iterate(iterable);
132 | iterate(iterable);
133 | }
134 | });
135 | }
136 |
137 | @Test
138 | public void asQueryResultIterableWithOptionsIsCollected() throws Exception {
139 | when(rawPq.asQueryResultIterable(any(FetchOptions.class))).thenReturn(ENTITIES);
140 |
141 | runTest(new Runnable() {
142 | @Override
143 | public void run() {
144 | QueryResultIterable iterable = service.prepare(QUERY).asQueryResultIterable(Builder.withDefaults());
145 | iterate(iterable);
146 | iterate(iterable);
147 | }
148 | });
149 | }
150 |
151 | @Test
152 | public void asIterableWithOptionsIsCollected() throws Exception {
153 | when(rawPq.asIterable(any(FetchOptions.class))).thenReturn(ENTITIES);
154 |
155 | runTest(new Runnable() {
156 | @Override
157 | public void run() {
158 | Iterable iterable = service.prepare(QUERY).asIterable(Builder.withDefaults());
159 | iterate(iterable);
160 | iterate(iterable);
161 | }
162 | });
163 | }
164 |
165 | @Test
166 | public void asListWithOptionsIsCollected() throws Exception {
167 | when(rawPq.asList(any(FetchOptions.class))).thenReturn(ENTITIES);
168 |
169 | runTest(new Runnable() {
170 | @Override
171 | public void run() {
172 | List list = service.prepare(QUERY).asList(Builder.withDefaults());
173 | iterate(list);
174 | iterate(list);
175 | }
176 | });
177 | }
178 |
179 | @Test
180 | public void asQueryResultListWithOptionsIsCollected() throws Exception {
181 | when(rawPq.asQueryResultList(any(FetchOptions.class))).thenReturn(ENTITIES);
182 |
183 | runTest(new Runnable() {
184 | @Override
185 | public void run() {
186 | QueryResultList list = service.prepare(QUERY).asQueryResultList(Builder.withDefaults());
187 | iterate(list);
188 | iterate(list);
189 | }
190 | });
191 | }
192 |
193 | @Test
194 | public void asIteratorIsCollected() throws Exception {
195 | when(rawPq.asIterator()).thenReturn(ENTITIES.iterator());
196 |
197 | runTest(new Runnable() {
198 | @Override
199 | public void run() {
200 | iterate(service.prepare(QUERY).asIterator());
201 | }
202 | });
203 | }
204 |
205 | @Test
206 | public void asIteratorWithOptionsIsCollected() throws Exception {
207 | when(rawPq.asIterator(any(FetchOptions.class))).thenReturn(ENTITIES.iterator());
208 |
209 | runTest(new Runnable() {
210 | @Override
211 | public void run() {
212 | iterate(service.prepare(QUERY).asIterator(FetchOptions.Builder.withDefaults()));
213 | }
214 | });
215 | }
216 |
217 | @Test
218 | public void asQueryResultIteratorIsCollected() throws Exception {
219 | when(rawPq.asQueryResultIterator()).thenReturn(ENTITIES.iterator());
220 |
221 | runTest(new Runnable() {
222 | @Override
223 | public void run() {
224 | iterate(service.prepare(QUERY).asQueryResultIterator());
225 | }
226 | });
227 | }
228 |
229 | @Test
230 | public void asQueryResultIteratorWithOptionsIsCollected() throws Exception {
231 | when(rawPq.asQueryResultIterator(any(FetchOptions.class))).thenReturn(ENTITIES.iterator());
232 |
233 | runTest(new Runnable() {
234 | @Override
235 | public void run() {
236 | iterate(service.prepare(QUERY).asQueryResultIterator(FetchOptions.Builder.withDefaults()));
237 | }
238 | });
239 | }
240 |
241 | @Test
242 | public void asSingleEntityIsCollected() throws Exception {
243 | when(rawPq.asSingleEntity()).thenReturn(ENTITIES.iterator().next());
244 |
245 | runTest(new Runnable() {
246 | @Override
247 | public void run() {
248 | service.prepare(QUERY).asSingleEntity();
249 | }
250 | });
251 | }
252 |
253 | @Test
254 | public void listToArrayIsCollected() throws Exception {
255 | when(rawPq.asList(any(FetchOptions.class))).thenReturn(ENTITIES);
256 |
257 | runTest(new Runnable() {
258 | @Override
259 | public void run() {
260 | List list = service.prepare(QUERY).asList(Builder.withDefaults());
261 | list.toArray();
262 | list.toArray();
263 | }
264 | });
265 | }
266 |
267 | @Test
268 | public void queryResultListToArrayIsCollected() throws Exception {
269 | when(rawPq.asQueryResultList(any(FetchOptions.class))).thenReturn(ENTITIES);
270 |
271 | runTest(new Runnable() {
272 | @Override
273 | public void run() {
274 | List list = service.prepare(QUERY).asQueryResultList(Builder.withDefaults());
275 | list.toArray();
276 | list.toArray();
277 | }
278 | });
279 | }
280 |
281 | @Test
282 | public void listToArray2IsCollected() throws Exception {
283 | when(rawPq.asList(any(FetchOptions.class))).thenReturn(ENTITIES);
284 |
285 | runTest(new Runnable() {
286 | @Override
287 | public void run() {
288 | List list = service.prepare(QUERY).asList(Builder.withDefaults());
289 | list.toArray(new Entity[0]);
290 | list.toArray(new Entity[0]);
291 | }
292 | });
293 | }
294 |
295 | @Test
296 | public void queryResultListToArray2IsCollected() throws Exception {
297 | when(rawPq.asQueryResultList(any(FetchOptions.class))).thenReturn(ENTITIES);
298 |
299 | runTest(new Runnable() {
300 | @Override
301 | public void run() {
302 | List list = service.prepare(QUERY).asQueryResultList(Builder.withDefaults());
303 | list.toArray(new Entity[0]);
304 | list.toArray(new Entity[0]);
305 | }
306 | });
307 | }
308 |
309 | }
310 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/RecorderTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test;
2 |
3 | import com.google.appengine.api.datastore.AsyncDatastoreService;
4 | import com.google.appengine.api.datastore.Key;
5 | import com.google.appengine.api.datastore.KeyFactory;
6 | import com.googlecode.objectify.insight.Bucket;
7 | import com.googlecode.objectify.insight.BucketFactory;
8 | import com.googlecode.objectify.insight.Codepointer;
9 | import com.googlecode.objectify.insight.Collector;
10 | import com.googlecode.objectify.insight.InsightAsyncDatastoreService;
11 | import com.googlecode.objectify.insight.Recorder;
12 | import com.googlecode.objectify.insight.test.util.TestBase;
13 | import org.mockito.Mock;
14 | import org.testng.annotations.BeforeMethod;
15 | import org.testng.annotations.Test;
16 | import static org.mockito.Matchers.any;
17 | import static org.mockito.Mockito.never;
18 | import static org.mockito.Mockito.verify;
19 | import static org.mockito.Mockito.when;
20 |
21 | /**
22 | * Make sure the recorder is behaving appropriately.
23 | */
24 | public class RecorderTest extends TestBase {
25 |
26 | private InsightAsyncDatastoreService service;
27 | private BucketFactory bucketFactory;
28 | private Recorder recorder;
29 |
30 | @Mock private AsyncDatastoreService raw;
31 | @Mock private Collector collector;
32 | @Mock private Codepointer codepointer;
33 |
34 | @BeforeMethod
35 | public void setUpFixture() throws Exception {
36 | bucketFactory = constantTimeBucketFactory();
37 |
38 | when(codepointer.getCodepoint()).thenReturn("here");
39 |
40 | recorder = new Recorder(bucketFactory, collector, codepointer);
41 |
42 | service = new InsightAsyncDatastoreService(raw, recorder);
43 | }
44 |
45 | @Test
46 | public void noRecording() throws Exception {
47 | runInNamespace("ns", new Runnable() {
48 | @Override
49 | public void run() {
50 | Key key = KeyFactory.createKey("Thing", 123L);
51 | service.get(key);
52 | }
53 | });
54 |
55 | verify(collector, never()).collect(any(Bucket.class));
56 | }
57 |
58 | @Test
59 | public void recordsKind() throws Exception {
60 | recorder.recordKind("Thing");
61 |
62 | runInNamespace("ns", new Runnable() {
63 | @Override
64 | public void run() {
65 | Key key = KeyFactory.createKey("Thing", 123L);
66 | service.get(key);
67 | }
68 | });
69 |
70 | verify(collector).collect(bucketFactory.forGet("here", "ns", "Thing", 1));
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/util/BucketsMatcher.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test.util;
2 |
3 | import com.googlecode.objectify.insight.Bucket;
4 | import lombok.RequiredArgsConstructor;
5 | import org.hamcrest.Description;
6 | import org.mockito.ArgumentMatcher;
7 | import java.util.Collection;
8 | import java.util.LinkedHashSet;
9 | import java.util.Set;
10 |
11 | /**
12 | * Hamcrest matcher for exactly matching a collection of buckets which are matched by their key.
13 | * Read/write counts are checked.
14 | */
15 | @RequiredArgsConstructor
16 | public class BucketsMatcher extends ArgumentMatcher> {
17 | private final Set patternSet;
18 |
19 | public BucketsMatcher(Collection pattern) {
20 | patternSet = new LinkedHashSet<>(pattern);
21 | }
22 |
23 | @Override
24 | public boolean matches(Object o) {
25 | @SuppressWarnings("unchecked")
26 | Collection other = (Collection)o;
27 | Set otherSet = new LinkedHashSet<>(other);
28 |
29 | return otherSet.equals(patternSet);
30 | }
31 |
32 | @Override
33 | public void describeTo(Description description) {
34 | description.appendText(patternSet.toString());
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/util/FakeQueryResultList.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test.util;
2 |
3 | import com.google.appengine.api.datastore.Entity;
4 | import com.google.appengine.api.datastore.QueryResultIterable;
5 | import com.google.appengine.api.datastore.QueryResultIterator;
6 | import com.google.appengine.api.datastore.QueryResultList;
7 | import com.googlecode.objectify.insight.util.ReflectionUtils;
8 | import lombok.RequiredArgsConstructor;
9 | import java.lang.reflect.InvocationHandler;
10 | import java.lang.reflect.Method;
11 | import java.lang.reflect.Proxy;
12 | import java.util.List;
13 |
14 | /**
15 | * Fake list, implemented as a proxy
16 | */
17 | @RequiredArgsConstructor
18 | public class FakeQueryResultList implements InvocationHandler {
19 | public static interface QueryResult extends QueryResultList, QueryResultIterable {}
20 |
21 | private static final Method ITERATOR_METHOD = ReflectionUtils.getMethod(List.class, "iterator");
22 |
23 | private final List list;
24 |
25 | @Override
26 | public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
27 | Object result = method.invoke(list, args);
28 |
29 | if (method.equals(ITERATOR_METHOD))
30 | return PassThroughProxy.create(result, QueryResultIterator.class);
31 | else
32 | return result;
33 | }
34 |
35 | public static QueryResult create(List list) {
36 | return (QueryResult)Proxy.newProxyInstance(QueryResult.class.getClassLoader(), new Class[]{QueryResult.class}, new FakeQueryResultList(list));
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/util/PassThroughProxy.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.test.util;
2 |
3 | import lombok.RequiredArgsConstructor;
4 | import java.lang.reflect.InvocationHandler;
5 | import java.lang.reflect.Method;
6 | import java.lang.reflect.Proxy;
7 |
8 | /**
9 | * Simple pass through proxy, lets us work around the type system
10 | */
11 | @RequiredArgsConstructor
12 | public class PassThroughProxy implements InvocationHandler {
13 |
14 | private final Object thing;
15 |
16 | @Override
17 | public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
18 | return method.invoke(thing, args);
19 | }
20 |
21 | public static T create(Object thing, Class intf) {
22 | final Class[] interfaces = { intf };
23 | return (T)Proxy.newProxyInstance(intf.getClassLoader(), interfaces, new PassThroughProxy(thing));
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/test/util/TestBase.java:
--------------------------------------------------------------------------------
1 | /*
2 | */
3 |
4 | package com.googlecode.objectify.insight.test.util;
5 |
6 | import com.fasterxml.jackson.core.JsonProcessingException;
7 | import com.fasterxml.jackson.databind.ObjectMapper;
8 | import com.google.appengine.api.NamespaceManager;
9 | import com.google.appengine.api.taskqueue.TaskHandle;
10 | import com.google.appengine.api.taskqueue.TaskOptions;
11 | import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig;
12 | import com.google.appengine.tools.development.testing.LocalServiceTestHelper;
13 | import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig;
14 | import com.google.common.base.Supplier;
15 | import com.googlecode.objectify.insight.Bucket;
16 | import com.googlecode.objectify.insight.BucketFactory;
17 | import com.googlecode.objectify.insight.Clock;
18 | import com.googlecode.objectify.insight.Flusher;
19 | import org.mockito.MockitoAnnotations;
20 | import org.testng.annotations.AfterMethod;
21 | import org.testng.annotations.BeforeMethod;
22 | import java.util.Collection;
23 | import java.util.Collections;
24 | import static org.mockito.Matchers.argThat;
25 | import static org.mockito.Mockito.mock;
26 | import static org.mockito.Mockito.when;
27 |
28 | /**
29 | * All tests should extend this class to set up the GAE environment.
30 | * @see Unit Testing in Appengine
31 | *
32 | * Also sets up any Mockito annotated fields.
33 | *
34 | * @author Jeff Schnitzer
35 | */
36 | public class TestBase
37 | {
38 | /** */
39 | private final LocalServiceTestHelper helper =
40 | new LocalServiceTestHelper(
41 | // Our tests assume strong consistency
42 | new LocalDatastoreServiceTestConfig().setApplyAllHighRepJobPolicy(),
43 | new LocalTaskQueueTestConfig());
44 | /** */
45 | @BeforeMethod
46 | public void setUp() {
47 | this.helper.setUp();
48 | MockitoAnnotations.initMocks(this);
49 | }
50 |
51 | /** */
52 | @AfterMethod
53 | public void tearDown() {
54 | this.helper.tearDown();
55 | }
56 |
57 | /** */
58 | protected void runInNamespace(String namespace, Runnable runnable) {
59 | String oldNamespace = NamespaceManager.get();
60 | try {
61 | NamespaceManager.set(namespace);
62 |
63 | runnable.run();
64 | } finally {
65 | NamespaceManager.set(oldNamespace);
66 | }
67 | }
68 |
69 | /** */
70 | protected T runInNamespace(String namespace, Supplier supplier) {
71 | String oldNamespace = NamespaceManager.get();
72 | try {
73 | NamespaceManager.set(namespace);
74 |
75 | return supplier.get();
76 | } finally {
77 | NamespaceManager.set(oldNamespace);
78 | }
79 | }
80 |
81 | /** Little bit of boilerplate that makes the tests read better */
82 | protected Collection buckets(Collection matching) {
83 | return argThat(new BucketsMatcher(matching));
84 | }
85 |
86 | /** Little bit of boilerplate that makes the tests read better */
87 | protected Collection buckets(Bucket singletonSetContent) {
88 | return buckets(Collections.singleton(singletonSetContent));
89 | }
90 |
91 | /** Convenience method */
92 | protected byte[] jsonify(Object object) throws JsonProcessingException {
93 | return new ObjectMapper().writeValueAsBytes(object);
94 | }
95 |
96 | /**
97 | * Make a task handle which holds the jsonified payload. Task name is an arbitrary unique string.
98 | */
99 | protected TaskHandle makeTaskHandle(Object payload) throws JsonProcessingException {
100 | return new TaskHandle(
101 | TaskOptions.Builder.withPayload(jsonify(payload), "application/json").taskName(makeUniqueString()),
102 | Flusher.DEFAULT_QUEUE);
103 | }
104 |
105 | protected String makeUniqueString() {
106 | return new Object().toString().split("@")[1];
107 | }
108 |
109 |
110 | /** Useful for making stable tests */
111 | protected BucketFactory constantTimeBucketFactory() {
112 | Clock clock = mock(Clock.class);
113 | when(clock.getTime()).thenReturn(100L); // just a stable value
114 | return new BucketFactory(clock, "module", "version");
115 | }
116 |
117 | }
118 |
--------------------------------------------------------------------------------
/src/test/java/com/googlecode/objectify/insight/util/test/StackTraceUtilsTest.java:
--------------------------------------------------------------------------------
1 | package com.googlecode.objectify.insight.util.test;
2 |
3 | import static org.hamcrest.Matchers.equalTo;
4 | import static org.junit.Assert.assertThat;
5 |
6 | import org.testng.annotations.Test;
7 |
8 | import com.googlecode.objectify.insight.util.StackTraceUtils;
9 |
10 | public class StackTraceUtilsTest {
11 |
12 | private static String nonEnhanced = "at com.googlecode.objectify.insight.StackTracer.stack(StackTracer.java:16)\r\n"
13 | + "\tat com.googlecode.objectify.insight.StackTracerTest.stack(StackTracerTest.java:12)\r\n"
14 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n"
15 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\r\n"
16 | + "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n"
17 | + "\tat java.lang.reflect.Method.invoke(Method.java:606)";
18 |
19 | private static String enhanced = "at com.googlecode.objectify.insight.Codepointer.getCodepoint(Codepointer.java:40)\r\n"
20 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext.codepoint(CodepointTestContext.java:17)\r\n"
21 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$4b065412.CGLIB$codepoint$0()\r\n"
22 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$4b065412$$FastClassByCGLIB$$b6c1cce6.invoke()\r\n"
23 | + "\tat net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)\r\n"
24 | + "\tat com.googlecode.objectify.insight.test.CodepointerTest$2.intercept(CodepointerTest.java:86)\r\n"
25 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$4b065412.codepoint()\r\n"
26 | + "\tat com.googlecode.objectify.insight.test.CodepointerTest.enhancedClassesShouldGenerateSameCodepointHash(CodepointerTest.java:46)\r\n"
27 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n"
28 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\r\n"
29 | + "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n"
30 | + "\tat java.lang.reflect.Method.invoke(Method.java:606)";
31 |
32 | private static String unEnhanced = "at com.googlecode.objectify.insight.Codepointer.getCodepoint(Codepointer.java:40)\r\n"
33 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext.codepoint(CodepointTestContext.java:17)\r\n"
34 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$.CGLIB$codepoint$0()\r\n"
35 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$$$FastClassByCGLIB$$.invoke()\r\n"
36 | + "\tat net.sf.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)\r\n\tat com.googlecode.objectify.insight.test.CodepointerTest$2.intercept(CodepointerTest.java:86)\r\n"
37 | + "\tat com.googlecode.objectify.insight.test.CodepointTestContext$$EnhancerByCGLIB$$.codepoint()\r\n"
38 | + "\tat com.googlecode.objectify.insight.test.CodepointerTest.enhancedClassesShouldGenerateSameCodepointHash(CodepointerTest.java:46)\r\n"
39 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\r\n"
40 | + "\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\r\n"
41 | + "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\r\n"
42 | + "\tat java.lang.reflect.Method.invoke(Method.java:606)";
43 |
44 | @Test
45 | public void nonEnhancedStackTraceShouldRemainTheSame() {
46 | assertThat(StackTraceUtils.removeMutableEnhancements(nonEnhanced), equalTo(nonEnhanced));
47 | }
48 |
49 | @Test
50 | public void enhancedStacktraceShouldHaveDynamicPartsRemoved() {
51 | assertThat(StackTraceUtils.removeMutableEnhancements(enhanced), equalTo(unEnhanced));
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------