├── .gitignore
├── src
├── main
│ ├── resources
│ │ └── META-INF
│ │ │ └── services
│ │ │ └── com.facebook.presto.spi.Plugin
│ └── java
│ │ └── jp
│ │ └── co
│ │ └── yahoo
│ │ └── presto
│ │ └── audit
│ │ ├── serializer
│ │ ├── SerializedLog.java
│ │ ├── LogSerializer.java
│ │ ├── QueryIOMetadataSerializer.java
│ │ ├── QueryFailureInfoSerializer.java
│ │ ├── QueryCompletedEventSerializer.java
│ │ ├── FullLogSerializer.java
│ │ ├── QueryStatisticsSerializer.java
│ │ ├── SimpleLogSerializer.java
│ │ └── AuditRecord.java
│ │ ├── AuditLogPlugin.java
│ │ ├── AuditLogFactory.java
│ │ ├── pulsar
│ │ └── PulsarProducer.java
│ │ ├── AuditLogListener.java
│ │ ├── AuditLogFileWriter.java
│ │ └── AuditConfig.java
├── license
│ └── LICENSE-HEADER.txt
├── sql
│ └── ddl.sql
├── test
│ └── java
│ │ └── jp
│ │ └── co
│ │ └── yahoo
│ │ └── presto
│ │ └── audit
│ │ ├── serializer
│ │ ├── TestSimpleLogSerializer.java
│ │ └── TestFullLogSerializer.java
│ │ ├── pulsar
│ │ └── TestPulsarProducer.java
│ │ ├── TestAuditConfig.java
│ │ ├── TestAuditLogListener.java
│ │ ├── TestHelper.java
│ │ └── TestAuditLogFileWriter.java
└── checkstyle
│ └── checks.xml
├── .mvn
└── wrapper
│ ├── maven-wrapper.jar
│ ├── maven-wrapper.properties
│ └── MavenWrapperDownloader.java
├── .travis.yml
├── README.md
├── mvnw.cmd
├── mvnw
└── pom.xml
/.gitignore:
--------------------------------------------------------------------------------
1 | *.iml
2 | .idea/
3 | target/
4 | **/.DS_Store
5 |
--------------------------------------------------------------------------------
/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin:
--------------------------------------------------------------------------------
1 | jp.co.yahoo.presto.audit.AuditLogPlugin
2 |
--------------------------------------------------------------------------------
/.mvn/wrapper/maven-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yahoojapan/presto-audit/HEAD/.mvn/wrapper/maven-wrapper.jar
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: java
2 | jdk:
3 | - oraclejdk8
4 | - openjdk8
5 | cache:
6 | directories:
7 | - .autoconf
8 | - $HOME/.m2
9 |
--------------------------------------------------------------------------------
/.mvn/wrapper/maven-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip
--------------------------------------------------------------------------------
/src/license/LICENSE-HEADER.txt:
--------------------------------------------------------------------------------
1 | Licensed under the Apache License, Version 2.0 (the "License");
2 | you may not use this file except in compliance with the License.
3 | You may obtain a copy of the License at
4 |
5 | http://www.apache.org/licenses/LICENSE-2.0
6 |
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/SerializedLog.java:
--------------------------------------------------------------------------------
1 | package jp.co.yahoo.presto.audit.serializer;
2 |
3 | public class SerializedLog
4 | {
5 | private String queryId;
6 | private String serializedLog;
7 |
8 | public SerializedLog(String queryId, String serializedLog)
9 | {
10 | this.queryId = queryId;
11 | this.serializedLog = serializedLog;
12 | }
13 |
14 | public String getQueryId()
15 | {
16 | return queryId;
17 | }
18 |
19 | public String getSerializedLog()
20 | {
21 | return serializedLog;
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/LogSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
17 |
18 | import java.io.IOException;
19 |
20 | public interface LogSerializer
21 | {
22 | SerializedLog serialize(QueryCompletedEvent event) throws IOException;
23 | boolean shouldOutput(QueryCompletedEvent event);
24 | }
25 |
--------------------------------------------------------------------------------
/src/sql/ddl.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE presto_audit_1 (
2 | queryId STRING,
3 | query STRING,
4 | uri STRING,
5 | state STRING,
6 |
7 | cpuTime DOUBLE,
8 | wallTime DOUBLE,
9 | queuedTime DOUBLE,
10 | peakMemoryBytes BIGINT, --Deprecated
11 | peakUserMemoryBytes BIGINT,
12 | peakTotalNonRevocableMemoryBytes BIGINT,
13 | totalBytes BIGINT,
14 | totalRows BIGINT,
15 | completedSplits INTEGER,
16 |
17 | createTime STRING, --Deprecated
18 | executionStartTime STRING, --Deprecated
19 | endTime STRING, --Decrecated
20 |
21 | createTimestamp DOUBLE,
22 | executionStartTimestamp DOUBLE,
23 | endTimestamp DOUBLE,
24 |
25 | errorCode INTEGER,
26 | errorName STRING,
27 | failureType STRING,
28 | failureMessage STRING,
29 | failuresJson STRING,
30 |
31 | remoteClientAddress STRING,
32 | clientUser STRING,
33 | userAgent STRING ,
34 | source STRING
35 | )
36 | PARTITIONED BY (
37 | `ymd` INTEGER
38 | )
39 | ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
40 | ;
41 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/AuditLogPlugin.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.facebook.presto.spi.Plugin;
17 | import com.facebook.presto.spi.eventlistener.EventListenerFactory;
18 | import com.google.common.collect.ImmutableList;
19 |
20 | public class AuditLogPlugin
21 | implements Plugin
22 | {
23 | @Override
24 | public Iterable getEventListenerFactories()
25 | {
26 | return ImmutableList.builder()
27 | .add(new AuditLogFactory())
28 | .build();
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/QueryIOMetadataSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryIOMetadata;
17 | import com.fasterxml.jackson.core.JsonGenerator;
18 | import com.fasterxml.jackson.databind.SerializerProvider;
19 | import com.fasterxml.jackson.databind.ser.std.StdSerializer;
20 |
21 | import java.io.IOException;
22 |
23 | public class QueryIOMetadataSerializer extends StdSerializer
24 | {
25 | public QueryIOMetadataSerializer()
26 | {
27 | this(null);
28 | }
29 |
30 | public QueryIOMetadataSerializer(Class t)
31 | {
32 | super(t);
33 | }
34 |
35 | @Override
36 | public void serialize(
37 | QueryIOMetadata value, JsonGenerator jsonGenerator, SerializerProvider provider)
38 | throws IOException
39 | {
40 | jsonGenerator.writeStartObject();
41 | jsonGenerator.writeObjectField("inputs", value.getInputs());
42 | jsonGenerator.writeObjectField("output", value.getOutput());
43 | jsonGenerator.writeEndObject();
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/QueryFailureInfoSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryFailureInfo;
17 | import com.fasterxml.jackson.core.JsonGenerator;
18 | import com.fasterxml.jackson.databind.SerializerProvider;
19 | import com.fasterxml.jackson.databind.ser.std.StdSerializer;
20 |
21 | import java.io.IOException;
22 |
23 | public class QueryFailureInfoSerializer extends StdSerializer
24 | {
25 | public QueryFailureInfoSerializer()
26 | {
27 | this(null);
28 | }
29 |
30 | public QueryFailureInfoSerializer(Class t)
31 | {
32 | super(t);
33 | }
34 |
35 | @Override
36 | public void serialize(
37 | QueryFailureInfo value, JsonGenerator jsonGenerator, SerializerProvider provider)
38 | throws IOException
39 | {
40 | jsonGenerator.writeStartObject();
41 | jsonGenerator.writeObjectField("errorCode", value.getErrorCode());
42 | jsonGenerator.writeObjectField("failureType", value.getFailureType());
43 | jsonGenerator.writeObjectField("failureMessage", value.getFailureMessage());
44 | jsonGenerator.writeObjectField("failureTask", value.getFailureTask());
45 | jsonGenerator.writeObjectField("failureHost", value.getFailureHost());
46 | jsonGenerator.writeObjectField("failuresJson", value.getFailuresJson());
47 | jsonGenerator.writeEndObject();
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/AuditLogFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.facebook.presto.spi.eventlistener.EventListener;
17 | import com.facebook.presto.spi.eventlistener.EventListenerFactory;
18 | import com.google.common.base.Throwables;
19 | import com.google.inject.Injector;
20 | import com.google.inject.Scopes;
21 | import io.airlift.bootstrap.Bootstrap;
22 |
23 | import java.util.Map;
24 |
25 | import static io.airlift.configuration.ConfigBinder.configBinder;
26 | import static java.util.Objects.requireNonNull;
27 |
28 | public class AuditLogFactory
29 | implements EventListenerFactory
30 | {
31 | @Override
32 | public String getName()
33 | {
34 | return "presto-audit-log";
35 | }
36 |
37 | @Override
38 | public EventListener create(Map config)
39 | {
40 | requireNonNull(config, "requiredConfig is null");
41 | try {
42 | Bootstrap app = new Bootstrap(
43 | binder -> {
44 | configBinder(binder).bindConfig(AuditConfig.class);
45 | binder.bind(AuditLogListener.class).in(Scopes.SINGLETON);
46 | });
47 |
48 | Injector injector = app
49 | .strictConfig()
50 | .doNotInitializeLogging()
51 | .setRequiredConfigurationProperties(config)
52 | .initialize();
53 |
54 | return injector.getInstance(AuditLogListener.class);
55 | }
56 | catch (Exception e) {
57 | throw Throwables.propagate(e);
58 | }
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/QueryCompletedEventSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
17 | import com.fasterxml.jackson.core.JsonGenerator;
18 | import com.fasterxml.jackson.databind.SerializerProvider;
19 | import com.fasterxml.jackson.databind.ser.std.StdSerializer;
20 |
21 | import java.io.IOException;
22 |
23 | public class QueryCompletedEventSerializer extends StdSerializer
24 | {
25 | public QueryCompletedEventSerializer()
26 | {
27 | this(null);
28 | }
29 |
30 | public QueryCompletedEventSerializer(Class t)
31 | {
32 | super(t);
33 | }
34 |
35 | @Override
36 | public void serialize(
37 | QueryCompletedEvent value, JsonGenerator jsonGenerator, SerializerProvider provider)
38 | throws IOException
39 | {
40 | jsonGenerator.writeStartObject();
41 | jsonGenerator.writeObjectField("metadata", value.getMetadata());
42 | jsonGenerator.writeObjectField("statistics", value.getStatistics());
43 | jsonGenerator.writeObjectField("context", value.getContext());
44 | jsonGenerator.writeObjectField("ioMetadata", value.getIoMetadata());
45 | jsonGenerator.writeObjectField("failureInfo", value.getFailureInfo());
46 | jsonGenerator.writeObjectField("createTime", value.getCreateTime());
47 | jsonGenerator.writeObjectField("executionStartTime", value.getExecutionStartTime());
48 | jsonGenerator.writeObjectField("endTime", value.getEndTime());
49 | jsonGenerator.writeEndObject();
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/serializer/TestSimpleLogSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import jp.co.yahoo.presto.audit.TestHelper;
17 | import org.testng.annotations.Test;
18 |
19 | import static org.assertj.core.api.Assertions.assertThat;
20 |
21 | @Test(singleThreaded = true)
22 | public class TestSimpleLogSerializer
23 | {
24 | private SimpleLogSerializer simpleLogSerializer = new SimpleLogSerializer();
25 | private TestHelper testHelper = new TestHelper();
26 |
27 | @Test
28 | public void testSerializeNormal()
29 | {
30 | SerializedLog record = simpleLogSerializer.serialize(testHelper.createNormalEvent());
31 | assertThat(record.getQueryId())
32 | .matches("20170606_044544_00024_nfhe3");
33 | assertThat(record.getSerializedLog())
34 | .contains("\"queryId\":\"20170606_044544_00024_nfhe3\"")
35 | .contains("\"query\":\"select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5\"")
36 | .contains("\"userAgent\":\"StatementClient 0.167\"")
37 | .contains("\"source\":\"presto-cli\"")
38 | .contains("\"errorCode\":0")
39 | .contains("\"createTimestamp\":1.5000804E9")
40 | .contains("\"executionStartTimestamp\":1.500080401E9")
41 | .contains("\"endTimestamp\":1.500080403E9")
42 | .contains("\"eventType\":\"QueryCompletedEvent\"")
43 | .doesNotContain("\"failureMessage\"")
44 | .doesNotContain("\"errorName\"");
45 | }
46 |
47 | @Test
48 | public void testSerializeFailure()
49 | {
50 | SerializedLog record = simpleLogSerializer.serialize(testHelper.createFailureEvent());
51 | assertThat(record.getSerializedLog())
52 | .contains("\"errorCode\":1")
53 | .contains("\"errorName\":\"SYNTAX_ERROR\"")
54 | .contains("\"failureMessage\":\"line 1:15: mismatched input '0' expecting ')'\"")
55 | .contains("\"failureType\":\"com.facebook.presto.sql.parser.ParsingException\"");
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/FullLogSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
17 | import com.facebook.presto.spi.eventlistener.QueryFailureInfo;
18 | import com.facebook.presto.spi.eventlistener.QueryIOMetadata;
19 | import com.facebook.presto.spi.eventlistener.QueryStatistics;
20 | import com.fasterxml.jackson.core.JsonProcessingException;
21 | import com.fasterxml.jackson.databind.ObjectMapper;
22 | import com.fasterxml.jackson.databind.module.SimpleModule;
23 | import io.airlift.json.ObjectMapperProvider;
24 |
25 | import java.util.Optional;
26 | import java.util.regex.Matcher;
27 | import java.util.regex.Pattern;
28 |
29 | public class FullLogSerializer
30 | implements LogSerializer
31 | {
32 | private final ObjectMapper objectMapper;
33 | private Pattern filter;
34 |
35 | public FullLogSerializer(Optional auditLogFullFilter)
36 | {
37 | filter = auditLogFullFilter.map(Pattern::compile).orElse(null);
38 | // Initialize serializer and objectMapper
39 | SimpleModule serializerModule = new SimpleModule("presto-audit-serializer");
40 | serializerModule.addSerializer(QueryCompletedEvent.class, new QueryCompletedEventSerializer());
41 | serializerModule.addSerializer(QueryStatistics.class, new QueryStatisticsSerializer());
42 | serializerModule.addSerializer(QueryIOMetadata.class, new QueryIOMetadataSerializer());
43 | serializerModule.addSerializer(QueryFailureInfo.class, new QueryFailureInfoSerializer());
44 | objectMapper = new ObjectMapperProvider().get().registerModule(serializerModule);
45 | }
46 |
47 | @Override
48 | public SerializedLog serialize(QueryCompletedEvent event) throws JsonProcessingException
49 | {
50 | return new SerializedLog(event.getMetadata().getQueryId(), objectMapper.writeValueAsString(event));
51 | }
52 |
53 | @Override
54 | public boolean shouldOutput(QueryCompletedEvent event)
55 | {
56 | // Don't log if regex matches source name
57 | if (filter != null && event.getContext().getSource().isPresent()) {
58 | String source = event.getContext().getSource().get();
59 | Matcher matcher = filter.matcher(source);
60 | return !matcher.find();
61 | }
62 | return true;
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/QueryStatisticsSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryStatistics;
17 | import com.fasterxml.jackson.core.JsonGenerator;
18 | import com.fasterxml.jackson.databind.SerializerProvider;
19 | import com.fasterxml.jackson.databind.ser.std.StdSerializer;
20 |
21 | import java.io.IOException;
22 |
23 | public class QueryStatisticsSerializer extends StdSerializer
24 | {
25 | public QueryStatisticsSerializer()
26 | {
27 | this(null);
28 | }
29 |
30 | public QueryStatisticsSerializer(Class t)
31 | {
32 | super(t);
33 | }
34 |
35 | @Override
36 | public void serialize(
37 | QueryStatistics value, JsonGenerator jsonGenerator, SerializerProvider provider)
38 | throws IOException
39 | {
40 | jsonGenerator.writeStartObject();
41 | jsonGenerator.writeNumberField("cpuTime", value.getCpuTime().toMillis());
42 | jsonGenerator.writeNumberField("wallTime", value.getWallTime().toMillis());
43 | jsonGenerator.writeNumberField("queuedTime", value.getQueuedTime().toMillis());
44 | if (value.getAnalysisTime().isPresent()) {
45 | jsonGenerator.writeObjectField("analysisTime", value.getAnalysisTime().get().toMillis());
46 | }
47 | if (value.getDistributedPlanningTime().isPresent()) {
48 | jsonGenerator.writeObjectField("distributedPlanningTime", value.getDistributedPlanningTime().get().toMillis());
49 | }
50 | jsonGenerator.writeObjectField("peakUserMemoryBytes", value.getPeakUserMemoryBytes());
51 | jsonGenerator.writeObjectField("peakTotalNonRevocableMemoryBytes", value.getPeakTotalNonRevocableMemoryBytes());
52 | jsonGenerator.writeObjectField("totalBytes", value.getTotalBytes());
53 | jsonGenerator.writeObjectField("totalRows", value.getTotalRows());
54 | jsonGenerator.writeObjectField("outputBytes", value.getOutputBytes());
55 | jsonGenerator.writeObjectField("outputRows", value.getOutputRows());
56 | jsonGenerator.writeObjectField("writtenBytes", value.getWrittenBytes());
57 | jsonGenerator.writeObjectField("writtenRows", value.getWrittenRows());
58 | jsonGenerator.writeObjectField("cumulativeMemory", value.getCumulativeMemory());
59 | jsonGenerator.writeObjectField("completedSplits", value.getCompletedSplits());
60 | jsonGenerator.writeObjectField("isComplete", value.isComplete());
61 | jsonGenerator.writeObjectField("cpuTimeDistribution", value.getCpuTimeDistribution());
62 | jsonGenerator.writeObjectField("operatorSummaries", value.getOperatorSummaries());
63 | jsonGenerator.writeObjectField("stageGcStatistics", value.getStageGcStatistics());
64 | jsonGenerator.writeObjectField("peakTaskTotalMemory", value.peakTaskTotalMemory());
65 | jsonGenerator.writeEndObject();
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/pulsar/TestPulsarProducer.java:
--------------------------------------------------------------------------------
1 | package jp.co.yahoo.presto.audit.pulsar;
2 |
3 | import jp.co.yahoo.presto.audit.serializer.SerializedLog;
4 | import org.apache.pulsar.client.api.ClientConfiguration;
5 | import org.apache.pulsar.client.api.CompressionType;
6 | import org.apache.pulsar.client.api.Producer;
7 | import org.apache.pulsar.client.api.ProducerConfiguration;
8 | import org.apache.pulsar.client.api.PulsarClientException;
9 | import org.apache.pulsar.client.impl.auth.AuthenticationAthenz;
10 | import org.testng.annotations.Test;
11 |
12 | import java.util.HashMap;
13 | import java.util.Map;
14 | import java.util.concurrent.TimeUnit;
15 |
16 | import static org.mockito.ArgumentMatchers.any;
17 | import static org.mockito.Mockito.mock;
18 | import static org.mockito.Mockito.verify;
19 | import static org.mockito.Mockito.when;
20 |
21 | public class TestPulsarProducer
22 | {
23 | @Test
24 | public void TestClientConfigurationBuilder()
25 | throws PulsarClientException.UnsupportedAuthenticationException
26 | {
27 | Map authParams = new HashMap<>();
28 | ClientConfiguration mockConf = mock(ClientConfiguration.class);
29 | ClientConfiguration conf = new PulsarProducer.Builder()
30 | .setUseTLS(false)
31 | .setTrustCerts("/path/to/trusted_cert")
32 | .setAuthParams(authParams)
33 | .buildClientConfiguration(mockConf);
34 |
35 | assert(mockConf == conf);
36 | verify(mockConf).setUseTls(false);
37 | verify(mockConf).setTlsTrustCertsFilePath("/path/to/trusted_cert");
38 | verify(mockConf).setAuthentication(AuthenticationAthenz.class.getName(), authParams);
39 | }
40 |
41 | @Test
42 | public void TestProducerConfigurationBuilderDefault()
43 | {
44 | ProducerConfiguration mockConf = mock(ProducerConfiguration.class);
45 | ProducerConfiguration conf = new PulsarProducer.Builder()
46 | .buildProducerConfiguration(mockConf);
47 |
48 | assert(mockConf == conf);
49 | verify(mockConf).setSendTimeout(3, TimeUnit.SECONDS);
50 | }
51 |
52 | @Test
53 | public void TestProducerConfigurationBuilderConfig()
54 | {
55 | ProducerConfiguration mockConf = mock(ProducerConfiguration.class);
56 | ProducerConfiguration conf = new PulsarProducer.Builder()
57 | .setSendTimeout(30, TimeUnit.MILLISECONDS)
58 | .setCompressionType(CompressionType.ZLIB)
59 | .buildProducerConfiguration(mockConf);
60 |
61 | assert(mockConf == conf);
62 | verify(mockConf).setSendTimeout(30, TimeUnit.MILLISECONDS);
63 | verify(mockConf).setCompressionType(CompressionType.ZLIB);
64 | }
65 |
66 | @Test(expectedExceptions = PulsarClientException.class)
67 | public void TestBuilderFail()
68 | throws PulsarClientException
69 | {
70 | new PulsarProducer.Builder()
71 | .setURL("pulsar+ssl://pulsar.cluster.com:6651")
72 | .setTopic("persistent://namespace/global/test/topic1").build();
73 | }
74 |
75 | @Test
76 | public void TestPulsarProducer()
77 | {
78 | Producer producer = mock(Producer.class);
79 | PulsarProducer pulsarProducer = new PulsarProducer(producer);
80 | pulsarProducer.send(new SerializedLog("queryID", "{\"a\":\"b\"}"));
81 | }
82 |
83 | @Test
84 | public void TestPulsarProducerSendError()
85 | throws PulsarClientException
86 | {
87 | Producer producer = mock(Producer.class);
88 | when(producer.send(any(byte[].class)))
89 | .thenThrow(new PulsarClientException("Mock IO Exception"));
90 | PulsarProducer pulsarProducer = new PulsarProducer(producer);
91 | pulsarProducer.send(new SerializedLog("being_dropped_queryID", "{\"a\":\"b\"}"));
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
16 | # ⛔️ THIS REPOSITORY IS DEPRECATED ⛔️
17 | This repository is no longer maintained. Please refer to the following developer guides:
18 | * https://prestodb.io/docs/current/develop/event-listener.html
19 | * https://trino.io/docs/current/develop/event-listener.html
20 |
21 |
22 | # Presto Query Audit Log(log history)
23 | Plugin for Presto to save queries and metrics into files.
24 |
25 | [](https://travis-ci.org/yahoojapan/presto-audit)
26 |
27 | ## Requirement
28 | * Presto 0.193 or later
29 |
30 | # Build
31 |
32 | ```bash
33 | % ./mvnw clean compile package
34 | ```
35 |
36 | ## packaging - rpm
37 |
38 | ```bash
39 | % ./mvnw clean compile package rpm:rpm
40 | ```
41 |
42 | # Usage
43 |
44 | ## install
45 |
46 | Copy jar file to target directory.
47 |
48 | ```bash
49 | % export PLUGIN_VER=1.0
50 | % cp target/presto-audit-plugin-${PLUGIN_VER}-SNAPSHOT-all.jar /usr/lib/presto/lib/plugin/yj-audit/
51 | ```
52 |
53 | ## run
54 | create a event-listener.properties file under /etc/presto/ .
55 |
56 | eg.
57 | /etc/presto/event-listener.properties
58 | ```text
59 | event-listener.name=presto-audit-log
60 | event-listener.audit-log-path=/var/log/presto/
61 | event-listener.audit-log-filename=presto-auditlog.log
62 | ```
63 |
64 | ## full log
65 | To enable full log, add the following line to the file
66 | ```text
67 | event-listener.audit-log-full-filename=presto-auditlog-full.log
68 | ```
69 |
70 | To add a filter for full log, add the following line to the file.
71 | The filter should be a regex. If the filter match with `--source` of a query, this query will not be
72 | logged in full log.
73 | ```text
74 | # Format:
75 | event-listener.audit-log-full-filter=
76 |
77 | # Example:
78 | event-listener.audit-log-full-filter=SRE_SYSTEM
79 | event-listener.audit-log-full-filter=(SRE_SYSTEM|Presto-team)
80 | ```
81 |
82 | ## Send message to Pulsar broker
83 | Send the log to Pulsar broker immediately after a query is completed.
84 | Config:
85 | ```text
86 | event-listener.pulsar.simple-log-topic=persistent://namespace/global/test/topic1
87 | event-listener.pulsar.full-log-topic=persistent://namespace/global/test/topic2
88 | event-listener.pulsar.pulsar-url=pulsar+ssl://pulsar.cluster.com:6651
89 | event-listener.pulsar.pulsar-cert-path=/etc/pki/tls/certs/ca-bundle.crt
90 | event-listener.pulsar.use-TLS=true # Optional
91 | event-listener.pulsar.compression-type=ZLIB # Optional (default: NONE)
92 | event-listener.athenz.private-key-path=/etc/presto/athenz/private.key
93 | event-listener.athenz.tenant-domain=tenant.pulsar.tenant
94 | event-listener.athenz.tenant-service=mq
95 | event-listener.athenz.provider-domain=provider.plusar.tenant
96 | event-listener.athenz.config-path=/usr/local/etc/pulsar-athenz-config/athenz.conf
97 | event-listener.athenz.principal-header=Athenz-Principal-Auth
98 | event-listener.athenz.role-header=Athenz-Role-Auth
99 | ```
100 |
101 | ## Analyze SQL samples
102 | Table DDL can be found in src/sql/ddl.sql
103 | ```sql
104 | SELECT
105 | date_parse(executionStartTime, '%Y%m%d%H%i%S.%f') AS EXEC_START,
106 | state,
107 | date_diff('millisecond', date_parse(executionStartTime, '%Y%m%d%H%i%S.%f'), date_parse(endTime, '%Y%m%d%H%i%S.%f')) AS PRESTO_EXEC_TIME_MS
108 | FROM
109 | presto_audit
110 | WHERE
111 | ymd = '20170912' AND createTime > '20170912233000.572'
112 | ORDER BY
113 | createTime;
114 | ```
115 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/SimpleLogSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
17 | import com.facebook.presto.spi.eventlistener.QueryFailureInfo;
18 | import com.google.gson.Gson;
19 | import com.google.gson.GsonBuilder;
20 |
21 | import java.time.ZoneId;
22 | import java.time.format.DateTimeFormatter;
23 |
24 | public class SimpleLogSerializer
25 | implements LogSerializer
26 | {
27 | Gson gson;
28 | public SimpleLogSerializer()
29 | {
30 | gson = new GsonBuilder().disableHtmlEscaping().create();
31 | }
32 |
33 | @Override
34 | public SerializedLog serialize(QueryCompletedEvent event)
35 | {
36 | AuditRecord record = buildAuditRecord(event);
37 | return new SerializedLog(event.getMetadata().getQueryId(), gson.toJson(record));
38 | }
39 |
40 | AuditRecord buildAuditRecord(QueryCompletedEvent event)
41 | {
42 | DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss.SSS").withZone(ZoneId.systemDefault());
43 |
44 | AuditRecord record = new AuditRecord();
45 | record.setEventType("QueryCompletedEvent");
46 | record.setQueryId(event.getMetadata().getQueryId());
47 |
48 | //SQL Query Text
49 | record.setQuery(event.getMetadata().getQuery());
50 | record.setUri(event.getMetadata().getUri().toString());
51 | record.setState(event.getMetadata().getQueryState());
52 |
53 | record.setCpuTime(event.getStatistics().getCpuTime().toMillis() / 1000.0);
54 | record.setWallTime(event.getStatistics().getWallTime().toMillis() / 1000.0);
55 | record.setQueuedTime(event.getStatistics().getQueuedTime().toMillis() / 1000.0);
56 | record.setPeakUserMemoryBytes(event.getStatistics().getPeakUserMemoryBytes());
57 | record.setPeakTotalNonRevocableMemoryBytes(event.getStatistics().getPeakTotalNonRevocableMemoryBytes());
58 | record.setTotalBytes(event.getStatistics().getTotalBytes());
59 | record.setTotalRows(event.getStatistics().getTotalRows());
60 | record.setCompletedSplits(event.getStatistics().getCompletedSplits());
61 |
62 | record.setCreateTimestamp(event.getCreateTime().toEpochMilli() / 1000.0);
63 | record.setExecutionStartTimestamp(event.getExecutionStartTime().toEpochMilli() / 1000.0);
64 | record.setEndTimestamp(event.getEndTime().toEpochMilli() / 1000.0);
65 |
66 | // Error information
67 | if (event.getFailureInfo().isPresent()) {
68 | QueryFailureInfo failureInfo = event.getFailureInfo().get();
69 | record.setErrorCode(failureInfo.getErrorCode().getCode());
70 | record.setErrorName(failureInfo.getErrorCode().getName());
71 | if (failureInfo.getFailureType().isPresent()) {
72 | record.setFailureType(failureInfo.getFailureType().get());
73 | }
74 | if (failureInfo.getFailureMessage().isPresent()) {
75 | record.setFailureMessage(failureInfo.getFailureMessage().get());
76 | }
77 | record.setFailuresJson(failureInfo.getFailuresJson());
78 | }
79 |
80 | record.setRemoteClientAddress(event.getContext().getRemoteClientAddress().orElse(""));
81 | record.setClientUser(event.getContext().getUser());
82 | record.setUserAgent(event.getContext().getUserAgent().orElse(""));
83 | record.setSource(event.getContext().getSource().orElse(""));
84 | return record;
85 | }
86 |
87 | @Override
88 | public boolean shouldOutput(QueryCompletedEvent event)
89 | {
90 | return true;
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/TestAuditConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.google.common.collect.ImmutableMap;
17 | import org.apache.pulsar.client.api.CompressionType;
18 | import org.testng.annotations.Test;
19 |
20 | import java.util.Map;
21 |
22 | import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
23 | import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults;
24 | import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults;
25 |
26 | public class TestAuditConfig
27 | {
28 | @Test
29 | public void testDefault()
30 | {
31 | assertRecordedDefaults(recordDefaults(AuditConfig.class)
32 | .setAuditLogPath(null)
33 | .setAuditSimpleLogName(null)
34 | .setAuditFullLogName(null)
35 | .setSimpleTopic(null)
36 | .setFullTopic(null)
37 | .setPrivateKeyPath(null)
38 | .setPulsarUrl(null)
39 | .setTrustCertsPath(null)
40 | .setLogFilter(null)
41 | .setTenantDomain(null)
42 | .setTenantService(null)
43 | .setProviderDomain(null)
44 | .setAthenzConfPath(null)
45 | .setPrincipalHeader(null)
46 | .setRoleHeader(null)
47 | .setUseTLS(true)
48 | .setCompressionType(CompressionType.NONE));
49 | }
50 |
51 | @Test
52 | public void testExplicitConfig()
53 | {
54 | Map properties = new ImmutableMap.Builder()
55 | .put("event-listener.audit-log-path", "/var/log/presto/")
56 | .put("event-listener.audit-log-filename", "presto-audit.log")
57 | .put("event-listener.audit-log-full-filename", "presto-auditlog-full.log")
58 | .put("event-listener.audit-log-full-filter", "DROP")
59 | .put("event-listener.pulsar.simple-log-topic", "persistent://namespace/global/test/topic1")
60 | .put("event-listener.pulsar.full-log-topic", "persistent://namespace/global/test/topic2")
61 | .put("event-listener.pulsar.pulsar-url", "pulsar+ssl://pulsar.cluster.com:6651")
62 | .put("event-listener.pulsar.pulsar-cert-path", "/etc/pki/tls/certs/ca-bundle.crt")
63 | .put("event-listener.pulsar.use-TLS", "false")
64 | .put("event-listener.pulsar.compression-type", "ZLIB")
65 | .put("event-listener.athenz.private-key-path", "/etc/presto/athenz/private.key")
66 | .put("event-listener.athenz.tenant-domain", "tenant.pulsar.tenant")
67 | .put("event-listener.athenz.tenant-service", "mq")
68 | .put("event-listener.athenz.provider-domain", "provider.plusar.tenant")
69 | .put("event-listener.athenz.config-path", "/usr/local/etc/pulsar-athenz-config/athenz.conf")
70 | .put("event-listener.athenz.principal-header", "Athenz-Principal-Auth")
71 | .put("event-listener.athenz.role-header", "Athenz-Role-Auth")
72 | .build();
73 |
74 | AuditConfig expected = new AuditConfig()
75 | .setAuditLogPath("/var/log/presto/")
76 | .setAuditSimpleLogName("presto-audit.log")
77 | .setAuditFullLogName("presto-auditlog-full.log")
78 | .setSimpleTopic("persistent://namespace/global/test/topic1")
79 | .setFullTopic("persistent://namespace/global/test/topic2")
80 | .setPrivateKeyPath("/etc/presto/athenz/private.key")
81 | .setPulsarUrl("pulsar+ssl://pulsar.cluster.com:6651")
82 | .setUseTLS(false)
83 | .setCompressionType(CompressionType.ZLIB)
84 | .setTrustCertsPath("/etc/pki/tls/certs/ca-bundle.crt")
85 | .setLogFilter("DROP")
86 | .setTenantDomain("tenant.pulsar.tenant")
87 | .setTenantService("mq")
88 | .setProviderDomain("provider.plusar.tenant")
89 | .setAthenzConfPath("/usr/local/etc/pulsar-athenz-config/athenz.conf")
90 | .setPrincipalHeader("Athenz-Principal-Auth")
91 | .setRoleHeader("Athenz-Role-Auth");
92 |
93 | assertFullMapping(properties, expected);
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/.mvn/wrapper/MavenWrapperDownloader.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | import java.net.*;
21 | import java.io.*;
22 | import java.nio.channels.*;
23 | import java.util.Properties;
24 |
25 | public class MavenWrapperDownloader {
26 |
27 | /**
28 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
29 | */
30 | private static final String DEFAULT_DOWNLOAD_URL =
31 | "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar";
32 |
33 | /**
34 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
35 | * use instead of the default one.
36 | */
37 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
38 | ".mvn/wrapper/maven-wrapper.properties";
39 |
40 | /**
41 | * Path where the maven-wrapper.jar will be saved to.
42 | */
43 | private static final String MAVEN_WRAPPER_JAR_PATH =
44 | ".mvn/wrapper/maven-wrapper.jar";
45 |
46 | /**
47 | * Name of the property which should be used to override the default download url for the wrapper.
48 | */
49 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
50 |
51 | public static void main(String args[]) {
52 | System.out.println("- Downloader started");
53 | File baseDirectory = new File(args[0]);
54 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
55 |
56 | // If the maven-wrapper.properties exists, read it and check if it contains a custom
57 | // wrapperUrl parameter.
58 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
59 | String url = DEFAULT_DOWNLOAD_URL;
60 | if(mavenWrapperPropertyFile.exists()) {
61 | FileInputStream mavenWrapperPropertyFileInputStream = null;
62 | try {
63 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
64 | Properties mavenWrapperProperties = new Properties();
65 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
66 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
67 | } catch (IOException e) {
68 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
69 | } finally {
70 | try {
71 | if(mavenWrapperPropertyFileInputStream != null) {
72 | mavenWrapperPropertyFileInputStream.close();
73 | }
74 | } catch (IOException e) {
75 | // Ignore ...
76 | }
77 | }
78 | }
79 | System.out.println("- Downloading from: : " + url);
80 |
81 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
82 | if(!outputFile.getParentFile().exists()) {
83 | if(!outputFile.getParentFile().mkdirs()) {
84 | System.out.println(
85 | "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'");
86 | }
87 | }
88 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
89 | try {
90 | downloadFileFromURL(url, outputFile);
91 | System.out.println("Done");
92 | System.exit(0);
93 | } catch (Throwable e) {
94 | System.out.println("- Error downloading");
95 | e.printStackTrace();
96 | System.exit(1);
97 | }
98 | }
99 |
100 | private static void downloadFileFromURL(String urlString, File destination) throws Exception {
101 | URL website = new URL(urlString);
102 | ReadableByteChannel rbc;
103 | rbc = Channels.newChannel(website.openStream());
104 | FileOutputStream fos = new FileOutputStream(destination);
105 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
106 | fos.close();
107 | rbc.close();
108 | }
109 |
110 | }
111 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/serializer/TestFullLogSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | /*
15 | * Licensed under the Apache License, Version 2.0 (the "License");
16 | * you may not use this file except in compliance with the License.
17 | * You may obtain a copy of the License at
18 | *
19 | * http://www.apache.org/licenses/LICENSE-2.0
20 | *
21 | * Unless required by applicable law or agreed to in writing, software
22 | * distributed under the License is distributed on an "AS IS" BASIS,
23 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24 | * See the License for the specific language governing permissions and
25 | * limitations under the License.
26 | */
27 | package jp.co.yahoo.presto.audit.serializer;
28 |
29 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
30 | import com.fasterxml.jackson.core.JsonProcessingException;
31 | import jp.co.yahoo.presto.audit.TestHelper;
32 | import org.testng.annotations.DataProvider;
33 | import org.testng.annotations.Test;
34 |
35 | import java.util.Optional;
36 |
37 | import static org.assertj.core.api.Assertions.assertThat;
38 | import static org.testng.Assert.assertEquals;
39 |
40 | @Test(singleThreaded = true)
41 | public class TestFullLogSerializer
42 | {
43 | private FullLogSerializer fullLogSerializer = new FullLogSerializer(Optional.empty());
44 | private TestHelper testHelper = new TestHelper();
45 |
46 | @Test
47 | public void testSerializeNormal() throws JsonProcessingException
48 | {
49 | SerializedLog record = fullLogSerializer.serialize(testHelper.createNormalEvent());
50 | assertThat(record.getQueryId())
51 | .matches("20170606_044544_00024_nfhe3");
52 | assertThat(record.getSerializedLog())
53 | .contains("\"queryId\":\"20170606_044544_00024_nfhe3\"")
54 | .contains("\"query\":\"select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5\"")
55 | .contains("\"userAgent\":\"StatementClient 0.167\"")
56 | .contains("\"source\":\"presto-cli\"")
57 | .contains("\"createTime\":\"2017-07-15T01:00:00Z\"")
58 | .contains("\"executionStartTime\":\"2017-07-15T01:00:01Z\"")
59 | .doesNotContain("\"failureMessage\"")
60 | .doesNotContain("\"code\"");
61 | }
62 |
63 | @Test
64 | public void testSerializeFailure() throws JsonProcessingException
65 | {
66 | SerializedLog record = fullLogSerializer.serialize(testHelper.createFailureEvent());
67 | assertThat(record.getSerializedLog())
68 | .contains("\"failureInfo\"")
69 | .contains("\"code\":1")
70 | .contains("\"name\":\"SYNTAX_ERROR\"")
71 | .contains("\"failureMessage\":\"line 1:15: mismatched input '0' expecting ')'\"")
72 | .contains("\"failureType\":\"com.facebook.presto.sql.parser.ParsingException\"");
73 | }
74 |
75 | @DataProvider(name="filter-provider")
76 | public Object[][] testData(){
77 | String filter1 = "sre_system";
78 | String filter2 = "(sre_system|presto-cli)";
79 | return new Object[][]{
80 | // No filter
81 | {Optional.empty(), Optional.empty(), true},
82 | {Optional.empty(), Optional.of("presto-cli"), true},
83 | {Optional.empty(), Optional.of("sre_system"), true},
84 | // Filter1
85 | {Optional.of(filter1), Optional.empty(), true},
86 | {Optional.of(filter1), Optional.of("presto-cli"), true},
87 | {Optional.of(filter1), Optional.of("sre_system"), false},
88 | // Filter2
89 | {Optional.of(filter2), Optional.empty(), true},
90 | {Optional.of(filter2), Optional.of("presto-cli"), false},
91 | {Optional.of(filter2), Optional.of("sre_system"), false}
92 | };
93 | }
94 |
95 | @Test(dataProvider = "filter-provider")
96 | public void testShouldOutput(Optional filter, Optional source, boolean result) {
97 | FullLogSerializer serializer = new FullLogSerializer(filter);
98 | QueryCompletedEvent event = testHelper.createQueryWithSource(source);
99 | boolean shouldOutput = serializer.shouldOutput(event);
100 | assertEquals(shouldOutput, result);
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/pulsar/PulsarProducer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.pulsar;
15 |
16 | import com.google.common.annotations.VisibleForTesting;
17 | import io.airlift.log.Logger;
18 | import jp.co.yahoo.presto.audit.serializer.SerializedLog;
19 | import org.apache.pulsar.client.api.ClientConfiguration;
20 | import org.apache.pulsar.client.api.CompressionType;
21 | import org.apache.pulsar.client.api.Producer;
22 | import org.apache.pulsar.client.api.ProducerConfiguration;
23 | import org.apache.pulsar.client.api.PulsarClient;
24 | import org.apache.pulsar.client.api.PulsarClientException;
25 | import org.apache.pulsar.client.impl.auth.AuthenticationAthenz;
26 |
27 | import java.util.Map;
28 | import java.util.concurrent.TimeUnit;
29 |
30 | public class PulsarProducer
31 | {
32 | private static final Logger log = Logger.get(PulsarProducer.class);
33 | private Producer producer;
34 |
35 | @VisibleForTesting
36 | public PulsarProducer(Producer producer)
37 | {
38 | this.producer = producer;
39 | }
40 |
41 | public void send(SerializedLog message)
42 | {
43 | try {
44 | producer.send(message.getSerializedLog().getBytes());
45 | }
46 | catch (PulsarClientException e) {
47 | log.error("Failed to send message to Pulsar broker. " + e);
48 | log.error("Dropped queryID: " + message.getQueryId());
49 | }
50 | }
51 |
52 | public static class Builder
53 | {
54 | private String topic;
55 | private String url;
56 | private String trustCerts;
57 | private Map authParams;
58 | private boolean useTLS;
59 | private int sendTimeout = 3;
60 | private TimeUnit sendTimeoutUnit = TimeUnit.SECONDS;
61 | private CompressionType compressionType = CompressionType.NONE;
62 |
63 | public Builder setTopic(String topic)
64 | {
65 | this.topic = topic;
66 | return this;
67 | }
68 |
69 | public Builder setURL(String url)
70 | {
71 | this.url = url;
72 | return this;
73 | }
74 |
75 | public Builder setTrustCerts(String trustCerts)
76 | {
77 | this.trustCerts = trustCerts;
78 | return this;
79 | }
80 |
81 | public Builder setAuthParams(Map authParams)
82 | {
83 | this.authParams = authParams;
84 | return this;
85 | }
86 |
87 | public Builder setUseTLS(boolean useTLS)
88 | {
89 | this.useTLS = useTLS;
90 | return this;
91 | }
92 |
93 | public Builder setSendTimeout(int sendTimeout, TimeUnit timeUnit)
94 | {
95 | this.sendTimeout = sendTimeout;
96 | this.sendTimeoutUnit = timeUnit;
97 | return this;
98 | }
99 |
100 | public Builder setCompressionType(CompressionType compressionType)
101 | {
102 | this.compressionType = compressionType;
103 | return this;
104 | }
105 |
106 | @VisibleForTesting
107 | ClientConfiguration buildClientConfiguration(ClientConfiguration conf)
108 | throws PulsarClientException.UnsupportedAuthenticationException
109 | {
110 | // Config PulsarClient
111 | conf.setUseTls(useTLS);
112 | conf.setTlsTrustCertsFilePath(trustCerts);
113 | conf.setAuthentication(AuthenticationAthenz.class.getName(), authParams);
114 | return conf;
115 | }
116 |
117 | @VisibleForTesting
118 | ProducerConfiguration buildProducerConfiguration(ProducerConfiguration prodConf)
119 | {
120 | prodConf.setSendTimeout(sendTimeout, sendTimeoutUnit);
121 | prodConf.setCompressionType(compressionType);
122 | return prodConf;
123 | }
124 |
125 | public PulsarProducer build()
126 | throws PulsarClientException
127 | {
128 | ClientConfiguration conf = buildClientConfiguration(new ClientConfiguration());
129 | ProducerConfiguration prodConf = buildProducerConfiguration(new ProducerConfiguration());
130 | PulsarClient pulsarClient = PulsarClient.create(url, conf);
131 | Producer producer = pulsarClient.createProducer(topic, prodConf);
132 | return new PulsarProducer(producer);
133 | }
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/AuditLogListener.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.facebook.presto.spi.eventlistener.EventListener;
17 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
18 | import com.facebook.presto.spi.eventlistener.QueryCreatedEvent;
19 | import com.fasterxml.jackson.core.JsonProcessingException;
20 | import io.airlift.log.Logger;
21 | import jp.co.yahoo.presto.audit.pulsar.PulsarProducer;
22 | import jp.co.yahoo.presto.audit.serializer.FullLogSerializer;
23 | import jp.co.yahoo.presto.audit.serializer.SerializedLog;
24 | import jp.co.yahoo.presto.audit.serializer.SimpleLogSerializer;
25 | import org.apache.pulsar.client.api.PulsarClientException;
26 |
27 | import javax.inject.Inject;
28 |
29 | import java.io.File;
30 | import java.util.HashMap;
31 | import java.util.Map;
32 | import java.util.Optional;
33 |
34 | import static java.util.Objects.requireNonNull;
35 |
36 | public class AuditLogListener
37 | implements EventListener
38 | {
39 | private static final Logger log = Logger.get(AuditLogListener.class);
40 | private final AuditLogFileWriter auditLogWriter;
41 | private final String simpleLogFilePath;
42 | private final Optional fullLogFilePath;
43 | private final FullLogSerializer fullLogSerializer;
44 | private final SimpleLogSerializer simpleLogSerializer;
45 | private PulsarProducer pulsarSimpleProducer = null;
46 | private PulsarProducer pulsarFullProducer = null;
47 |
48 | @Inject
49 | public AuditLogListener(AuditConfig auditConfig)
50 | throws PulsarClientException
51 | {
52 | this.auditLogWriter = auditConfig.getAuditLogFileWriter();
53 |
54 | // File logger setting
55 | String auditLogPath = requireNonNull(auditConfig.getAuditLogPath(), "auditLogPath is null");
56 | String simpleLogName = requireNonNull(auditConfig.getAuditSimpleLogName(), "simpleLogName is null");
57 | Optional fullLogName = Optional.ofNullable(auditConfig.getAuditFullLogName());
58 | this.simpleLogFilePath = auditLogPath + File.separator + simpleLogName;
59 | this.fullLogFilePath = fullLogName.map(s -> auditLogPath + File.separator + s);
60 | Optional auditLogFullFilter = Optional.ofNullable(auditConfig.getLogFilter());
61 | fullLogSerializer = new FullLogSerializer(auditLogFullFilter);
62 | simpleLogSerializer = new SimpleLogSerializer();
63 |
64 | // Pulsar setting
65 | Optional simpleLogTopic = Optional.ofNullable(auditConfig.getSimpleTopic());
66 | Optional fullLogTopic = Optional.ofNullable(auditConfig.getFullTopic());
67 | if (simpleLogTopic.isPresent() || fullLogTopic.isPresent()) {
68 | Map authParams = new HashMap<>();
69 | authParams.put("tenantDomain", requireNonNull(auditConfig.getTenantDomain()));
70 | authParams.put("tenantService", requireNonNull(auditConfig.getTenantService()));
71 | authParams.put("providerDomain", requireNonNull(auditConfig.getProviderDomain()));
72 | authParams.put("privateKey", requireNonNull(auditConfig.getPrivateKeyPath()));
73 | authParams.put("athenzConfPath", requireNonNull(auditConfig.getAthenzConfPath()));
74 | authParams.put("principalHeader", requireNonNull(auditConfig.getPrincipalHeader()));
75 | authParams.put("roleHeader", requireNonNull(auditConfig.getRoleHeader()));
76 |
77 | PulsarProducer.Builder builder = new PulsarProducer.Builder()
78 | .setURL(requireNonNull(auditConfig.getPulsarUrl()))
79 | .setTrustCerts(requireNonNull(auditConfig.getTrustCertsPath()))
80 | .setAuthParams(authParams)
81 | .setCompressionType(auditConfig.getCompressionType())
82 | .setUseTLS(auditConfig.getUseTLS());
83 |
84 | if (simpleLogTopic.isPresent()) {
85 | pulsarSimpleProducer = builder.setTopic(simpleLogTopic.get()).build();
86 | }
87 |
88 | if (fullLogTopic.isPresent()) {
89 | pulsarFullProducer = builder.setTopic(fullLogTopic.get()).build();
90 | }
91 | }
92 | }
93 |
94 | @Override
95 | public void queryCreated(QueryCreatedEvent queryCreatedEvent)
96 | {
97 | log.debug("QUERY SQL : [ %s ]", queryCreatedEvent.getMetadata().getQuery());
98 | }
99 |
100 | @Override
101 | public void queryCompleted(QueryCompletedEvent queryCompletedEvent)
102 | {
103 | simpleLog(queryCompletedEvent);
104 | fullLog(queryCompletedEvent);
105 | }
106 |
107 | private void simpleLog(QueryCompletedEvent queryCompletedEvent)
108 | {
109 | SerializedLog simpleLog = simpleLogSerializer.serialize(queryCompletedEvent);
110 | auditLogWriter.write(simpleLogFilePath, simpleLog);
111 | if (pulsarSimpleProducer != null) {
112 | pulsarSimpleProducer.send(simpleLog);
113 | }
114 | }
115 |
116 | private void fullLog(QueryCompletedEvent queryCompletedEvent)
117 | {
118 | if (fullLogFilePath.isPresent() && fullLogSerializer.shouldOutput(queryCompletedEvent)) {
119 | try {
120 | SerializedLog fullLog = fullLogSerializer.serialize(queryCompletedEvent);
121 | auditLogWriter.write(fullLogFilePath.get(), fullLog);
122 | if (pulsarFullProducer != null) {
123 | pulsarFullProducer.send(fullLog);
124 | }
125 | }
126 | catch (JsonProcessingException e) {
127 | log.error("Error in serializing full audit log: " + e.getMessage());
128 | log.error("Query failed: " + queryCompletedEvent.getMetadata().getQueryId());
129 | }
130 | }
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/mvnw.cmd:
--------------------------------------------------------------------------------
1 | @REM ----------------------------------------------------------------------------
2 | @REM Licensed to the Apache Software Foundation (ASF) under one
3 | @REM or more contributor license agreements. See the NOTICE file
4 | @REM distributed with this work for additional information
5 | @REM regarding copyright ownership. The ASF licenses this file
6 | @REM to you under the Apache License, Version 2.0 (the
7 | @REM "License"); you may not use this file except in compliance
8 | @REM with the License. You may obtain a copy of the License at
9 | @REM
10 | @REM http://www.apache.org/licenses/LICENSE-2.0
11 | @REM
12 | @REM Unless required by applicable law or agreed to in writing,
13 | @REM software distributed under the License is distributed on an
14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | @REM KIND, either express or implied. See the License for the
16 | @REM specific language governing permissions and limitations
17 | @REM under the License.
18 | @REM ----------------------------------------------------------------------------
19 |
20 | @REM ----------------------------------------------------------------------------
21 | @REM Maven2 Start Up Batch script
22 | @REM
23 | @REM Required ENV vars:
24 | @REM JAVA_HOME - location of a JDK home dir
25 | @REM
26 | @REM Optional ENV vars
27 | @REM M2_HOME - location of maven2's installed home dir
28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
31 | @REM e.g. to debug Maven itself, use
32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
34 | @REM ----------------------------------------------------------------------------
35 |
36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
37 | @echo off
38 | @REM set title of command window
39 | title %0
40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
42 |
43 | @REM set %HOME% to equivalent of $HOME
44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
45 |
46 | @REM Execute a user defined script before this one
47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending
49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
51 | :skipRcPre
52 |
53 | @setlocal
54 |
55 | set ERROR_CODE=0
56 |
57 | @REM To isolate internal variables from possible post scripts, we use another setlocal
58 | @setlocal
59 |
60 | @REM ==== START VALIDATION ====
61 | if not "%JAVA_HOME%" == "" goto OkJHome
62 |
63 | echo.
64 | echo Error: JAVA_HOME not found in your environment. >&2
65 | echo Please set the JAVA_HOME variable in your environment to match the >&2
66 | echo location of your Java installation. >&2
67 | echo.
68 | goto error
69 |
70 | :OkJHome
71 | if exist "%JAVA_HOME%\bin\java.exe" goto init
72 |
73 | echo.
74 | echo Error: JAVA_HOME is set to an invalid directory. >&2
75 | echo JAVA_HOME = "%JAVA_HOME%" >&2
76 | echo Please set the JAVA_HOME variable in your environment to match the >&2
77 | echo location of your Java installation. >&2
78 | echo.
79 | goto error
80 |
81 | @REM ==== END VALIDATION ====
82 |
83 | :init
84 |
85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
86 | @REM Fallback to current working directory if not found.
87 |
88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
90 |
91 | set EXEC_DIR=%CD%
92 | set WDIR=%EXEC_DIR%
93 | :findBaseDir
94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound
95 | cd ..
96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound
97 | set WDIR=%CD%
98 | goto findBaseDir
99 |
100 | :baseDirFound
101 | set MAVEN_PROJECTBASEDIR=%WDIR%
102 | cd "%EXEC_DIR%"
103 | goto endDetectBaseDir
104 |
105 | :baseDirNotFound
106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
107 | cd "%EXEC_DIR%"
108 |
109 | :endDetectBaseDir
110 |
111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
112 |
113 | @setlocal EnableExtensions EnableDelayedExpansion
114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
116 |
117 | :endReadAdditionalConfig
118 |
119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
122 |
123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar"
124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO (
125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
126 | )
127 |
128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data.
130 | if exist %WRAPPER_JAR% (
131 | echo Found %WRAPPER_JAR%
132 | ) else (
133 | echo Couldn't find %WRAPPER_JAR%, downloading it ...
134 | echo Downloading from: %DOWNLOAD_URL%
135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"
136 | echo Finished downloading %WRAPPER_JAR%
137 | )
138 | @REM End of extension
139 |
140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
141 | if ERRORLEVEL 1 goto error
142 | goto end
143 |
144 | :error
145 | set ERROR_CODE=1
146 |
147 | :end
148 | @endlocal & set ERROR_CODE=%ERROR_CODE%
149 |
150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending
152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
154 | :skipRcPost
155 |
156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause
158 |
159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
160 |
161 | exit /B %ERROR_CODE%
162 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/AuditLogFileWriter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.google.common.annotations.VisibleForTesting;
17 | import com.google.common.cache.CacheBuilder;
18 | import com.google.common.cache.CacheLoader;
19 | import com.google.common.cache.LoadingCache;
20 | import com.google.common.cache.RemovalListener;
21 | import io.airlift.log.Logger;
22 | import jp.co.yahoo.presto.audit.serializer.SerializedLog;
23 |
24 | import java.io.FileWriter;
25 | import java.io.IOException;
26 | import java.util.AbstractMap;
27 | import java.util.Map;
28 | import java.util.concurrent.ArrayBlockingQueue;
29 | import java.util.concurrent.BlockingQueue;
30 | import java.util.concurrent.TimeUnit;
31 |
32 | public class AuditLogFileWriter
33 | implements Runnable
34 | {
35 | private static final int QUEUE_CAPACITY = 10000;
36 | private static final int FILE_TIMEOUT_SEC = 3;
37 |
38 | private static Logger log = Logger.get(AuditLogFileWriter.class);
39 | private static AuditLogFileWriter singleton;
40 | private final Thread t;
41 |
42 | private volatile boolean isTerminate = false;
43 | private final BlockingQueue> queue = new ArrayBlockingQueue<>(QUEUE_CAPACITY);
44 | private LoadingCache fileWriters;
45 |
46 | @VisibleForTesting
47 | private AuditLogFileWriter(WriterFactory writerFactory, Logger logger)
48 | {
49 | this(writerFactory);
50 | log = logger;
51 | }
52 |
53 | private AuditLogFileWriter(WriterFactory writerFactory)
54 | {
55 | t = new Thread(this, "AuditLogWriter Thread");
56 |
57 | // Close file handler when cache timeout
58 | RemovalListener removalListener = removal -> {
59 | FileWriter h = removal.getValue();
60 | try {
61 | log.debug("Close FileWriter: " + removal.getKey());
62 | h.close();
63 | }
64 | catch (Exception e) {
65 | log.error("Failed to close file: " + removal.getKey());
66 | }
67 | };
68 |
69 | // Open file handler when cache is needed
70 | fileWriters = CacheBuilder.newBuilder()
71 | .expireAfterWrite(FILE_TIMEOUT_SEC, TimeUnit.SECONDS)
72 | .removalListener(removalListener)
73 | .build(new CacheLoader()
74 | {
75 | public FileWriter load(String filename)
76 | throws IOException
77 | {
78 | try {
79 | log.debug("Open new FileWriter: " + filename);
80 | return writerFactory.getFileWriter(filename);
81 | }
82 | catch (Exception e) {
83 | log.error("Failed to open file: " + e.getMessage());
84 | throw e;
85 | }
86 | }
87 | });
88 | }
89 |
90 | /**
91 | * Return the singleton instance for this class
92 | *
93 | * @return singleton instance
94 | */
95 | static synchronized AuditLogFileWriter getInstance()
96 | {
97 | if (singleton == null) {
98 | singleton = new AuditLogFileWriter(new WriterFactory());
99 | singleton.start();
100 | }
101 | return singleton;
102 | }
103 |
104 | /**
105 | * Start the thread for file writing
106 | */
107 | void start()
108 | {
109 | isTerminate = false;
110 | t.start();
111 | }
112 |
113 | /**
114 | * Terminate the thread for file writing
115 | */
116 | void stop()
117 | {
118 | isTerminate = true;
119 | }
120 |
121 | /**
122 | * Write data to a particular file indicated by path
123 | */
124 | void write(String path, SerializedLog data)
125 | {
126 | try {
127 | queue.add(new AbstractMap.SimpleEntry<>(path, data));
128 | }
129 | catch (IllegalStateException e) {
130 | log.error("Error adding error log to queue. Queue full while capacity is " + QUEUE_CAPACITY + ". Error: " + e.getMessage());
131 | log.error("Dropped queryID: " + data.getQueryId());
132 | }
133 | catch (Exception e) {
134 | log.error("Unknown error adding error log to queue. ErrorMessage: " + e.getMessage());
135 | log.error("Dropped queryID: " + data.getQueryId());
136 | }
137 | }
138 |
139 | @Override
140 | public void run()
141 | {
142 | while (!isTerminate) {
143 | Map.Entry record;
144 |
145 | // Poll record
146 | try {
147 | // + 1 second before cleanUP to ensure files are marked timeout
148 | record = queue.poll(FILE_TIMEOUT_SEC + 1, TimeUnit.SECONDS);
149 | }
150 | catch (InterruptedException e) {
151 | log.error("Unknown interruptedException." + e);
152 | continue;
153 | }
154 |
155 | if (record == null) {
156 | // Timeout from poll() -> release file handlers
157 | fileWriters.cleanUp();
158 | }
159 | else {
160 | try {
161 | // New record for writing
162 | FileWriter fileWriter = fileWriters.get(record.getKey());
163 | fileWriter.write(record.getValue().getSerializedLog());
164 | fileWriter.write(System.lineSeparator());
165 | }
166 | catch (Exception e) {
167 | log.error("Error writing event log to file in run()." + e);
168 | log.error("Dropped queryID: " + record.getValue().getQueryId());
169 | }
170 | }
171 | }
172 | }
173 |
174 | static class WriterFactory
175 | {
176 | FileWriter getFileWriter(String filename)
177 | throws IOException
178 | {
179 | return new FileWriter(filename, true);
180 | }
181 | }
182 | }
183 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/serializer/AuditRecord.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit.serializer;
15 |
16 | public class AuditRecord
17 | {
18 | private String eventType;
19 |
20 | private String queryId;
21 | private String query;
22 | private String uri;
23 | private String state;
24 |
25 | private Double cpuTime;
26 | private Double wallTime;
27 | private Double queuedTime;
28 | private Long peakUserMemoryBytes;
29 | private Long peakTotalNonRevocableMemoryBytes;
30 | private Long totalBytes;
31 | private Long totalRows;
32 | private int completedSplits;
33 |
34 | private Double createTimestamp;
35 | private Double executionStartTimestamp;
36 | private Double endTimestamp;
37 |
38 | private int errorCode;
39 | private String errorName;
40 | private String failureType;
41 | private String failureMessage;
42 | private String failuresJson;
43 |
44 | private String remoteClientAddress;
45 | private String clientUser;
46 | private String userAgent;
47 | private String source;
48 |
49 | public String getEventType()
50 | {
51 | return eventType;
52 | }
53 |
54 | public void setEventType(String eventType)
55 | {
56 | this.eventType = eventType;
57 | }
58 |
59 | public String getQueryId()
60 | {
61 | return queryId;
62 | }
63 |
64 | public void setQueryId(String queryId)
65 | {
66 | this.queryId = queryId;
67 | }
68 |
69 | public String getQuery()
70 | {
71 | return query;
72 | }
73 |
74 | public void setQuery(String query)
75 | {
76 | this.query = query;
77 | }
78 |
79 | public String getUri()
80 | {
81 | return uri;
82 | }
83 |
84 | public void setUri(String uri)
85 | {
86 | this.uri = uri;
87 | }
88 |
89 | public String getState()
90 | {
91 | return state;
92 | }
93 |
94 | public void setState(String state)
95 | {
96 | this.state = state;
97 | }
98 |
99 | public Double getCpuTime()
100 | {
101 | return cpuTime;
102 | }
103 |
104 | public void setCpuTime(Double cpuTime)
105 | {
106 | this.cpuTime = cpuTime;
107 | }
108 |
109 | public Double getWallTime()
110 | {
111 | return wallTime;
112 | }
113 |
114 | public void setWallTime(Double wallTime)
115 | {
116 | this.wallTime = wallTime;
117 | }
118 |
119 | public Double getQueuedTime()
120 | {
121 | return queuedTime;
122 | }
123 |
124 | public void setQueuedTime(Double queuedTime)
125 | {
126 | this.queuedTime = queuedTime;
127 | }
128 |
129 | public Long getTotalBytes()
130 | {
131 | return totalBytes;
132 | }
133 |
134 | public void setTotalBytes(Long totalBytes)
135 | {
136 | this.totalBytes = totalBytes;
137 | }
138 |
139 | public Long getTotalRows()
140 | {
141 | return totalRows;
142 | }
143 |
144 | public void setTotalRows(Long totalRows)
145 | {
146 | this.totalRows = totalRows;
147 | }
148 |
149 | public int getCompletedSplits()
150 | {
151 | return completedSplits;
152 | }
153 |
154 | public void setCompletedSplits(int completedSplits)
155 | {
156 | this.completedSplits = completedSplits;
157 | }
158 |
159 | public int getErrorCode()
160 | {
161 | return errorCode;
162 | }
163 |
164 | public void setErrorCode(int errorCode)
165 | {
166 | this.errorCode = errorCode;
167 | }
168 |
169 | public String getErrorName()
170 | {
171 | return errorName;
172 | }
173 |
174 | public void setErrorName(String errorName)
175 | {
176 | this.errorName = errorName;
177 | }
178 |
179 | public String getFailureType()
180 | {
181 | return failureType;
182 | }
183 |
184 | public void setFailureType(String failureType)
185 | {
186 | this.failureType = failureType;
187 | }
188 |
189 | public String getFailureMessage()
190 | {
191 | return failureMessage;
192 | }
193 |
194 | public void setFailureMessage(String failureMessage)
195 | {
196 | this.failureMessage = failureMessage;
197 | }
198 |
199 | public String getFailuresJson()
200 | {
201 | return failuresJson;
202 | }
203 |
204 | public void setFailuresJson(String failuresJson)
205 | {
206 | this.failuresJson = failuresJson;
207 | }
208 |
209 | public String getRemoteClientAddress()
210 | {
211 | return remoteClientAddress;
212 | }
213 |
214 | public void setRemoteClientAddress(String remoteClientAddress)
215 | {
216 | this.remoteClientAddress = remoteClientAddress;
217 | }
218 |
219 | public String getClientUser()
220 | {
221 | return clientUser;
222 | }
223 |
224 | public void setClientUser(String clientUser)
225 | {
226 | this.clientUser = clientUser;
227 | }
228 |
229 | public String getUserAgent()
230 | {
231 | return userAgent;
232 | }
233 |
234 | public void setUserAgent(String userAgent)
235 | {
236 | this.userAgent = userAgent;
237 | }
238 |
239 | public String getSource()
240 | {
241 | return source;
242 | }
243 |
244 | public void setSource(String source)
245 | {
246 | this.source = source;
247 | }
248 |
249 | public Double getCreateTimestamp()
250 | {
251 | return createTimestamp;
252 | }
253 |
254 | public void setCreateTimestamp(Double createTimestamp)
255 | {
256 | this.createTimestamp = createTimestamp;
257 | }
258 |
259 | public Double getExecutionStartTimestamp()
260 | {
261 | return executionStartTimestamp;
262 | }
263 |
264 | public void setExecutionStartTimestamp(Double executionStartTimestamp)
265 | {
266 | this.executionStartTimestamp = executionStartTimestamp;
267 | }
268 |
269 | public Double getEndTimestamp()
270 | {
271 | return endTimestamp;
272 | }
273 |
274 | public void setEndTimestamp(Double endTimestamp)
275 | {
276 | this.endTimestamp = endTimestamp;
277 | }
278 |
279 | public Long getPeakUserMemoryBytes()
280 | {
281 | return peakUserMemoryBytes;
282 | }
283 |
284 | public void setPeakUserMemoryBytes(Long peakUserMemoryBytes)
285 | {
286 | this.peakUserMemoryBytes = peakUserMemoryBytes;
287 | }
288 |
289 | public Long getPeakTotalNonRevocableMemoryBytes()
290 | {
291 | return peakTotalNonRevocableMemoryBytes;
292 | }
293 |
294 | public void setPeakTotalNonRevocableMemoryBytes(Long peakTotalNonRevocableMemoryBytes)
295 | {
296 | this.peakTotalNonRevocableMemoryBytes = peakTotalNonRevocableMemoryBytes;
297 | }
298 | }
299 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/TestAuditLogListener.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import org.apache.pulsar.client.api.PulsarClientException;
17 | import org.testng.annotations.Test;
18 |
19 | import java.util.Optional;
20 |
21 | import static org.mockito.ArgumentMatchers.any;
22 | import static org.mockito.ArgumentMatchers.eq;
23 | import static org.mockito.Mockito.mock;
24 | import static org.mockito.Mockito.times;
25 | import static org.mockito.Mockito.verify;
26 |
27 | @Test(singleThreaded = true)
28 | public class TestAuditLogListener
29 | {
30 | TestHelper testHelper = new TestHelper();
31 |
32 | private AuditLogListener createSimpleAuditLogListener(AuditLogFileWriter auditLogFileWriter)
33 | throws PulsarClientException
34 | {
35 | AuditConfig config = new AuditConfig()
36 | .setAuditLogFileWriter(auditLogFileWriter)
37 | .setAuditLogPath("/test/path")
38 | .setAuditSimpleLogName("test-filename.log");
39 | return new AuditLogListener(config);
40 | }
41 |
42 | private AuditLogListener createFullAuditLogListener(AuditLogFileWriter auditLogFileWriter)
43 | throws PulsarClientException
44 | {
45 | AuditConfig config = new AuditConfig()
46 | .setAuditLogFileWriter(auditLogFileWriter)
47 | .setAuditLogPath("/test/path_full")
48 | .setAuditSimpleLogName("test-filename.log")
49 | .setAuditFullLogName("test-filename-full.log");
50 | return new AuditLogListener(config);
51 | }
52 |
53 | private AuditLogListener createFullAuditLogListenerWithFilter(AuditLogFileWriter auditLogFileWriter, String filter)
54 | throws PulsarClientException
55 | {
56 | AuditConfig config = new AuditConfig()
57 | .setAuditLogFileWriter(auditLogFileWriter)
58 | .setAuditLogPath("/test/path_full")
59 | .setAuditSimpleLogName("test-filename.log")
60 | .setAuditFullLogName("test-filename-full.log")
61 | .setLogFilter(filter);
62 | return new AuditLogListener(config);
63 | }
64 |
65 | @Test(expectedExceptions = NullPointerException.class)
66 | public void testCreateAuditLogListenerEmpty()
67 | throws PulsarClientException
68 | {
69 | AuditConfig config = new AuditConfig();
70 | new AuditLogListener(config);
71 | }
72 |
73 | @Test(expectedExceptions = NullPointerException.class)
74 | public void testCreateAuditLogListenerMissingPath()
75 | throws PulsarClientException
76 | {
77 | AuditConfig config = new AuditConfig()
78 | .setAuditSimpleLogName("test-filename.log");
79 | new AuditLogListener(config);
80 | }
81 |
82 | @Test(expectedExceptions = NullPointerException.class)
83 | public void testCreateAuditLogListenerMissingFilename()
84 | throws PulsarClientException
85 | {
86 | AuditConfig config = new AuditConfig()
87 | .setAuditLogPath("/test/path");
88 | new AuditLogListener(config);
89 | }
90 |
91 | @Test
92 | public void testCreateAuditLogListenerSuccessWithoutFullFilename()
93 | throws PulsarClientException
94 | {
95 | createSimpleAuditLogListener(AuditLogFileWriter.getInstance());
96 | }
97 |
98 | @Test
99 | public void testCreateAuditLogListenerSuccessWithFullFilename()
100 | throws PulsarClientException
101 | {
102 | createFullAuditLogListener(AuditLogFileWriter.getInstance());
103 | }
104 |
105 | @Test
106 | public void testSimpleLogQueryCompleted()
107 | throws PulsarClientException
108 | {
109 | AuditLogFileWriter auditLogFileWriterMock = mock(AuditLogFileWriter.class);
110 | AuditLogListener auditLogListener = createSimpleAuditLogListener(auditLogFileWriterMock);
111 |
112 | auditLogListener.queryCompleted(testHelper.createNormalEvent());
113 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path/test-filename.log"), any());
114 | }
115 |
116 | @Test
117 | public void testFullLogQueryCompleted()
118 | throws PulsarClientException
119 | {
120 | AuditLogFileWriter auditLogFileWriterMock = mock(AuditLogFileWriter.class);
121 | AuditLogListener auditLogListener = createFullAuditLogListener(auditLogFileWriterMock);
122 |
123 | auditLogListener.queryCompleted(testHelper.createNormalEvent());
124 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename.log"), any());
125 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
126 | }
127 |
128 | @Test
129 | public void testFilter()
130 | throws PulsarClientException
131 | {
132 | AuditLogFileWriter auditLogFileWriterMock = mock(AuditLogFileWriter.class);
133 | AuditLogListener auditLogListener = createFullAuditLogListenerWithFilter(auditLogFileWriterMock, "SRE_SYSTEM");
134 |
135 | auditLogListener.queryCompleted(testHelper.createNormalEvent());
136 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename.log"), any());
137 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
138 |
139 | auditLogListener.queryCompleted(testHelper.createQueryWithSource(Optional.of("SRE_SYSTEM")));
140 | verify(auditLogFileWriterMock, times(2)).write(eq("/test/path_full/test-filename.log"), any());
141 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
142 | }
143 |
144 | @Test
145 | public void testFilterComplex()
146 | throws PulsarClientException
147 | {
148 | AuditLogFileWriter auditLogFileWriterMock = mock(AuditLogFileWriter.class);
149 | AuditLogListener auditLogListener = createFullAuditLogListenerWithFilter(auditLogFileWriterMock, "(SRE_SYSTEM|Presto_team)");
150 |
151 | auditLogListener.queryCompleted(testHelper.createNormalEvent());
152 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename.log"), any());
153 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
154 |
155 | auditLogListener.queryCompleted(testHelper.createQueryWithSource(Optional.of("SRE_SYSTEM")));
156 | verify(auditLogFileWriterMock, times(2)).write(eq("/test/path_full/test-filename.log"), any());
157 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
158 |
159 | auditLogListener.queryCompleted(testHelper.createQueryWithSource(Optional.of("Presto_team")));
160 | verify(auditLogFileWriterMock, times(3)).write(eq("/test/path_full/test-filename.log"), any());
161 | verify(auditLogFileWriterMock, times(1)).write(eq("/test/path_full/test-filename-full.log"), any());
162 | }
163 | }
164 |
--------------------------------------------------------------------------------
/src/checkstyle/checks.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 |
8 |
10 |
11 |
14 |
15 |
18 |
19 |
22 |
23 |
26 |
27 |
30 |
31 |
34 |
35 |
38 |
39 |
42 |
43 |
46 |
47 |
50 |
51 |
54 |
55 |
57 |
59 |
64 |
67 |
69 |
70 |
73 |
77 |
80 |
81 |
86 |
87 |
89 |
114 |
116 |
117 |
119 |
120 |
124 |
129 |
130 |
131 |
144 |
145 |
146 |
155 |
156 |
157 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/TestHelper.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import com.facebook.presto.spi.ErrorCode;
17 | import com.facebook.presto.spi.ErrorType;
18 | import com.facebook.presto.spi.eventlistener.QueryCompletedEvent;
19 | import com.facebook.presto.spi.eventlistener.QueryContext;
20 | import com.facebook.presto.spi.eventlistener.QueryFailureInfo;
21 | import com.facebook.presto.spi.eventlistener.QueryIOMetadata;
22 | import com.facebook.presto.spi.eventlistener.QueryInputMetadata;
23 | import com.facebook.presto.spi.eventlistener.QueryMetadata;
24 | import com.facebook.presto.spi.eventlistener.QueryStatistics;
25 | import com.facebook.presto.spi.eventlistener.StageCpuDistribution;
26 | import com.facebook.presto.spi.resourceGroups.ResourceGroupId;
27 | import com.facebook.presto.spi.session.ResourceEstimates;
28 | import io.airlift.units.DataSize;
29 |
30 | import java.net.URI;
31 | import java.time.Duration;
32 | import java.time.Instant;
33 | import java.time.ZoneId;
34 | import java.time.ZonedDateTime;
35 | import java.util.ArrayList;
36 | import java.util.Collections;
37 | import java.util.HashMap;
38 | import java.util.HashSet;
39 | import java.util.Optional;
40 |
41 | public class TestHelper
42 | {
43 | private URI uri;
44 | private QueryStatistics statistics;
45 | private QueryContext context;
46 | private QueryIOMetadata ioMetadata;
47 | private Instant createTime;
48 | private Instant executionStartTime;
49 | private Instant endTime;
50 |
51 | public void setUp()
52 | {
53 | try {
54 | uri = new URI("http://example.com:8080/v1/query/20170521_140224_00002_gd5k3");
55 | statistics = new QueryStatistics(Duration.ofMillis(100),
56 | Duration.ofMillis(200),
57 | Duration.ofMillis(300),
58 | Optional.of(Duration.ofMillis(400)),
59 | Optional.of(Duration.ofMillis(500)),
60 | 10001,
61 | 10002,
62 | 10003,
63 | 0,
64 | 14,
65 | 0,
66 | 0,
67 | 0,
68 | 0,
69 | 2048.0,
70 | new ArrayList<>(),
71 | 4096,
72 | true,
73 | new ArrayList() {},
74 | new ArrayList()
75 | {{
76 | add("operatorSummaries 01");
77 | }}
78 | );
79 | context = new QueryContext(
80 | "test-user",
81 | Optional.of("principal"),
82 | Optional.of("example.com"),
83 | Optional.of("StatementClient 0.167"),
84 | Optional.of("clientInfo"),
85 | new HashSet<>(),
86 | new HashSet<>(),
87 | Optional.of("presto-cli"),
88 | Optional.of("catalog"),
89 | Optional.of("schema"),
90 | Optional.of(new ResourceGroupId("testResourceGroup")),
91 | new HashMap<>(),
92 | new ResourceEstimates(
93 | Optional.of(io.airlift.units.Duration.valueOf("1s")),
94 | Optional.of(io.airlift.units.Duration.valueOf("1s")),
95 | Optional.of(DataSize.valueOf("6TB"))),
96 | "127.0.0.1",
97 | "0.175",
98 | "environment");
99 | ioMetadata = new QueryIOMetadata(new ArrayList(),
100 | Optional.empty());
101 | ZoneId jst_zone = ZoneId.of("Asia/Tokyo");
102 | createTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 0, 0, jst_zone).toInstant();
103 | executionStartTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 1, 0, jst_zone).toInstant();
104 | endTime = ZonedDateTime.of(2017, 6 + 1, 15, 10, 0, 3, 0, jst_zone).toInstant();
105 | }
106 | catch (Exception ignoreException) {
107 | ignoreException.printStackTrace();
108 |
109 | }
110 | }
111 |
112 | public QueryCompletedEvent createNormalEvent()
113 | {
114 | setUp();
115 | QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3",
116 | Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"),
117 | "select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5",
118 | "FINISHED",
119 | uri,
120 | Optional.empty(),
121 | Optional.empty());
122 | return new QueryCompletedEvent(metadata,
123 | statistics,
124 | context,
125 | ioMetadata,
126 | Optional.empty(),
127 | Collections.emptyList(),
128 | createTime,
129 | executionStartTime,
130 | endTime);
131 | }
132 |
133 | public QueryCompletedEvent createFailureEvent()
134 | {
135 | setUp();
136 | QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3",
137 | Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"),
138 | "select 2a",
139 | "FAILED",
140 | uri,
141 | Optional.empty(),
142 | Optional.empty());
143 | ErrorCode errorCode = new ErrorCode(1, "SYNTAX_ERROR", ErrorType.USER_ERROR);
144 | QueryFailureInfo failureInfo = new QueryFailureInfo(errorCode,
145 | Optional.of("com.facebook.presto.sql.parser.ParsingException"),
146 | Optional.of("line 1:15: mismatched input '0' expecting ')'"),
147 | Optional.empty(),
148 | Optional.empty(),
149 | "{json-error}"
150 | );
151 |
152 | return new QueryCompletedEvent(metadata,
153 | statistics,
154 | context,
155 | ioMetadata,
156 | Optional.of(failureInfo),
157 | Collections.emptyList(),
158 | createTime,
159 | executionStartTime,
160 | endTime);
161 | }
162 |
163 | public QueryCompletedEvent createQueryWithSource(Optional source)
164 | {
165 | setUp();
166 | QueryMetadata metadata = new QueryMetadata("20170606_044544_00024_nfhe3",
167 | Optional.of("4c52973c-14c6-4534-837f-238e21d9b061"),
168 | "select * from airdelays_s3_csv WHERE kw = 'presto-kw-example' limit 5",
169 | "FINISHED",
170 | uri,
171 | Optional.empty(),
172 | Optional.empty());
173 | QueryContext context = new QueryContext(
174 | "test-user",
175 | Optional.of("principal"),
176 | Optional.of("example.com"),
177 | Optional.of("StatementClient 0.167"),
178 | Optional.of("clientInfo"),
179 | new HashSet<>(),
180 | new HashSet<>(),
181 | source,
182 | Optional.of("catalog"),
183 | Optional.of("schema"),
184 | Optional.of(new ResourceGroupId("testResourceGroup")),
185 | new HashMap<>(),
186 | new ResourceEstimates(
187 | Optional.of(io.airlift.units.Duration.valueOf("1s")),
188 | Optional.of(io.airlift.units.Duration.valueOf("1s")),
189 | Optional.of(DataSize.valueOf("6TB"))),
190 | "127.0.0.1",
191 | "0.175",
192 | "environment");
193 | return new QueryCompletedEvent(metadata,
194 | statistics,
195 | context,
196 | ioMetadata,
197 | Optional.empty(),
198 | Collections.emptyList(),
199 | createTime,
200 | executionStartTime,
201 | endTime);
202 | }
203 | }
204 |
--------------------------------------------------------------------------------
/src/main/java/jp/co/yahoo/presto/audit/AuditConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import io.airlift.configuration.Config;
17 | import io.airlift.configuration.ConfigDescription;
18 | import org.apache.pulsar.client.api.CompressionType;
19 |
20 | import javax.annotation.Nullable;
21 | import javax.validation.constraints.NotNull;
22 |
23 | public class AuditConfig
24 | {
25 | private String auditLogPath;
26 | private String auditSimpleLogName;
27 | private String auditFullLogName;
28 | private String simpleLogTopic;
29 | private String fullLogTopic;
30 | private String url;
31 | private String trustCerts;
32 | private String logFilter;
33 | private String tenantDomain;
34 | private String tenantService;
35 | private String providerDomain;
36 | private String privateKey;
37 | private String athenzConfPath;
38 | private String principalHeader;
39 | private String roleHeader;
40 | private CompressionType compressionType = CompressionType.NONE;
41 | private AuditLogFileWriter auditLogFileWriter = AuditLogFileWriter.getInstance();
42 | private boolean useTLS = true;
43 |
44 | @NotNull
45 | public AuditLogFileWriter getAuditLogFileWriter()
46 | {
47 | return auditLogFileWriter;
48 | }
49 |
50 | public AuditConfig setAuditLogFileWriter(AuditLogFileWriter auditLogFileWriter)
51 | {
52 | this.auditLogFileWriter = auditLogFileWriter;
53 | return this;
54 | }
55 |
56 | @NotNull
57 | public String getAuditLogPath()
58 | {
59 | return auditLogPath;
60 | }
61 |
62 | @Config("event-listener.audit-log-path")
63 | @ConfigDescription("audit log path")
64 | public AuditConfig setAuditLogPath(String eventLogPath)
65 | {
66 | this.auditLogPath = eventLogPath;
67 | return this;
68 | }
69 |
70 | @NotNull
71 | public String getAuditSimpleLogName()
72 | {
73 | return auditSimpleLogName;
74 | }
75 |
76 | @Config("event-listener.audit-log-filename")
77 | @ConfigDescription("audit simple log file name")
78 | public AuditConfig setAuditSimpleLogName(String auditSimpleLogName)
79 | {
80 | this.auditSimpleLogName = auditSimpleLogName;
81 | return this;
82 | }
83 |
84 | @Nullable
85 | public String getAuditFullLogName()
86 | {
87 | return auditFullLogName;
88 | }
89 |
90 | @Config("event-listener.audit-log-full-filename")
91 | @ConfigDescription("audit full log file name")
92 | public AuditConfig setAuditFullLogName(String auditFullLogName)
93 | {
94 | this.auditFullLogName = auditFullLogName;
95 | return this;
96 | }
97 |
98 | @Nullable
99 | public String getSimpleTopic()
100 | {
101 | return simpleLogTopic;
102 | }
103 |
104 | @Config("event-listener.pulsar.simple-log-topic")
105 | @ConfigDescription("simple log topic")
106 | public AuditConfig setSimpleTopic(String simpleLogTopic)
107 | {
108 | this.simpleLogTopic = simpleLogTopic;
109 | return this;
110 | }
111 |
112 | @Nullable
113 | public String getFullTopic()
114 | {
115 | return fullLogTopic;
116 | }
117 |
118 | @Config("event-listener.pulsar.full-log-topic")
119 | @ConfigDescription("full log topic")
120 | public AuditConfig setFullTopic(String fullLogTopic)
121 | {
122 | this.fullLogTopic = fullLogTopic;
123 | return this;
124 | }
125 |
126 | @Nullable
127 | public String getPrivateKeyPath()
128 | {
129 | return privateKey;
130 | }
131 |
132 | @Config("event-listener.athenz.private-key-path")
133 | @ConfigDescription("private key path")
134 | public AuditConfig setPrivateKeyPath(String privateKey)
135 | {
136 | this.privateKey = privateKey;
137 | return this;
138 | }
139 |
140 | @Nullable
141 | public String getPulsarUrl()
142 | {
143 | return url;
144 | }
145 |
146 | @Config("event-listener.pulsar.pulsar-url")
147 | @ConfigDescription("pulsar url")
148 | public AuditConfig setPulsarUrl(String pulsarUrl)
149 | {
150 | this.url = pulsarUrl;
151 | return this;
152 | }
153 |
154 | @Nullable
155 | public String getTrustCertsPath()
156 | {
157 | return trustCerts;
158 | }
159 |
160 | @Config("event-listener.pulsar.pulsar-cert-path")
161 | @ConfigDescription("pulsar cert path")
162 | public AuditConfig setTrustCertsPath(String trustCertsPath)
163 | {
164 | this.trustCerts = trustCertsPath;
165 | return this;
166 | }
167 |
168 | @NotNull
169 | public boolean getUseTLS()
170 | {
171 | return useTLS;
172 | }
173 |
174 | @Config("event-listener.pulsar.use-TLS")
175 | @ConfigDescription("audit log path")
176 | public AuditConfig setUseTLS(boolean useTLS)
177 | {
178 | this.useTLS = useTLS;
179 | return this;
180 | }
181 |
182 | @NotNull
183 | public CompressionType getCompressionType()
184 | {
185 | return compressionType;
186 | }
187 |
188 | @Config("event-listener.pulsar.compression-type")
189 | @ConfigDescription("message compression type (NONE, LZ4 or ZLIB)")
190 | public AuditConfig setCompressionType(CompressionType compressionType)
191 | {
192 | this.compressionType = compressionType;
193 | return this;
194 | }
195 |
196 | @Nullable
197 | public String getLogFilter()
198 | {
199 | return logFilter;
200 | }
201 |
202 | @Config("event-listener.audit-log-full-filter")
203 | @ConfigDescription("audit log full filter")
204 | public AuditConfig setLogFilter(String auditLogFullFilter)
205 | {
206 | this.logFilter = auditLogFullFilter;
207 | return this;
208 | }
209 |
210 | @Nullable
211 | public String getTenantDomain()
212 | {
213 | return tenantDomain;
214 | }
215 |
216 | @Config("event-listener.athenz.tenant-domain")
217 | @ConfigDescription("tenant domain name")
218 | public AuditConfig setTenantDomain(String athenzTenantDomain)
219 | {
220 | this.tenantDomain = athenzTenantDomain;
221 | return this;
222 | }
223 |
224 | @Nullable
225 | public String getTenantService()
226 | {
227 | return tenantService;
228 | }
229 |
230 | @Config("event-listener.athenz.tenant-service")
231 | @ConfigDescription("tenant service name")
232 | public AuditConfig setTenantService(String athenzTenantService)
233 | {
234 | this.tenantService = athenzTenantService;
235 | return this;
236 | }
237 |
238 | @Nullable
239 | public String getProviderDomain()
240 | {
241 | return providerDomain;
242 | }
243 |
244 | @Config("event-listener.athenz.provider-domain")
245 | @ConfigDescription("athenz provider domain")
246 | public AuditConfig setProviderDomain(String athenzProviderDomain)
247 | {
248 | this.providerDomain = athenzProviderDomain;
249 | return this;
250 | }
251 |
252 | @Nullable
253 | public String getAthenzConfPath()
254 | {
255 | return athenzConfPath;
256 | }
257 |
258 | @Config("event-listener.athenz.config-path")
259 | @ConfigDescription("athenz config path")
260 | public AuditConfig setAthenzConfPath(String athenzConfigPath)
261 | {
262 | this.athenzConfPath = athenzConfigPath;
263 | return this;
264 | }
265 |
266 | @Nullable
267 | public String getPrincipalHeader()
268 | {
269 | return principalHeader;
270 | }
271 |
272 | @Config("event-listener.athenz.principal-header")
273 | @ConfigDescription("athenz principal header")
274 | public AuditConfig setPrincipalHeader(String athenzPrincipalHeader)
275 | {
276 | this.principalHeader = athenzPrincipalHeader;
277 | return this;
278 | }
279 |
280 | @Nullable
281 | public String getRoleHeader()
282 | {
283 | return roleHeader;
284 | }
285 |
286 | @Config("event-listener.athenz.role-header")
287 | @ConfigDescription("athenz role header")
288 | public AuditConfig setRoleHeader(String athenzRoleHeader)
289 | {
290 | this.roleHeader = athenzRoleHeader;
291 | return this;
292 | }
293 | }
294 |
--------------------------------------------------------------------------------
/mvnw:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # ----------------------------------------------------------------------------
3 | # Licensed to the Apache Software Foundation (ASF) under one
4 | # or more contributor license agreements. See the NOTICE file
5 | # distributed with this work for additional information
6 | # regarding copyright ownership. The ASF licenses this file
7 | # to you under the Apache License, Version 2.0 (the
8 | # "License"); you may not use this file except in compliance
9 | # with the License. You may obtain a copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 | # ----------------------------------------------------------------------------
20 |
21 | # ----------------------------------------------------------------------------
22 | # Maven2 Start Up Batch script
23 | #
24 | # Required ENV vars:
25 | # ------------------
26 | # JAVA_HOME - location of a JDK home dir
27 | #
28 | # Optional ENV vars
29 | # -----------------
30 | # M2_HOME - location of maven2's installed home dir
31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven
32 | # e.g. to debug Maven itself, use
33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files
35 | # ----------------------------------------------------------------------------
36 |
37 | if [ -z "$MAVEN_SKIP_RC" ] ; then
38 |
39 | if [ -f /etc/mavenrc ] ; then
40 | . /etc/mavenrc
41 | fi
42 |
43 | if [ -f "$HOME/.mavenrc" ] ; then
44 | . "$HOME/.mavenrc"
45 | fi
46 |
47 | fi
48 |
49 | # OS specific support. $var _must_ be set to either true or false.
50 | cygwin=false;
51 | darwin=false;
52 | mingw=false
53 | case "`uname`" in
54 | CYGWIN*) cygwin=true ;;
55 | MINGW*) mingw=true;;
56 | Darwin*) darwin=true
57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
59 | if [ -z "$JAVA_HOME" ]; then
60 | if [ -x "/usr/libexec/java_home" ]; then
61 | export JAVA_HOME="`/usr/libexec/java_home`"
62 | else
63 | export JAVA_HOME="/Library/Java/Home"
64 | fi
65 | fi
66 | ;;
67 | esac
68 |
69 | if [ -z "$JAVA_HOME" ] ; then
70 | if [ -r /etc/gentoo-release ] ; then
71 | JAVA_HOME=`java-config --jre-home`
72 | fi
73 | fi
74 |
75 | if [ -z "$M2_HOME" ] ; then
76 | ## resolve links - $0 may be a link to maven's home
77 | PRG="$0"
78 |
79 | # need this for relative symlinks
80 | while [ -h "$PRG" ] ; do
81 | ls=`ls -ld "$PRG"`
82 | link=`expr "$ls" : '.*-> \(.*\)$'`
83 | if expr "$link" : '/.*' > /dev/null; then
84 | PRG="$link"
85 | else
86 | PRG="`dirname "$PRG"`/$link"
87 | fi
88 | done
89 |
90 | saveddir=`pwd`
91 |
92 | M2_HOME=`dirname "$PRG"`/..
93 |
94 | # make it fully qualified
95 | M2_HOME=`cd "$M2_HOME" && pwd`
96 |
97 | cd "$saveddir"
98 | # echo Using m2 at $M2_HOME
99 | fi
100 |
101 | # For Cygwin, ensure paths are in UNIX format before anything is touched
102 | if $cygwin ; then
103 | [ -n "$M2_HOME" ] &&
104 | M2_HOME=`cygpath --unix "$M2_HOME"`
105 | [ -n "$JAVA_HOME" ] &&
106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
107 | [ -n "$CLASSPATH" ] &&
108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
109 | fi
110 |
111 | # For Mingw, ensure paths are in UNIX format before anything is touched
112 | if $mingw ; then
113 | [ -n "$M2_HOME" ] &&
114 | M2_HOME="`(cd "$M2_HOME"; pwd)`"
115 | [ -n "$JAVA_HOME" ] &&
116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
117 | # TODO classpath?
118 | fi
119 |
120 | if [ -z "$JAVA_HOME" ]; then
121 | javaExecutable="`which javac`"
122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
123 | # readlink(1) is not available as standard on Solaris 10.
124 | readLink=`which readlink`
125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
126 | if $darwin ; then
127 | javaHome="`dirname \"$javaExecutable\"`"
128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
129 | else
130 | javaExecutable="`readlink -f \"$javaExecutable\"`"
131 | fi
132 | javaHome="`dirname \"$javaExecutable\"`"
133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'`
134 | JAVA_HOME="$javaHome"
135 | export JAVA_HOME
136 | fi
137 | fi
138 | fi
139 |
140 | if [ -z "$JAVACMD" ] ; then
141 | if [ -n "$JAVA_HOME" ] ; then
142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
143 | # IBM's JDK on AIX uses strange locations for the executables
144 | JAVACMD="$JAVA_HOME/jre/sh/java"
145 | else
146 | JAVACMD="$JAVA_HOME/bin/java"
147 | fi
148 | else
149 | JAVACMD="`which java`"
150 | fi
151 | fi
152 |
153 | if [ ! -x "$JAVACMD" ] ; then
154 | echo "Error: JAVA_HOME is not defined correctly." >&2
155 | echo " We cannot execute $JAVACMD" >&2
156 | exit 1
157 | fi
158 |
159 | if [ -z "$JAVA_HOME" ] ; then
160 | echo "Warning: JAVA_HOME environment variable is not set."
161 | fi
162 |
163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
164 |
165 | # traverses directory structure from process work directory to filesystem root
166 | # first directory with .mvn subdirectory is considered project base directory
167 | find_maven_basedir() {
168 |
169 | if [ -z "$1" ]
170 | then
171 | echo "Path not specified to find_maven_basedir"
172 | return 1
173 | fi
174 |
175 | basedir="$1"
176 | wdir="$1"
177 | while [ "$wdir" != '/' ] ; do
178 | if [ -d "$wdir"/.mvn ] ; then
179 | basedir=$wdir
180 | break
181 | fi
182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc)
183 | if [ -d "${wdir}" ]; then
184 | wdir=`cd "$wdir/.."; pwd`
185 | fi
186 | # end of workaround
187 | done
188 | echo "${basedir}"
189 | }
190 |
191 | # concatenates all lines of a file
192 | concat_lines() {
193 | if [ -f "$1" ]; then
194 | echo "$(tr -s '\n' ' ' < "$1")"
195 | fi
196 | }
197 |
198 | BASE_DIR=`find_maven_basedir "$(pwd)"`
199 | if [ -z "$BASE_DIR" ]; then
200 | exit 1;
201 | fi
202 |
203 | ##########################################################################################
204 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
205 | # This allows using the maven wrapper in projects that prohibit checking in binary data.
206 | ##########################################################################################
207 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
208 | if [ "$MVNW_VERBOSE" = true ]; then
209 | echo "Found .mvn/wrapper/maven-wrapper.jar"
210 | fi
211 | else
212 | if [ "$MVNW_VERBOSE" = true ]; then
213 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
214 | fi
215 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.0/maven-wrapper-0.4.0.jar"
216 | while IFS="=" read key value; do
217 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
218 | esac
219 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
220 | if [ "$MVNW_VERBOSE" = true ]; then
221 | echo "Downloading from: $jarUrl"
222 | fi
223 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
224 |
225 | if command -v wget > /dev/null; then
226 | if [ "$MVNW_VERBOSE" = true ]; then
227 | echo "Found wget ... using wget"
228 | fi
229 | wget "$jarUrl" -O "$wrapperJarPath"
230 | elif command -v curl > /dev/null; then
231 | if [ "$MVNW_VERBOSE" = true ]; then
232 | echo "Found curl ... using curl"
233 | fi
234 | curl -o "$wrapperJarPath" "$jarUrl"
235 | else
236 | if [ "$MVNW_VERBOSE" = true ]; then
237 | echo "Falling back to using Java to download"
238 | fi
239 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
240 | if [ -e "$javaClass" ]; then
241 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
242 | if [ "$MVNW_VERBOSE" = true ]; then
243 | echo " - Compiling MavenWrapperDownloader.java ..."
244 | fi
245 | # Compiling the Java class
246 | ("$JAVA_HOME/bin/javac" "$javaClass")
247 | fi
248 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
249 | # Running the downloader
250 | if [ "$MVNW_VERBOSE" = true ]; then
251 | echo " - Running MavenWrapperDownloader.java ..."
252 | fi
253 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
254 | fi
255 | fi
256 | fi
257 | fi
258 | ##########################################################################################
259 | # End of extension
260 | ##########################################################################################
261 |
262 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
263 | if [ "$MVNW_VERBOSE" = true ]; then
264 | echo $MAVEN_PROJECTBASEDIR
265 | fi
266 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
267 |
268 | # For Cygwin, switch paths to Windows format before running java
269 | if $cygwin; then
270 | [ -n "$M2_HOME" ] &&
271 | M2_HOME=`cygpath --path --windows "$M2_HOME"`
272 | [ -n "$JAVA_HOME" ] &&
273 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
274 | [ -n "$CLASSPATH" ] &&
275 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
276 | [ -n "$MAVEN_PROJECTBASEDIR" ] &&
277 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
278 | fi
279 |
280 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
281 |
282 | exec "$JAVACMD" \
283 | $MAVEN_OPTS \
284 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
285 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
286 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
287 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | 4.0.0
4 |
5 | jp.co.yahoo.presto.plugin
6 | presto-audit-plugin
7 | Presto - Audit log
8 | 2.2-SNAPSHOT
9 | jar
10 |
11 |
12 |
13 | Apache License 2.0
14 | http://www.apache.org/licenses/LICENSE-2.0
15 | repo
16 |
17 |
18 |
19 |
20 | scm:git:git://github.com/yahoojapan/presto-audit.git
21 | https://github.com/yahoojapan/presto-audit
22 | HEAD
23 |
24 |
25 |
26 | 0.176
27 | 0.214
28 | UTF-8
29 | 1.8
30 | 1.8
31 | 0
32 |
33 |
34 |
35 |
36 | io.airlift
37 | log
38 | ${dep.airlift.version}
39 |
40 |
41 | io.airlift
42 | json
43 | ${dep.airlift.version}
44 |
45 |
46 | io.airlift
47 | configuration
48 | ${dep.airlift.version}
49 |
50 |
51 | io.airlift
52 | bootstrap
53 | ${dep.airlift.version}
54 |
55 |
56 |
57 | com.google.guava
58 | guava
59 | 22.0
60 |
61 |
62 |
63 |
64 | com.facebook.presto
65 | presto-spi
66 | ${presto.version}
67 | provided
68 |
69 |
70 |
71 | io.airlift
72 | units
73 | 1.0
74 | provided
75 |
76 |
77 |
78 | com.fasterxml.jackson.core
79 | jackson-annotations
80 | 2.9.7
81 | provided
82 |
83 |
84 |
85 |
86 | com.google.code.gson
87 | gson
88 | 2.2.4
89 |
90 |
91 |
92 |
93 | com.facebook.presto
94 | presto-main
95 | ${presto.version}
96 | test
97 |
98 |
99 |
100 | org.assertj
101 | assertj-core
102 | 3.0.0
103 | test
104 |
105 |
106 |
107 | org.testng
108 | testng
109 | 6.10
110 | test
111 |
112 |
113 |
114 | io.airlift
115 | testing
116 | ${dep.airlift.version}
117 | test
118 |
119 |
120 |
121 | io.airlift
122 | http-server
123 | ${dep.airlift.version}
124 | test
125 |
126 |
127 |
128 | org.mockito
129 | mockito-core
130 | 2.16.0
131 | test
132 |
133 |
134 |
135 | io.airlift
136 | node
137 | ${dep.airlift.version}
138 | test
139 |
140 |
141 |
142 | org.apache.pulsar
143 | pulsar-client
144 | 1.22.1-incubating
145 |
146 |
147 |
148 | org.apache.pulsar
149 | pulsar-client-auth-athenz
150 | 1.22.1-incubating
151 |
152 |
153 |
154 |
155 | src/main/java
156 |
157 |
158 | org.apache.maven.plugins
159 | maven-checkstyle-plugin
160 | 2.17
161 |
162 |
163 | validate
164 | validate
165 |
166 | src/checkstyle/checks.xml
167 | UTF-8
168 | true
169 | true
170 |
171 |
172 | check
173 |
174 |
175 |
176 |
177 |
178 |
179 | org.apache.maven.plugins
180 | maven-compiler-plugin
181 | 3.1
182 |
183 | ${maven.compiler.source}
184 | ${maven.compiler.target}
185 | UTF-8
186 | true
187 |
188 |
189 |
190 |
191 | io.takari.maven.plugins
192 | presto-maven-plugin
193 | 0.1.12
194 | true
195 |
196 |
197 |
198 |
199 | org.apache.maven.plugins
200 | maven-shade-plugin
201 | 3.1.0
202 |
203 |
204 | package
205 |
206 | shade
207 |
208 |
209 | true
210 | all
211 |
212 |
213 | *:*
214 |
215 | META-INF/*.SF
216 | META-INF/*.DSA
217 | META-INF/*.RSA
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 | com.mycila
228 | license-maven-plugin
229 | 3.0
230 |
231 | ${project.basedir}/src/license/LICENSE-HEADER.txt
232 |
233 | *.xml
234 | src/checkstyle/*
235 | src/sql/*
236 | .mvn/**
237 | mvnw*
238 |
239 |
240 |
241 |
242 |
243 | org.apache.maven.plugins
244 | maven-release-plugin
245 | 2.5.3
246 |
247 | forked-path
248 | false
249 | true
250 | false
251 | clean install
252 | true
253 | @{project.version}
254 |
255 |
256 |
257 |
258 | org.apache.maven.plugins
259 | maven-surefire-plugin
260 | 2.20.1
261 |
262 |
263 | Asia/Tokyo
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 | org.codehaus.mojo
273 | rpm-maven-plugin
274 | 2.1.5
275 |
276 |
286 |
287 |
288 | presto-audit-plugin
289 | ${project.version}
290 | Applications/Databases
291 | Presto audit plugin Package.
292 | ASL 2.0
293 | ${build.no}.el7
294 | true
295 | 0
296 | https://github.com/yahoojapan/presto-audit
297 |
298 |
299 | /usr/lib/presto/lib/plugin/yj-audit
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
--------------------------------------------------------------------------------
/src/test/java/jp/co/yahoo/presto/audit/TestAuditLogFileWriter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | */
14 | package jp.co.yahoo.presto.audit;
15 |
16 | import io.airlift.log.Logger;
17 | import jp.co.yahoo.presto.audit.serializer.SerializedLog;
18 | import org.mockito.ArgumentCaptor;
19 | import org.mockito.stubbing.Answer;
20 | import org.testng.annotations.Test;
21 |
22 | import java.io.FileWriter;
23 | import java.io.IOException;
24 | import java.lang.reflect.Constructor;
25 | import java.util.List;
26 | import java.util.stream.Collectors;
27 |
28 | import static jp.co.yahoo.presto.audit.AuditLogFileWriter.WriterFactory;
29 | import static org.mockito.ArgumentMatchers.any;
30 | import static org.mockito.ArgumentMatchers.anyString;
31 | import static org.mockito.ArgumentMatchers.eq;
32 | import static org.mockito.Mockito.atLeastOnce;
33 | import static org.mockito.Mockito.doAnswer;
34 | import static org.mockito.Mockito.doThrow;
35 | import static org.mockito.Mockito.mock;
36 | import static org.mockito.Mockito.spy;
37 | import static org.mockito.Mockito.times;
38 | import static org.mockito.Mockito.verify;
39 | import static org.mockito.Mockito.when;
40 |
41 | @Test(singleThreaded = true, threadPoolSize = 1)
42 | public class TestAuditLogFileWriter
43 | {
44 | private static final String QUERY_ID = "20170606_044544_00024_abcde";
45 | private void pause()
46 | {
47 | try {
48 | Thread.sleep(6000);
49 | }
50 | catch (Exception e) {
51 | }
52 | }
53 |
54 | private void initTest()
55 | {
56 | pause();
57 | StackTraceElement[] stackTrace = Thread.currentThread()
58 | .getStackTrace();
59 | System.out.println("===========================");
60 | System.out.println(stackTrace[2].getMethodName());
61 | System.out.flush();
62 | }
63 |
64 | private AuditLogFileWriter getNewAuditLogFileWriter(WriterFactory writerFactory, Logger logger) throws Exception
65 | {
66 | Constructor constructor = AuditLogFileWriter.class.getDeclaredConstructor(WriterFactory.class, Logger.class);
67 | constructor.setAccessible(true);
68 | AuditLogFileWriter auditLogFileWriter = constructor.newInstance(writerFactory, logger);
69 | auditLogFileWriter.start();
70 | return auditLogFileWriter;
71 | }
72 |
73 | @Test
74 | public void testSingleton()
75 | {
76 | AuditLogFileWriter auditLogFileWriter_1 = AuditLogFileWriter.getInstance();
77 | AuditLogFileWriter auditLogFileWriter_2 = AuditLogFileWriter.getInstance();
78 | assert(auditLogFileWriter_1==auditLogFileWriter_2);
79 | }
80 |
81 | @Test
82 | public void testNormalWrite()
83 | {
84 | initTest();
85 | final String FILE_NAME = "/tmp/file1";
86 | final String DATA = "{\"data\":\"value\"}";
87 |
88 | AuditLogFileWriter auditLogFileWriter = AuditLogFileWriter.getInstance();
89 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
90 | }
91 |
92 | @Test
93 | public void testWriteAutoCloseFile() throws Exception
94 | {
95 | initTest();
96 | final String FILE_NAME = "/tmp/file1";
97 | final String DATA = "{\"data\":\"value\"}";
98 |
99 | // Setup Spy FileWriter
100 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
101 | final FileWriter[] spyFileWriter = new FileWriter[10];
102 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
103 | String filename = i.getArgument(0);
104 | assert(filename.equals(FILE_NAME));
105 | FileWriter fileWriter = new FileWriter(filename, true);
106 | spyFileWriter[0] = spy(fileWriter);
107 | doAnswer((Answer) var1 -> {
108 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
109 | return "";
110 | }).when(spyFileWriter[0]).write(anyString());
111 | return spyFileWriter[0];
112 | });
113 |
114 | // Test write
115 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, Logger.get("testWriteAutoCloseFile"));
116 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
117 |
118 | // Verify
119 | pause();
120 | verify(spyFileWriter[0]).write(DATA);
121 | verify(spyFileWriter[0]).close();
122 | }
123 |
124 | @Test
125 | public void testWriteCloseFileException() throws Exception
126 | {
127 | initTest();
128 | final String FILE_NAME = "/tmp/file1";
129 | final String DATA = "{\"data\":\"value\"}";
130 |
131 | // Setup Spy FileWriter
132 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
133 | final FileWriter[] spyFileWriter = new FileWriter[10];
134 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
135 | String filename = i.getArgument(0);
136 | assert(filename.equals(FILE_NAME));
137 | FileWriter fileWriter = new FileWriter(filename, true);
138 | spyFileWriter[0] = spy(fileWriter);
139 | doAnswer((Answer) var1 -> {
140 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
141 | return "";
142 | }).when(spyFileWriter[0]).write(anyString());
143 | doThrow(new IOException("Mock close file exception")).when(spyFileWriter[0]).close();
144 | return spyFileWriter[0];
145 | });
146 |
147 | // Test write
148 | Logger logger = spy(Logger.get("testWriteCloseFileException"));
149 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, logger);
150 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
151 |
152 | // Verify writer
153 | pause();
154 | verify(spyFileWriter[0]).write(DATA);
155 |
156 | // Verify logger
157 | verify(logger, atLeastOnce()).error(anyString());
158 | }
159 |
160 | @Test
161 | public void testOpenFileException() throws Exception
162 | {
163 | // initTest();
164 | final String FILE_NAME = "/tmp/file1";
165 | final String DATA = "{\"data\":\"value\"}";
166 |
167 | // Setup Spy FileWriter
168 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
169 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
170 | throw new IOException("Mock open file error exception");
171 | });
172 |
173 | // Test write
174 | Logger logger = spy(Logger.get("testOpenFileException"));
175 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, logger);
176 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
177 |
178 | pause();
179 | ArgumentCaptor argument = ArgumentCaptor.forClass(String.class);
180 | verify(logger, atLeastOnce()).error(argument.capture());
181 | List values = argument.getAllValues();
182 | assert(values.stream().filter(k -> k.contains(QUERY_ID))
183 | .collect(Collectors.toList()).size() > 0);
184 | }
185 |
186 | @Test
187 | public void testWriteException() throws Exception
188 | {
189 | initTest();
190 | final String FILE_NAME = "/tmp/file1";
191 | final String DATA = "{\"data\":\"value\"}";
192 |
193 | // Setup Spy FileWriter
194 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
195 | final FileWriter[] spyFileWriter = new FileWriter[10];
196 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
197 | String filename = i.getArgument(0);
198 | assert(filename.equals(FILE_NAME));
199 | FileWriter fileWriter = new FileWriter(filename, true);
200 | spyFileWriter[0] = spy(fileWriter);
201 | doThrow(new IOException("Mock write file exception")).when(spyFileWriter[0]).write(anyString());
202 | return spyFileWriter[0];
203 | });
204 |
205 | // Test write
206 | Logger logger = spy(Logger.get("testWriteException"));
207 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, logger);
208 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
209 |
210 | pause();
211 | ArgumentCaptor argument = ArgumentCaptor.forClass(String.class);
212 | verify(logger, atLeastOnce()).error(argument.capture());
213 | List values = argument.getAllValues();
214 | assert(values.stream().filter(k -> k.contains(QUERY_ID))
215 | .collect(Collectors.toList()).size() > 0);
216 | }
217 |
218 | @Test
219 | public void testFullCapacityWrite() throws Exception
220 | {
221 | initTest();
222 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
223 | final FileWriter[] spyFileWriter = new FileWriter[10];
224 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
225 | String filename = i.getArgument(0);
226 | FileWriter fileWriter = new FileWriter(filename, true);
227 | spyFileWriter[0] = spy(fileWriter);
228 | return spyFileWriter[0];
229 | });
230 |
231 | // Should log Queue full error
232 | Logger logger = spy(Logger.get("testFullCapacityWrite"));
233 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, logger);
234 | auditLogFileWriter.stop();
235 | for(int i=0; i<10005; i++) {
236 | auditLogFileWriter.write("/tmp/file1", new SerializedLog(QUERY_ID, "data1"));
237 | }
238 | }
239 |
240 | @Test
241 | public void testMultipleWriteThenClose() throws Exception
242 | {
243 | initTest();
244 | final String FILE_NAME = "/tmp/file1";
245 | final String DATA = "{\"data\":\"value\"}";
246 |
247 | // Setup Spy FileWriter
248 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
249 | final FileWriter[] spyFileWriter = new FileWriter[10];
250 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
251 | String filename = i.getArgument(0);
252 | assert(filename.equals(FILE_NAME));
253 | FileWriter fileWriter = new FileWriter(filename, true);
254 | spyFileWriter[0] = spy(fileWriter);
255 | doAnswer((Answer) var1 -> {
256 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
257 | return "";
258 | }).when(spyFileWriter[0]).write(anyString());
259 | return spyFileWriter[0];
260 | });
261 |
262 | // Test write
263 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, Logger.get("testMultipleWriteThenClose"));
264 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
265 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
266 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
267 |
268 | // Verify
269 | pause();
270 | verify(spyFileWriter[0], times(3)).write(DATA);
271 | verify(spyFileWriter[0], times(1)).close();
272 | }
273 |
274 | @Test
275 | public void testTimeoutReopenFile() throws Exception
276 | {
277 | initTest();
278 | final String FILE_NAME = "/tmp/file1";
279 | final String DATA = "{\"data\":\"value\"}";
280 | final String DATA2 = "{\"data2\":\"value\"}";
281 |
282 | // Setup Spy FileWriter
283 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
284 | final FileWriter[] spyFileWriter = new FileWriter[10];
285 | when(writerFactoryMock.getFileWriter(any(String .class))).thenAnswer(i -> {
286 | String filename = i.getArgument(0);
287 | assert(filename.equals(FILE_NAME));
288 | FileWriter fileWriter = new FileWriter(filename, true);
289 | spyFileWriter[0] = spy(fileWriter);
290 | doAnswer((Answer) var1 -> {
291 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
292 | return "";
293 | }).when(spyFileWriter[0]).write(anyString());
294 | return spyFileWriter[0];
295 | });
296 |
297 | // Test write
298 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, Logger.get("testTimeoutReopenFile"));
299 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
300 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
301 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA));
302 |
303 | // Verify
304 | pause();
305 | verify(spyFileWriter[0], times(3)).write(DATA);
306 | verify(spyFileWriter[0], times(1)).close();
307 |
308 | // Write again after timeout
309 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA2));
310 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA2));
311 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA2));
312 | verify(spyFileWriter[0], times(3)).write(DATA);
313 | verify(spyFileWriter[0], times(1)).close();
314 | }
315 |
316 | @Test
317 | public void testMultiFileMultiWriteThenClose() throws Exception
318 | {
319 | initTest();
320 | final String FILE_NAME = "/tmp/file1";
321 | final String DATA_A1 = "{\"dataA1\":\"value\"}";
322 | final String DATA_A2 = "{\"dataA2\":\"value\"}";
323 |
324 | final String FILE_NAME_2 = "/tmp/file2";
325 | final String DATA_B1 = "{\"data2B1\":\"value2\"}";
326 | final String DATA_B2 = "{\"data2B2\":\"value2\"}";
327 |
328 | // Setup Spy FileWriter
329 | WriterFactory writerFactoryMock = mock(WriterFactory.class);
330 | final FileWriter[] spyFileWriter = new FileWriter[10];
331 | when(writerFactoryMock.getFileWriter(eq(FILE_NAME))).thenAnswer(i -> {
332 | String filename = i.getArgument(0);
333 | assert(filename.equals(FILE_NAME));
334 | FileWriter fileWriter = new FileWriter(filename, true);
335 | spyFileWriter[0] = spy(fileWriter);
336 | doAnswer((Answer) var1 -> {
337 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
338 | return "";
339 | }).when(spyFileWriter[0]).write(anyString());
340 | return spyFileWriter[0];
341 | });
342 | when(writerFactoryMock.getFileWriter(eq(FILE_NAME_2))).thenAnswer(i -> {
343 | String filename = i.getArgument(0);
344 | assert(filename.equals(FILE_NAME_2));
345 | FileWriter fileWriter = new FileWriter(filename, true);
346 | spyFileWriter[1] = spy(fileWriter);
347 | doAnswer((Answer) var1 -> {
348 | System.out.println("WRITING: " + filename + " -- " + var1.getArgument(0).toString().replaceAll("\n", "\\\\n"));
349 | return "";
350 | }).when(spyFileWriter[1]).write(anyString());
351 | return spyFileWriter[1];
352 | });
353 |
354 | // Test write
355 | AuditLogFileWriter auditLogFileWriter = getNewAuditLogFileWriter(writerFactoryMock, Logger.get("testMultiFileMultiWriteThenClose"));
356 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA_A1));
357 | auditLogFileWriter.write(FILE_NAME, new SerializedLog(QUERY_ID, DATA_A2));
358 | auditLogFileWriter.write(FILE_NAME_2, new SerializedLog(QUERY_ID, DATA_B1));
359 | auditLogFileWriter.write(FILE_NAME_2, new SerializedLog(QUERY_ID, DATA_B2));
360 |
361 | // Verify
362 | pause();
363 | verify(spyFileWriter[0], times(1)).write(DATA_A1);
364 | verify(spyFileWriter[0], times(1)).write(DATA_A2);
365 | verify(spyFileWriter[0], times(1)).close();
366 | verify(spyFileWriter[1], times(1)).write(DATA_B1);
367 | verify(spyFileWriter[1], times(1)).write(DATA_B2);
368 | verify(spyFileWriter[1], times(1)).close();
369 | }
370 | }
371 |
--------------------------------------------------------------------------------