--------------------------------------------------------------------------------
/ui/angular/src/typings.d.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | /* SystemJS module definition */
20 | declare var module: NodeModule;
21 |
22 | interface NodeModule {
23 | id: string;
24 | }
25 |
--------------------------------------------------------------------------------
/service/src/test/java/org/apache/griffin/core/job/entity/JobScheduleTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | package org.apache.griffin.core.job.entity;
20 |
21 |
22 | public class JobScheduleTest {
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/event/EventSourceType.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.event;
21 |
22 | public enum EventSourceType {
23 | JOB,
24 | MEASURE
25 | }
26 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/confirmModal/confirmModal.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | .viewrule-content {
21 | border: 1px solid #fff;
22 | border-radius: 4px;
23 | padding: 10px;
24 | }
25 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/event/EventPointcutType.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.event;
21 |
22 | public enum EventPointcutType {
23 | BEFORE,
24 | PENDING,
25 | AFTER
26 | }
27 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_profiling-batch-griffindsl_malformed.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prof_batch",
3 | "process.type": "batch",
4 | "timestamp": 123456,
5 | "data.sources": [
6 | {
7 | "name": "source",
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "dataframe.name": "this_table",
12 | "config": {
13 | "file.name": "src/test/resources/users_info_src.avro"
14 | },
15 | "pre.proc": [
16 | "select * from this_table where user_id < 10014"
17 | ]
18 | }
19 | }
20 | ],
21 | "evaluate.rule": {
22 | "rules": [
23 | {
24 | "dsl.type": "griffin-dsl",
25 | "dq.type": "profiling",
26 | "out.dataframe.name": "prof",
27 | "rule": "abc",
28 | "out": [
29 | {
30 | "type": "metric",
31 | "name": "prof",
32 | "flatten": "array"
33 | }
34 | ]
35 | }
36 | ]
37 | },
38 | "sinks": [
39 | "CONSOLESink"
40 | ]
41 | }
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/event/EventType.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.event;
21 |
22 | public enum EventType {
23 | CREATION_EVENT,
24 | CHANGE_EVENT,
25 | REMOVAL_EVENT
26 | }
27 |
--------------------------------------------------------------------------------
/service/src/main/resources/shell/stop.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | #
3 | # Licensed to the Apache Software Foundation (ASF) under one
4 | # or more contributor license agreements. See the NOTICE file
5 | # distributed with this work for additional information
6 | # regarding copyright ownership. The ASF licenses this file
7 | # to you under the Apache License, Version 2.0 (the
8 | # "License"); you may not use this file except in compliance
9 | # with the License. You may obtain a copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 | #
20 |
21 | PRG="$0"
22 | PRGDIR=`dirname "$PRG"`
23 | GRIFFIN_HOME=`cd "$PRGDIR/.." >/dev/null; pwd`
24 | EXECUTABLE=griffin.sh
25 | exec "$PRGDIR"/"$EXECUTABLE" stop "$@"
26 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/expr/ExprTag.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder.dsl.expr
19 |
20 | trait ExprTag { this: Expr =>
21 | var tag: String = ""
22 | }
23 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_profiling-batch-griffindsl.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prof_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "connector": {
8 | "type": "file",
9 | "config": {
10 | "format": "avro",
11 | "paths": [
12 | "src/test/resources/users_info_src.avro"
13 | ]
14 | },
15 | "pre.proc": [
16 | "select * from this where user_id < 10014"
17 | ]
18 | }
19 | }
20 | ],
21 | "measures": [
22 | {
23 | "name": "profiling_measure",
24 | "type": "profiling",
25 | "data.source": "source",
26 | "config": {
27 | "expr": "first_name, user_id",
28 | "approx.distinct.count": true,
29 | "round.scale": 2
30 | },
31 | "out": [
32 | {
33 | "type": "metric",
34 | "name": "prof_metric",
35 | "flatten": "map"
36 | }
37 | ]
38 | }
39 | ],
40 | "sinks": [
41 | "CONSOLESink",
42 | "customSink"
43 | ]
44 | }
--------------------------------------------------------------------------------
/service/src/main/resources/shell/start.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | #
3 | # Licensed to the Apache Software Foundation (ASF) under one
4 | # or more contributor license agreements. See the NOTICE file
5 | # distributed with this work for additional information
6 | # regarding copyright ownership. The ASF licenses this file
7 | # to you under the Apache License, Version 2.0 (the
8 | # "License"); you may not use this file except in compliance
9 | # with the License. You may obtain a copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 | #
20 |
21 | PRG="$0"
22 | PRGDIR=`dirname "$PRG"`
23 | GRIFFIN_HOME=`cd "$PRGDIR/.." >/dev/null; pwd`
24 | EXECUTABLE=griffin.sh
25 | exec "$PRGDIR"/"$EXECUTABLE" start "$@"
26 |
--------------------------------------------------------------------------------
/ui/angular/src/app/shared/time-utils.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | export class TimeUtils {
20 |
21 | static getBrowserTimeZone() {
22 | let time = -(new Date().getTimezoneOffset() / 60);
23 | return "GMT" + (time >= 0 ? "+"+time : time) + ":00";
24 | }
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/BatchJobRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.apache.griffin.core.job.entity.BatchJob;
23 |
24 | public interface BatchJobRepo extends JobRepo {
25 | }
26 |
--------------------------------------------------------------------------------
/ui/angular/src/app/service/measure-format.service.ts:
--------------------------------------------------------------------------------
1 | import {Injectable} from "@angular/core";
2 | import * as yaml from "js-yaml";
3 |
4 | @Injectable()
5 | export class MeasureFormatService {
6 | constructor() {
7 | }
8 |
9 | format(measure: any, format: Format) {
10 | switch (format) {
11 | case Format.json:
12 | return JSON.stringify(measure, null, 4);
13 | case Format.yaml:
14 | return yaml.dump(measure);
15 |
16 | }
17 | }
18 |
19 | parse(data: string, format: Format) {
20 | switch (format) {
21 | case Format.json:
22 | return JSON.parse(data);
23 | case Format.yaml:
24 | return yaml.load(data);
25 |
26 | }
27 | }
28 |
29 | determineFormat(data: string) {
30 | try {
31 | JSON.parse(data);
32 | return Format.json;
33 | } catch (e) {}
34 | try {
35 | if (yaml.load(data)) {
36 | return Format.yaml;
37 | }
38 | } catch (e) {}
39 | return null;
40 | }
41 | }
42 |
43 | export enum Format {
44 | json = 1,
45 | yaml = 2
46 | }
47 |
48 |
49 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/VirtualJobRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.apache.griffin.core.job.entity.VirtualJob;
23 |
24 | public interface VirtualJobRepo extends JobRepo {
25 | }
26 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/StreamingJobRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.apache.griffin.core.job.entity.StreamingJob;
23 |
24 | public interface StreamingJobRepo extends JobRepo {
25 | }
26 |
--------------------------------------------------------------------------------
/.scalafmt.conf:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
17 |
18 | align = none
19 | align.openParenDefnSite = false
20 | align.openParenCallSite = false
21 | align.tokens = []
22 | optIn = {
23 | configStyleArguments = false
24 | }
25 | danglingParentheses = false
26 | docstrings = JavaDoc
27 | maxColumn = 98
28 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/expr/AliasableExpr.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder.dsl.expr
19 |
20 | trait AliasableExpr extends Expr {
21 |
22 | def alias: Option[String]
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/service/src/test/java/org/apache/griffin/core/util/URLHelper.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.util;
21 |
22 | public class URLHelper {
23 | public static final String API_VERSION_PATH = "/api/v1";
24 | public static final String KAFKA_API_PATH = "/metadata/kafka";
25 | }
26 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/repo/GriffinMeasureRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure.repo;
21 |
22 | import org.apache.griffin.core.measure.entity.GriffinMeasure;
23 |
24 | public interface GriffinMeasureRepo extends MeasureRepo {
25 | }
26 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/repo/ExternalMeasureRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure.repo;
21 |
22 | import org.apache.griffin.core.measure.entity.ExternalMeasure;
23 |
24 | public interface ExternalMeasureRepo extends MeasureRepo {
25 | }
26 |
--------------------------------------------------------------------------------
/ui/angular/src/app/shared/shared.module.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import { NgModule } from '@angular/core';
20 | import { CommonModule } from '@angular/common';
21 |
22 | @NgModule({
23 | imports: [
24 | CommonModule
25 | ],
26 | declarations: [],
27 | exports: [
28 | ]
29 | })
30 | export class SharedModule { }
31 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/dqdefinition/Param.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.dqdefinition
19 |
20 | trait Param extends Serializable {
21 |
22 | /**
23 | * validate param internally
24 | */
25 | def validate(): Unit
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/ui/angular/src/app/health/health.component.html:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/JobDataSegmentRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.apache.griffin.core.job.entity.JobDataSegment;
23 | import org.springframework.data.repository.CrudRepository;
24 |
25 | public interface JobDataSegmentRepo extends CrudRepository {
26 | }
27 |
--------------------------------------------------------------------------------
/ui/angular/src/app/sidebar/truncate.pipe.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {Pipe, PipeTransform} from '@angular/core'
20 |
21 | @Pipe({
22 | name: 'truncate'
23 | })
24 | export class TruncatePipe {
25 | transform(value: string, limit: number): string {
26 | let trail = '...';
27 | return value.length > limit ? value.substring(0, limit) + trail : value;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/measure/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing,
13 | # software distributed under the License is distributed on an
14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | # KIND, either express or implied. See the License for the
16 | # specific language governing permissions and limitations
17 | # under the License.
18 | #
19 | log4j.rootLogger=INFO, stdout
20 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
21 | log4j.appender.stdout.Target=System.out
22 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
23 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%c] - %m%n
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/expr/ExtraConditionExpr.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder.dsl.expr
19 |
20 | case class ExtraConditionExpr(cdtn: String) extends Expr {
21 |
22 | def desc: String = cdtn.toUpperCase
23 |
24 | def coalesceDesc: String = desc
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/repo/EvaluateRuleRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure.repo;
21 |
22 | import org.apache.griffin.core.measure.entity.EvaluateRule;
23 | import org.springframework.data.repository.CrudRepository;
24 |
25 | public interface EvaluateRuleRepo extends CrudRepository {
26 | }
27 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/entity/DqType.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure.entity;
21 |
22 | public enum DqType {
23 | /**
24 | * Currently we support six dimensions of measure.
25 | */
26 | ACCURACY,
27 | PROFILING,
28 | TIMELINESS,
29 | UNIQUENESS,
30 | COMPLETENESS,
31 | CONSISTENCY
32 | }
33 |
--------------------------------------------------------------------------------
/.github/workflows/stale.yml:
--------------------------------------------------------------------------------
1 | name: "Close stale PRs"
2 | on:
3 | schedule:
4 | - cron: "0 0 * * *"
5 |
6 | jobs:
7 | stale:
8 | runs-on: ubuntu-latest
9 | steps:
10 | - name: Check stale PRs
11 | uses: actions/stale@v3
12 | with:
13 | repo-token: ${{ secrets.GITHUB_TOKEN }}
14 | stale-pr-message: >
15 | Automated Message: This PR is being labelled as stale and will be closed in
16 | next 15 days due to lack of activity. To avoid this push new commits or
17 | ask the committers for a review/ resolution.
18 | close-pr-message: >
19 | Automated Message: We're closing this PR because it hasn't been updated in a while.
20 | This isn't a judgement on the merit of the PR in any way. It's just
21 | a way of keeping the PR queue manageable.
22 |
23 | If you'd like to revive this PR, please reopen it and ask a
24 | committer to remove the 'no-pr-activity' tag!
25 | days-before-stale: 30
26 | days-before-close: 15
27 | stale-pr-label: 'no-pr-activity'
28 | exempt-pr-labels: 'awaiting-approval,work-in-progress,wip'
29 |
--------------------------------------------------------------------------------
/ui/angular/src/vendor.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import '@angular/platform-browser';
20 | import '@angular/platform-browser-dynamic';
21 | import '@angular/core';
22 | import '@angular/common';
23 | import '@angular/http';
24 | import '@angular/router';
25 |
26 |
27 | // RxJS
28 | import 'rxjs';
29 |
30 | import 'jquery';
31 | import 'bootstrap/dist/js/bootstrap';
32 | import 'bootstrap-loader';
33 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/datasource/connector/batch/BatchDataConnector.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.datasource.connector.batch
19 |
20 | import org.apache.griffin.measure.datasource.connector.DataConnector
21 |
22 | trait BatchDataConnector extends DataConnector {
23 |
24 | def init(): Unit = {}
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/write/WriteStep.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.write
19 |
20 | import org.apache.griffin.measure.step.DQStep
21 |
22 | trait WriteStep extends DQStep {
23 |
24 | val inputName: String
25 |
26 | val writeTimestampOpt: Option[Long]
27 |
28 | override def getNames: Seq[String] = Nil
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/measure/src/test/resources/env-streaming.json:
--------------------------------------------------------------------------------
1 | {
2 | "spark": {
3 | "log.level": "WARN",
4 | "checkpoint.dir": "hdfs://localhost/test/griffin/cp",
5 | "batch.interval": "2s",
6 | "process.interval": "10s",
7 | "init.clear": true,
8 | "config": {
9 | "spark.master": "local[*]",
10 | "spark.task.maxFailures": 5,
11 | "spark.streaming.kafkaMaxRatePerPartition": 1000,
12 | "spark.streaming.concurrentJobs": 4,
13 | "spark.yarn.maxAppAttempts": 5,
14 | "spark.yarn.am.attemptFailuresValidityInterval": "1h",
15 | "spark.yarn.max.executor.failures": 120,
16 | "spark.yarn.executor.failuresValidityInterval": "1h",
17 | "spark.hadoop.fs.hdfs.impl.disable.cache": true
18 | }
19 | },
20 | "sinks": [
21 | {
22 | "type": "consoleSink",
23 | "config": {
24 | "max.log.lines": 100
25 | }
26 | }
27 | ],
28 | "griffin.checkpoint": [
29 | {
30 | "type": "zk",
31 | "config": {
32 | "hosts": "localhost:2181",
33 | "namespace": "griffin/infocache",
34 | "lock.path": "lock",
35 | "mode": "persist",
36 | "init.clear": true,
37 | "close.clear": false
38 | }
39 | }
40 | ]
41 | }
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/context/ContextId.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.context
19 |
20 | /**
21 | * context id, unique by different timestamp and tag
22 | */
23 | case class ContextId(timestamp: Long, tag: String = "") extends Serializable {
24 | def id: String = {
25 | if (tag.nonEmpty) s"${tag}_$timestamp" else s"$timestamp"
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/measure/src/main/resources/config-batch-path.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "accu_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "config": {
12 | "file.path": "measure/src/test/resources/users_info_src"
13 | }
14 | }
15 | },
16 | {
17 | "name": "target",
18 | "connector": {
19 | "type": "avro",
20 | "version": "1.7",
21 | "config": {
22 | "file.path": "measure/src/test/resources/users_info_target"
23 | }
24 | }
25 | }
26 | ],
27 | "evaluate.rule": {
28 | "rules": [
29 | {
30 | "dsl.type": "griffin-dsl",
31 | "dq.type": "accuracy",
32 | "out.dataframe.name": "accu",
33 | "rule": "source.user_id = target.user_id AND upper(source.first_name) = upper(target.first_name) AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code"
34 | }
35 | ]
36 | },
37 | "sinks": [
38 | "consoleSink",
39 | "elasticSink"
40 | ]
41 | }
42 |
--------------------------------------------------------------------------------
/measure/src/main/resources/config-batch-old.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "accu_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "config": {
12 | "file.name": "measure/src/test/resources/users_info_src.avro"
13 | }
14 | }
15 | },
16 | {
17 | "name": "target",
18 | "connector": {
19 | "type": "avro",
20 | "version": "1.7",
21 | "config": {
22 | "file.name": "measure/src/test/resources/users_info_target.avro"
23 | }
24 | }
25 | }
26 | ],
27 | "evaluate.rule": {
28 | "rules": [
29 | {
30 | "dsl.type": "griffin-dsl",
31 | "dq.type": "accuracy",
32 | "out.dataframe.name": "accu",
33 | "rule": "source.user_id = target.user_id AND upper(source.first_name) = upper(target.first_name) AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code"
34 | }
35 | ]
36 | },
37 | "sinks": [
38 | "consoleSink",
39 | "elasticSink"
40 | ]
41 | }
42 |
--------------------------------------------------------------------------------
/measure/src/test/resources/invalidconfigs/invalidtype_completeness_batch_griffindal.json:
--------------------------------------------------------------------------------
1 | {
2 | "data.sources": [
3 | {
4 | "connectors": [
5 | {
6 | "dataframe.name": "prof_table",
7 | "config": {
8 | "table.name": "efg",
9 | "database": "abc",
10 | "where": "`date`=\"20190825\""
11 | },
12 | "type": "hive"
13 | }
14 | ],
15 | "name": "source"
16 | }
17 | ],
18 | "sinks": [
19 | "CONSOLESink"
20 | ],
21 | "name": "test_griffin_complete",
22 | "evaluate.rule": {
23 | "rules": [
24 | {
25 | "rule": "user",
26 | "out.dataframe.name": "prof",
27 | "dsl.type": "griffin-dsl",
28 | "dq.type": "completeness",
29 | "error.confs": [
30 | {
31 | "column.name": "user",
32 | "type": "abc",
33 | "values": [
34 | "1",
35 | "2",
36 | "hive_none",
37 | ""
38 | ]
39 | }
40 | ],
41 | "out": [
42 | {
43 | "type": "metric",
44 | "flatten": "map"
45 | }
46 | ]
47 | }
48 | ]
49 | },
50 | "process.type": "batch"
51 | }
52 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/rule/rule.component.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {Component, OnInit} from '@angular/core';
20 |
21 | @Component({
22 | selector: 'app-rule',
23 | templateUrl: './rule.component.html',
24 | styleUrls: ['./rule.component.css']
25 | })
26 | export class RuleComponent implements OnInit {
27 |
28 | vaType: string;
29 |
30 | constructor() {
31 | }
32 |
33 | ngOnInit() {
34 | this.vaType = '2';
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/measure-detail/measure-detail.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | .code-viewport {
21 | background-color: #333333;
22 | border: 1px solid #1d1d1d;
23 | border-radius: 4px;
24 | font-family: Menlo, Monaco, Consolas, "Courier New", monospace;
25 | font-size: 13px;
26 | line-height: 1.5;
27 | color: #e4e4e4;
28 | word-break: break-all;
29 | word-wrap: break-word;
30 | }
31 |
32 | .btn-flat {
33 | outline: none !important;
34 | }
35 |
--------------------------------------------------------------------------------
/measure/src/main/resources/config-batch-es.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "accu_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "elasticsearch",
10 | "config": {
11 | "selectionExprs": [
12 | "account_number",
13 | "city",
14 | "gender",
15 | "age > 18"
16 | ],
17 | "filterExprs": [
18 | "account_number < 10"
19 | ],
20 | "paths": [
21 | "bank",
22 | "customer"
23 | ],
24 | "options": {
25 | "es.nodes": "localhost",
26 | "es.port": 9200
27 | }
28 | }
29 | }
30 | }
31 | ],
32 | "evaluate.rule": {
33 | "rules": [
34 | {
35 | "dsl.type": "griffin-dsl",
36 | "dq.type": "completeness",
37 | "out.dataframe.name": "comp_df",
38 | "rule": "account_number, city",
39 | "out": [
40 | {
41 | "type": "metric",
42 | "name": "comp_metric",
43 | "flatten": "map"
44 | }
45 | ]
46 | }
47 | ]
48 | },
49 | "sinks": [
50 | "consoleSink"
51 | ]
52 | }
53 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/entity/JobType.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.entity;
21 |
22 | public enum JobType {
23 |
24 | BATCH("batch"), //
25 | STREAMING("streaming"), //
26 | VIRTUAL("virtual");
27 |
28 | private String name;
29 |
30 | private JobType(String name) {
31 | this.name = name;
32 | }
33 |
34 | public String getName() {
35 | return name;
36 | }
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/MeasureOperator.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure;
21 |
22 |
23 | import org.apache.griffin.core.measure.entity.Measure;
24 | import org.quartz.SchedulerException;
25 |
26 | public interface MeasureOperator {
27 |
28 | Measure create(Measure measure);
29 |
30 | Measure update(Measure measure);
31 |
32 | void delete(Measure measure) throws SchedulerException;
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/step3/step3.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | @import url('../../../../../../node_modules/angular2-toaster/toaster.css');
20 | @import url('../../../measure.component.css');
21 |
22 | fieldset {
23 | height: 60vh;
24 | }
25 |
26 | label.stepDesc {
27 | padding-left: 30px;
28 | padding-right: 30px;
29 | }
30 |
31 | div.btn-container {
32 | width: 100%;
33 | padding-left: 15px;
34 | padding-right: 15px;
35 | clear: both;
36 | }
37 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/context/streaming/metric/Metric.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.context.streaming.metric
19 |
20 | trait Metric extends Serializable {
21 |
22 | type T <: Metric
23 |
24 | def isLegal: Boolean = true
25 |
26 | def update(delta: T): T
27 |
28 | def initial(): Boolean
29 |
30 | def eventual(): Boolean
31 |
32 | def differsFrom(other: T): Boolean
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/ui/angular/src/app/mydashboard/mydashboard.component.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {Component, OnInit} from '@angular/core';
20 | import {Router} from '@angular/router';
21 |
22 | @Component({
23 | selector: 'app-mydashboard',
24 | templateUrl: './mydashboard.component.html',
25 | styleUrls: ['./mydashboard.component.css']
26 | })
27 | export class MydashboardComponent implements OnInit {
28 |
29 | constructor(public router: Router) {
30 | }
31 |
32 | ngOnInit() {
33 | }
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/pr.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | @import url('../../../../../node_modules/angular2-toaster/toaster.css');
20 | @import url('../../measure.component.css');
21 |
22 | div.formStep {
23 | min-height: 65vh;
24 | margin-left: 30px;
25 | margin-right: 30px;
26 | }
27 |
28 | h5.over-title {
29 | margin-left: 30px;
30 | }
31 |
32 | label {
33 | font-weight: normal;
34 | }
35 |
36 | .container {
37 | max-height: 40vh;
38 | overflow-y: scroll;
39 | }
40 |
--------------------------------------------------------------------------------
/ui/angular/src/environments/environment.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | // The file contents for the current environment will overwrite these during build.
20 | // The build system defaults to the dev environment which uses `environment.ts`, but if you do
21 | // `ng build --env=prod` then `environment.prod.ts` will be used instead.
22 | // The list of which env maps to which file can be found in `.angular-cli.json`.
23 |
24 | export const environment = {
25 | production: false,
26 | BACKEND_SERVER: '',
27 | };
28 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_profiling-batch-sparksql.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prof_batch",
3 | "process.type": "batch",
4 | "timestamp": 123456,
5 | "data.sources": [
6 | {
7 | "name": "source",
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "config": {
12 | "file.name": "src/test/resources/users_info_src.avro"
13 | }
14 | }
15 | }
16 | ],
17 | "evaluate.rule": {
18 | "rules": [
19 | {
20 | "dsl.type": "spark-sql",
21 | "out.dataframe.name": "prof",
22 | "rule": "select count(*) as `cnt`, count(distinct `post_code`) as `dis-cnt`, max(user_id) as `max` from source",
23 | "out": [
24 | {
25 | "type": "metric",
26 | "name": "prof"
27 | }
28 | ]
29 | },
30 | {
31 | "dsl.type": "spark-sql",
32 | "out.dataframe.name": "grp",
33 | "rule": "select post_code as `pc`, count(*) as `cnt` from source group by post_code",
34 | "out": [
35 | {
36 | "type": "metric",
37 | "name": "post_group",
38 | "flatten": "array"
39 | }
40 | ]
41 | }
42 | ]
43 | },
44 | "sinks": [
45 | "CONSOLESink",
46 | "ELASTICSEARCHSink"
47 | ]
48 | }
--------------------------------------------------------------------------------
/service/src/main/resources/application-mysql.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing,
13 | # software distributed under the License is distributed on an
14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | # KIND, either express or implied. See the License for the
16 | # specific language governing permissions and limitations
17 | # under the License.
18 | #
19 |
20 | #Data Access Properties
21 | spring.datasource.url=jdbc:mysql://localhost:3306/quartz?autoReconnect=true&useSSL=false
22 | spring.datasource.username=griffin
23 | spring.datasource.password=123456
24 | spring.jpa.generate-ddl=true
25 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver
26 | spring.jpa.show-sql=true
27 | spring.jpa.hibernate.ddl-auto=update
28 |
29 | logging.file=logs/griffin-service.log
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/dsl/expr/Expr.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder.dsl.expr
19 |
20 | /**
21 | * expr parsed by griffin dsl
22 | */
23 | trait Expr extends TreeNode with ExprTag with Serializable {
24 |
25 | def desc: String
26 |
27 | def coalesceDesc: String
28 |
29 | def extractSelf: Expr = this
30 |
31 | // execution
32 | def map(func: Expr => Expr): Expr = func(this)
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/ui/angular/src/environments/environment.js:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | "use strict";
20 | // The file contents for the current environment will overwrite these during build.
21 | // The build system defaults to the dev environment which uses `environment.ts`, but if you do
22 | // `ng build --env=prod` then `environment.prod.ts` will be used instead.
23 | // The list of which env maps to which file can be found in `.angular-cli.json`.
24 | exports.__esModule = true;
25 | exports.environment = {
26 | production: false
27 | };
28 |
--------------------------------------------------------------------------------
/measure/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing,
13 | # software distributed under the License is distributed on an
14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | # KIND, either express or implied. See the License for the
16 | # specific language governing permissions and limitations
17 | # under the License.
18 | #
19 | log4j.rootLogger=INFO, stdout
20 | log4j.logger.org.apache=ERROR
21 | log4j.logger.DataNucleus=ERROR
22 | log4j.logger.org.spark_project=ERROR
23 | log4j.logger.org.apache.griffin=INFO
24 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
25 | log4j.appender.stdout.Target=System.out
26 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
27 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%c] - %m%n
28 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/context/streaming/checkpoint/lock/CheckpointLock.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.context.streaming.checkpoint.lock
19 |
20 | import java.util.concurrent.TimeUnit
21 |
22 | import org.apache.griffin.measure.Loggable
23 |
24 | /**
25 | * lock for checkpoint
26 | */
27 | trait CheckpointLock extends Loggable with Serializable {
28 |
29 | def lock(outtime: Long, unit: TimeUnit): Boolean
30 |
31 | def unlock(): Unit
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/service/src/main/resources/hive-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/DQStepNameGenerator.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder
19 |
20 | import java.util.concurrent.atomic.AtomicLong
21 |
22 | object DQStepNameGenerator {
23 | private val counter: AtomicLong = new AtomicLong(0L)
24 | private val head: String = "step"
25 |
26 | def genName: String = {
27 | s"$head$increment"
28 | }
29 |
30 | private def increment: Long = {
31 | counter.incrementAndGet()
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/service/src/main/resources/hiveserver2-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/service/src/test/resources/metricvalue.json:
--------------------------------------------------------------------------------
1 | {
2 | "took": 47,
3 | "timed_out": false,
4 | "_shards": {
5 | "total": 5,
6 | "successful": 5,
7 | "skipped": 0,
8 | "failed": 0
9 | },
10 | "hits": {
11 | "total": 17,
12 | "max_score": 3.226844,
13 | "hits": [
14 | {
15 | "_index": "griffin",
16 | "_type": "accuracy",
17 | "_id": "RZFP4mgBkZqzqlKSwWtJ",
18 | "_score": 3.226844,
19 | "_source": {
20 | "name": "de_demo_results_comparision",
21 | "tmst": 1549985089648,
22 | "value": {
23 | "total": 74,
24 | "miss": 0,
25 | "matched": 74,
26 | "matchedFraction": 1
27 | },
28 | "metadata": {
29 | "applicationId": "application_1549876136110_0018"
30 | }
31 | }
32 | },
33 | {
34 | "_index": "griffin",
35 | "_type": "accuracy",
36 | "_id": "taMpvmgBfOpRJiYFj5Xg",
37 | "_score": 2.4107988,
38 | "_source": {
39 | "name": "de_demo_results_comparision",
40 | "tmst": 1549378607658,
41 | "value": {
42 | "total": 74,
43 | "miss": 0,
44 | "matched": 74,
45 | "matchedFraction": 1
46 | }
47 | }
48 | }
49 | ]
50 | }
51 | }
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/step4/step4.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | @import url('../../../../../../node_modules/angular2-toaster/toaster.css');
20 | @import url('../../../measure.component.css');
21 |
22 | label.control-label {
23 | margin-top: 7px;
24 | }
25 |
26 | fieldset {
27 | height: 60vh;
28 | }
29 |
30 | label.stepDesc {
31 | padding-left: 30px;
32 | padding-right: 30px;
33 | }
34 |
35 | div.btn-container {
36 | width: 100%;
37 | padding-left: 15px;
38 | padding-right: 15px;
39 | clear: both;
40 | }
41 |
42 |
--------------------------------------------------------------------------------
/service/src/main/resources/hivemetastore-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/service/src/test/java/org/apache/griffin/core/util/PredicatorMock.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.util;
21 |
22 | import org.apache.griffin.core.job.Predicator;
23 | import org.apache.griffin.core.job.entity.SegmentPredicate;
24 |
25 | import java.io.IOException;
26 |
27 | public class PredicatorMock implements Predicator {
28 | public PredicatorMock(SegmentPredicate segmentPredicate) {
29 | }
30 |
31 | @Override
32 | public boolean predicate() throws IOException {
33 | return true;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/event/JobEventHook.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.event;
21 |
22 | import org.apache.griffin.core.exception.GriffinException;
23 | import org.springframework.context.annotation.Configuration;
24 |
25 | @Configuration(value = "GriffinJobEventHook")
26 | public class JobEventHook implements GriffinHook {
27 | @Override
28 | public void onEvent(GriffinEvent event) throws GriffinException {
29 | // This method needs to be reimplemented by event-consuming purpose
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/ui/angular/src/app/service/http.service.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {TestBed, inject} from '@angular/core/testing';
20 |
21 | import {HttpService} from './http.service';
22 | import { LoaderService } from '../loader/loader.service';
23 |
24 | describe('HttpService', () => {
25 | beforeEach(() => {
26 | TestBed.configureTestingModule({
27 | providers: [HttpService, LoaderService]
28 | });
29 | });
30 |
31 | it('should be created', inject([HttpService], (service: HttpService) => {
32 | expect(service).toBeTruthy();
33 | }));
34 | });
35 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_profiling-batch-griffindsl-hive.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prof_batch",
3 | "process.type": "batch",
4 | "timestamp": 123456,
5 | "data.sources": [
6 | {
7 | "name": "source",
8 | "connector": {
9 | "type": "hive",
10 | "version": "1.2",
11 | "config": {
12 | "database": "default",
13 | "table.name": "s1"
14 | }
15 | }
16 | }
17 | ],
18 | "evaluate.rule": {
19 | "rules": [
20 | {
21 | "dsl.type": "griffin-dsl",
22 | "dq.type": "profiling",
23 | "out.dataframe.name": "prof",
24 | "rule": "name, count(*) as cnt from source group by name",
25 | "out": [
26 | {
27 | "type": "metric",
28 | "name": "name_group",
29 | "flatten": "array"
30 | }
31 | ]
32 | },
33 | {
34 | "dsl.type": "griffin-dsl",
35 | "dq.type": "profiling",
36 | "out.dataframe.name": "grp",
37 | "rule": "age, count(*) as cnt from source group by age order by cnt",
38 | "out": [
39 | {
40 | "type": "metric",
41 | "name": "age_group",
42 | "flatten": "array"
43 | }
44 | ]
45 | }
46 | ]
47 | },
48 | "sinks": [
49 | "CONSOLESink",
50 | "ELASTICSEARCHSink"
51 | ]
52 | }
--------------------------------------------------------------------------------
/measure/src/test/resources/_timeliness-batch-griffindsl.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "timeliness_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "connector": {
8 | "type": "avro",
9 | "version": "1.7",
10 | "config": {
11 | "file.name": "src/test/resources/timeliness_data.avro"
12 | }
13 | }
14 | }
15 | ],
16 | "evaluate.rule": {
17 | "rules": [
18 | {
19 | "dsl.type": "griffin-dsl",
20 | "dq.type": "timeliness",
21 | "out.dataframe.name": "timeliness",
22 | "rule": "ts, end_ts",
23 | "details": {
24 | "source": "source",
25 | "latency": "latency",
26 | "total": "total",
27 | "avg": "avg",
28 | "threshold": "3m",
29 | "step": "step",
30 | "count": "cnt",
31 | "step.size": "2m",
32 | "percentile": "percentile",
33 | "percentile.values": [
34 | 0.95
35 | ]
36 | },
37 | "out": [
38 | {
39 | "type": "metric",
40 | "name": "timeliness"
41 | },
42 | {
43 | "type": "record",
44 | "name": "lateRecords"
45 | }
46 | ]
47 | }
48 | ]
49 | },
50 | "sinks": [
51 | "CONSOLESink"
52 | ]
53 | }
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/Predicator.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job;
21 |
22 | import java.io.IOException;
23 |
24 | /**
25 | * Predicator is an object that judges if one condition is met.
26 | * This interface only has one method {@link #predicate()}
27 | */
28 | public interface Predicator {
29 | /**
30 | * predicate a condition
31 | *
32 | * @return True condition is met, otherwise False
33 | * @throws IOException
34 | */
35 | boolean predicate() throws IOException;
36 | }
37 |
--------------------------------------------------------------------------------
/ui/angular/src/app/service/service.service.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {TestBed, inject} from '@angular/core/testing';
20 |
21 | import {ServiceService} from './service.service';
22 | import { LoaderService } from '../loader/loader.service';
23 |
24 | describe('ServiceService', () => {
25 | beforeEach(() => {
26 | TestBed.configureTestingModule({
27 | providers: [ServiceService, LoaderService]
28 | });
29 | });
30 |
31 | it('should be created', inject([ServiceService], (service: ServiceService) => {
32 | expect(service).toBeTruthy();
33 | }));
34 | });
35 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/context/streaming/checkpoint/lock/CheckpointLockSeq.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.context.streaming.checkpoint.lock
19 |
20 | import java.util.concurrent.TimeUnit
21 |
22 | case class CheckpointLockSeq(locks: Seq[CheckpointLock]) extends CheckpointLock {
23 |
24 | def lock(outtime: Long, unit: TimeUnit): Boolean = {
25 | locks.headOption.forall(_.lock(outtime, unit))
26 | }
27 |
28 | def unlock(): Unit = {
29 | locks.headOption.foreach(_.unlock())
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure;
21 |
22 | import java.util.List;
23 | import java.util.Map;
24 |
25 | public interface MeasureOrgService {
26 |
27 | List getOrgs();
28 |
29 | List getMetricNameListByOrg(String org);
30 |
31 | Map> getMeasureNamesGroupByOrg();
32 |
33 | Map>>>
34 | getMeasureWithJobDetailsGroupByOrg(Map>> jobDetailsGroupByMeasure);
35 | }
36 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/BatchDataSourceStepBuilder.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder
19 |
20 | import org.apache.griffin.measure.configuration.dqdefinition._
21 | import org.apache.griffin.measure.context.DQContext
22 | import org.apache.griffin.measure.step.read.ReadStep
23 |
24 | case class BatchDataSourceStepBuilder() extends DataSourceParamStepBuilder {
25 |
26 | def buildReadSteps(context: DQContext, dcParam: DataConnectorParam): Option[ReadStep] = {
27 | None
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/info/GriffinInfoController.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.info;
21 |
22 | import org.springframework.web.bind.annotation.RequestMapping;
23 | import org.springframework.web.bind.annotation.RequestMethod;
24 | import org.springframework.web.bind.annotation.RestController;
25 |
26 | @RestController
27 | @RequestMapping("/api/v1")
28 | public class GriffinInfoController {
29 |
30 | @RequestMapping(value = "/version", method = RequestMethod.GET)
31 | public String greeting() {
32 | return "0.5.0";
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/ui/angular/src/app/loader/loader.service.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {Injectable} from "@angular/core";
20 | import {Subject} from "rxjs/Subject";
21 | import {LoadingState} from "./loader.state";
22 |
23 | @Injectable()
24 | export class LoaderService {
25 |
26 | private loaderSubject = new Subject();
27 | LoadingState = this.loaderSubject.asObservable();
28 |
29 | constructor() {
30 | }
31 |
32 | show() {
33 | this.loaderSubject.next({show: true});
34 | }
35 |
36 | hide() {
37 | this.loaderSubject.next({show: false});
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/StreamingDataSourceStepBuilder.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder
19 |
20 | import org.apache.griffin.measure.configuration.dqdefinition._
21 | import org.apache.griffin.measure.context.DQContext
22 | import org.apache.griffin.measure.step.read.ReadStep
23 |
24 | case class StreamingDataSourceStepBuilder() extends DataSourceParamStepBuilder {
25 |
26 | def buildReadSteps(context: DQContext, dcParam: DataConnectorParam): Option[ReadStep] = {
27 | None
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/ui/angular/src/app/service/chart.service.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {TestBed, inject} from '@angular/core/testing';
20 |
21 | import {ChartService} from './chart.service';
22 | import { LoaderService } from '../loader/loader.service';
23 |
24 | describe('ChartService', () => {
25 | beforeEach(() => {
26 | TestBed.configureTestingModule({
27 | providers: [ChartService, LoaderService]
28 | });
29 | });
30 |
31 | it('should be created', inject([ChartService, LoaderService], (service: ChartService, loaderService: LoaderService) => {
32 | expect(service).toBeTruthy();
33 | }));
34 | });
35 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/repo/DataConnectorRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure.repo;
21 |
22 | import java.util.List;
23 |
24 | import org.apache.griffin.core.measure.entity.DataConnector;
25 | import org.springframework.data.jpa.repository.Query;
26 | import org.springframework.data.repository.CrudRepository;
27 |
28 | public interface DataConnectorRepo extends CrudRepository {
29 |
30 | @Query("select dc from DataConnector dc where dc.name in ?1")
31 | List findByConnectorNames(List names);
32 | }
33 |
--------------------------------------------------------------------------------
/service/src/main/resources/application-dev.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing,
13 | # software distributed under the License is distributed on an
14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | # KIND, either express or implied. See the License for the
16 | # specific language governing permissions and limitations
17 | # under the License.
18 | #
19 | spring.datasource.url=jdbc:h2:mem:quartz;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
20 | spring.datasource.username=griffin
21 | spring.datasource.password=123456
22 | spring.jpa.generate-ddl=true
23 | spring.datasource.driver-class-name=org.h2.Driver
24 | # Create quartz table structure
25 | spring.datasource.schema=classpath:init_quartz_h2.sql
26 | # enable h2 console, default path: http://localhost:8080/h2-console/
27 | spring.h2.console.enabled=true
28 | spring.jpa.show-sql=true
29 |
30 | logging.file=logs/griffin-service.log
--------------------------------------------------------------------------------
/measure/src/test/resources/_completeness_errorconf-batch-griffindsl.json:
--------------------------------------------------------------------------------
1 | {
2 | "data.sources": [
3 | {
4 | "connector": {
5 | "dataframe.name": "prof_table",
6 | "config": {
7 | "table.name": "efg",
8 | "database": "abc",
9 | "where": "`date`=\"20190825\""
10 | },
11 | "type": "hive"
12 | },
13 | "name": "source"
14 | }
15 | ],
16 | "sinks": [
17 | "CONSOLESink"
18 | ],
19 | "name": "test_griffin_complete_lizhao.bd",
20 | "evaluate.rule": {
21 | "rules": [
22 | {
23 | "rule": "user",
24 | "out.dataframe.name": "prof",
25 | "dsl.type": "griffin-dsl",
26 | "dq.type": "completeness",
27 | "error.confs": [
28 | {
29 | "column.name": "user",
30 | "type": "enumeration",
31 | "values": [
32 | "1",
33 | "2",
34 | "hive_none",
35 | ""
36 | ]
37 | },
38 | {
39 | "column.name": "name",
40 | "type": "regex",
41 | "values": [
42 | "^zhanglei.natur\\w{1}$"
43 | ]
44 | }
45 | ],
46 | "out": [
47 | {
48 | "type": "metric",
49 | "flatten": "map"
50 | }
51 | ]
52 | }
53 | ]
54 | },
55 | "process.type": "batch"
56 | }
57 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/enums/GriffinEnum.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.enums
19 |
20 | trait GriffinEnum extends Enumeration {
21 | type GriffinEnum = Value
22 |
23 | val Unknown: Value = Value
24 |
25 | /**
26 | *
27 | * @param name Constant value in String
28 | * @return Enum constant value
29 | */
30 | def withNameWithDefault(name: String): Value =
31 | values
32 | .find(_.toString.toLowerCase == name.replace("-", "").toLowerCase())
33 | .getOrElse(Unknown)
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/measure/src/main/resources/config-batch-preproc.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "Batch-Preprocessing-Example",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "crime_report_source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "file",
10 | "config": {
11 | "format": "text",
12 | "paths": [
13 | "measure/src/main/resources/crime_report.csv"
14 | ]
15 | },
16 | "pre.proc": [
17 | "select split(value, ',') as part from this",
18 | "select part[0] as date_time, part[1] as incident, part[2] as address, part[3] as city, part[4] as zipcode from this",
19 | "select cast(date_time as timestamp) as date_time, incident, address, city, cast(zipcode as int) as zipcode from this"
20 | ]
21 | }
22 | }
23 | ],
24 | "measures": [
25 | {
26 | "name": "completeness_measure",
27 | "type": "completeness",
28 | "data.source": "crime_report_source",
29 | "config": {
30 | "expr": "zipcode is null OR city is null"
31 | },
32 | "out": [
33 | {
34 | "type": "metric",
35 | "name": "comp_metric",
36 | "flatten": "map"
37 | },
38 | {
39 | "type": "record",
40 | "name": "comp_records"
41 | }
42 | ]
43 | }
44 | ],
45 | "sinks": [
46 | "consoleSink"
47 | ]
48 | }
49 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/enums/ProcessType.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.enums
19 |
20 | import org.apache.griffin.measure.configuration.enums
21 |
22 | /**
23 | * process type enum
24 | *
- Process in batch mode
25 | *
- Process in streaming mode
26 | */
27 | object ProcessType extends GriffinEnum {
28 | type ProcessType = Value
29 |
30 | val BatchProcessType: enums.ProcessType.Value = Value("Batch")
31 | val StreamingProcessType: enums.ProcessType.Value = Value("Streaming")
32 | }
33 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/login/LoginService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.login;
21 |
22 | import java.util.Map;
23 |
24 | import org.springframework.http.ResponseEntity;
25 |
26 | /**
27 | * LoginService defines an abstract validation method for login action, you can implement
28 | * it to customize authentication business.
29 | *
30 | * @see org.apache.griffin.core.config.LoginConfig
31 | * @see LoginServiceDefaultImpl
32 | * @see LoginServiceLdapImpl
33 | */
34 | public interface LoginService {
35 |
36 | ResponseEntity> login(Map map);
37 | }
38 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_uniqueness-batch-griffindsl.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "unique_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "config": {
12 | "file.name": "src/test/resources/users_info_src.avro"
13 | }
14 | }
15 | },
16 | {
17 | "name": "target",
18 | "connector": {
19 | "type": "avro",
20 | "version": "1.7",
21 | "config": {
22 | "file.name": "src/test/resources/users_info_src.avro"
23 | }
24 | }
25 | }
26 | ],
27 | "evaluate.rule": {
28 | "rules": [
29 | {
30 | "dsl.type": "griffin-dsl",
31 | "dq.type": "uniqueness",
32 | "out.dataframe.name": "dup",
33 | "rule": "user_id",
34 | "details": {
35 | "source": "source",
36 | "target": "target",
37 | "total": "total",
38 | "unique": "unique",
39 | "dup": "dup",
40 | "num": "num"
41 | },
42 | "out": [
43 | {
44 | "type": "metric",
45 | "name": "unique"
46 | },
47 | {
48 | "type": "record",
49 | "name": "dupRecords"
50 | }
51 | ]
52 | }
53 | ]
54 | },
55 | "sinks": [
56 | "CONSOLESink"
57 | ]
58 | }
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/enums/MeasureTypes.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.enums
19 |
20 | /**
21 | * Types of Griffin measures.
22 | *
- The completeness of data source based on user defined expressions
23 | */
24 | object MeasureTypes extends GriffinEnum {
25 |
26 | type MeasureType = Value
27 |
28 | val Completeness, Duplication, Profiling, Accuracy, SparkSQL, SchemaConformance = Value
29 |
30 | override def withNameWithDefault(name: String): MeasureType = {
31 | super.withNameWithDefault(name)
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/measure/src/test/resources/env-streaming-mongo.json:
--------------------------------------------------------------------------------
1 | {
2 | "spark": {
3 | "log.level": "WARN",
4 | "checkpoint.dir": "hdfs://localhost/test/griffin/cp",
5 | "batch.interval": "2s",
6 | "process.interval": "10s",
7 | "init.clear": true,
8 | "config": {
9 | "spark.master": "local[*]",
10 | "spark.task.maxFailures": 5,
11 | "spark.streaming.kafkaMaxRatePerPartition": 1000,
12 | "spark.streaming.concurrentJobs": 4,
13 | "spark.yarn.maxAppAttempts": 5,
14 | "spark.yarn.am.attemptFailuresValidityInterval": "1h",
15 | "spark.yarn.max.executor.failures": 120,
16 | "spark.yarn.executor.failuresValidityInterval": "1h",
17 | "spark.hadoop.fs.hdfs.impl.disable.cache": true
18 | }
19 | },
20 | "sinks": [
21 | {
22 | "name": "consoleSink",
23 | "type": "console",
24 | "config": {
25 | "max.log.lines": 100
26 | }
27 | },
28 | {
29 | "type": "mongoSink",
30 | "config": {
31 | "url": "10.149.247.156",
32 | "database": "test",
33 | "collection": "sss"
34 | }
35 | }
36 | ],
37 | "griffin.checkpoint": [
38 | {
39 | "type": "zk",
40 | "config": {
41 | "hosts": "localhost:2181",
42 | "namespace": "griffin/infocache",
43 | "lock.path": "lock",
44 | "mode": "persist",
45 | "init.clear": true,
46 | "close.clear": false
47 | }
48 | }
49 | ]
50 | }
--------------------------------------------------------------------------------
/ui/angular/src/main.js:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | "use strict";
20 | exports.__esModule = true;
21 | var core_1 = require("@angular/core");
22 | var platform_browser_dynamic_1 = require("@angular/platform-browser-dynamic");
23 | var app_module_1 = require("./app/app.module");
24 | var environment_1 = require("./environments/environment");
25 | require("jquery");
26 | // import 'bootstrap';
27 | // import 'bootstrap/dist/css/bootstrap.css';
28 | require("bootstrap/dist/css/bootstrap.css");
29 | if (environment_1.environment.production) {
30 | core_1.enableProdMode();
31 | }
32 | platform_browser_dynamic_1.platformBrowserDynamic().bootstrapModule(app_module_1.AppModule);
33 |
--------------------------------------------------------------------------------
/ui/angular/src/main.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {enableProdMode} from '@angular/core';
20 | import {platformBrowserDynamic} from '@angular/platform-browser-dynamic';
21 | import {AppModule} from './app/app.module';
22 | import {environment} from './environments/environment';
23 | import {RouterModule, Routes} from '@angular/router';
24 | import {HttpClientModule} from '@angular/common/http';
25 | import {BrowserAnimationsModule} from '@angular/platform-browser/animations';
26 |
27 | import 'bootstrap/dist/css/bootstrap.css';
28 |
29 | if (environment.production) {
30 | enableProdMode();
31 | }
32 |
33 | platformBrowserDynamic().bootstrapModule(AppModule);
34 |
35 |
--------------------------------------------------------------------------------
/ui/angular/src/app/dataasset/dataasset.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | .icon {
20 | color: #fff;
21 | position: absolute;
22 | left: 50%;
23 | top: 20%;
24 | }
25 |
26 | .co {
27 | border-collapse: separate;
28 | }
29 |
30 | .co > tbody > tr:nth-of-type(even) {
31 | background-color: #1f1f1f;
32 | }
33 |
34 | .co > tbody > tr:nth-of-type(odd) {
35 | background-color: #080808;
36 | }
37 |
38 | .table-striped > tbody > tr {
39 | background-color: #1f1f1f;
40 | border: 1px solid transparent;
41 | }
42 |
43 | .table > tbody + tbody {
44 | border: 1px solid transparent;
45 | }
46 |
47 | .reco > tbody:nth-of-type(even) > tr {
48 | background-color: #080808;
49 | }
50 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.metastore.hive;
21 |
22 | import java.util.List;
23 | import java.util.Map;
24 |
25 | import org.apache.hadoop.hive.metastore.api.Table;
26 |
27 | public interface HiveMetaStoreService {
28 |
29 | Iterable getAllDatabases();
30 |
31 | Iterable getAllTableNames(String dbName);
32 |
33 | Map> getAllTableNames();
34 |
35 | List
getAllTable(String db);
36 |
37 | Map> getAllTable();
38 |
39 | Table getTable(String dbName, String tableName);
40 |
41 | void evictHiveCache();
42 | }
43 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/ConstantColumns.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder
19 |
20 | /**
21 | * for griffin dsl rules, the constant columns might be used during calculation,
22 | */
23 | object ConstantColumns {
24 | val tmst = "__tmst"
25 | val metric = "__metric"
26 | val record = "__record"
27 | val empty = "__empty"
28 |
29 | val beginTs = "__begin_ts"
30 | val endTs = "__end_ts"
31 |
32 | val distinct = "__distinct"
33 |
34 | val rowNumber = "__rn"
35 |
36 | val columns: List[String] =
37 | List[String](tmst, metric, record, empty, beginTs, endTs, distinct, rowNumber)
38 | }
39 |
--------------------------------------------------------------------------------
/service/src/test/resources/quartz.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing,
13 | # software distributed under the License is distributed on an
14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | # KIND, either express or implied. See the License for the
16 | # specific language governing permissions and limitations
17 | # under the License.
18 | #
19 | org.quartz.scheduler.instanceName=spring-boot-quartz-test
20 | org.quartz.scheduler.instanceId=AUTO
21 | org.quartz.threadPool.threadCount=5
22 | org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX
23 | # be careful to configure driver with PostgreSQLDelegate in postgresql not StdJDBCDelegate in mysql
24 | org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
25 | org.quartz.jobStore.useProperties=true
26 | org.quartz.jobStore.misfireThreshold=60000
27 | org.quartz.jobStore.tablePrefix=QRTZ_
28 | org.quartz.jobStore.isClustered=true
29 | org.quartz.jobStore.clusterCheckinInterval=20000
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/BaseJpaRepository.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.springframework.data.jpa.repository.JpaRepository;
23 | import org.springframework.data.repository.NoRepositoryBean;
24 |
25 | @NoRepositoryBean
26 | public interface BaseJpaRepository extends JpaRepository {
27 | /**
28 | * This method is to make findOne method from spring boot 1.5.x compatible with spring 2.x
29 | *
30 | * @param id id
31 | * @return object of specified id, return null if cannot find the id
32 | */
33 | default T findOne(ID id) {
34 | return (T) findById(id).orElse(null);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/griffin-doc/docker/compose/docker-compose-batch.yml:
--------------------------------------------------------------------------------
1 | #Licensed to the Apache Software Foundation (ASF) under one
2 | #or more contributor license agreements. See the NOTICE file
3 | #distributed with this work for additional information
4 | #regarding copyright ownership. The ASF licenses this file
5 | #to you under the Apache License, Version 2.0 (the
6 | #"License"); you may not use this file except in compliance
7 | #with the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | #Unless required by applicable law or agreed to in writing,
12 | #software distributed under the License is distributed on an
13 | #"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 | #KIND, either express or implied. See the License for the
15 | #specific language governing permissions and limitations
16 | #under the License.
17 |
18 | griffin:
19 | image: apachegriffin/griffin_spark2:0.3.0
20 | hostname: griffin
21 | links:
22 | - es
23 | environment:
24 | ES_HOSTNAME: es
25 | volumes:
26 | - /var/lib/mysql
27 | ports:
28 | - 32122:2122
29 | - 38088:8088 # yarn rm web ui
30 | - 33306:3306 # mysql
31 | - 35432:5432 # postgres
32 | - 38042:8042 # yarn nm web ui
33 | - 39083:9083 # hive-metastore
34 | - 38998:8998 # livy
35 | - 38080:8080 # griffin ui
36 | tty: true
37 | container_name: griffin
38 |
39 | es:
40 | image: apachegriffin/elasticsearch
41 | hostname: es
42 | ports:
43 | - 39200:9200
44 | - 39300:9300
45 | container_name: es
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/job/DQJob.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.job
19 |
20 | import scala.util.{Failure, Success, Try}
21 |
22 | import org.apache.griffin.measure.context.DQContext
23 | import org.apache.griffin.measure.step.DQStep
24 |
25 | case class DQJob(dqSteps: Seq[DQStep]) extends Serializable {
26 |
27 | def execute(context: DQContext): Try[Boolean] = {
28 | dqSteps
29 | .map(_.execute(context))
30 | .foldLeft(Try(true)) { (ret, stepResult) =>
31 | (ret, stepResult) match {
32 | case (Success(_), nextResult) => nextResult
33 | case (Failure(_), _) => ret
34 | }
35 | }
36 | }
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/entity/VirtualJob.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.entity;
21 |
22 | import javax.persistence.DiscriminatorValue;
23 | import javax.persistence.Entity;
24 |
25 | @Entity
26 | @DiscriminatorValue("virtualJob")
27 | public class VirtualJob extends AbstractJob {
28 | private static final long serialVersionUID = 1130038058433818835L;
29 |
30 | @Override
31 | public String getType() {
32 | return JobType.VIRTUAL.getName();
33 | }
34 |
35 | public VirtualJob() {
36 | super();
37 | }
38 |
39 | public VirtualJob(String jobName, Long measureId, String metricName) {
40 | super(jobName, measureId, metricName);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/ui/angular/src/app/service/user.service.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {Injectable} from "@angular/core";
20 |
21 | @Injectable()
22 | export class UserService {
23 | ntAccount: string;
24 | timestamp: Date;
25 |
26 | setCookie(name, value, days) {
27 | let expires;
28 | if (days) {
29 | var date = new Date();
30 | date.setTime(date.getTime() + days * 24 * 60 * 60 * 1000);
31 | expires = "; expires=" + date.toUTCString();
32 | } else {
33 | expires = "";
34 | }
35 | document.cookie = name + "=" + value + expires + "; path=/";
36 | }
37 |
38 | getCookie(key) {
39 | var keyValue = document.cookie.match("(^|;) ?" + key + "=([^;]*)(;|$)");
40 | return keyValue ? keyValue[2] : null;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/measure/src/test/resources/_sparksql-batch-griffindsl.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prof_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "connector": {
8 | "type": "file",
9 | "config": {
10 | "format": "avro",
11 | "paths": [
12 | "src/test/resources/users_info_src.avro"
13 | ]
14 | },
15 | "pre.proc": [
16 | "select * from this where user_id < 10014"
17 | ]
18 | }
19 | }
20 | ],
21 | "measures": [
22 | {
23 | "name": "query_measure1",
24 | "type": "sparkSQL",
25 | "data.source": "source",
26 | "config": {
27 | "expr": "select user_id, count(*) as cnt from source group by user_id",
28 | "bad.record.definition": "cnt > 1"
29 | },
30 | "out": [
31 | {
32 | "type": "metric",
33 | "name": "sql_metric",
34 | "flatten": "map"
35 | }
36 | ]
37 | },
38 | {
39 | "name": "query_measure2",
40 | "type": "sparkSQL",
41 | "data.source": "source",
42 | "config": {
43 | "expr": "select post_code, count(*) as cnt from source group by post_code order by cnt desc",
44 | "bad.record.definition": "cnt > 1"
45 | },
46 | "out": [
47 | {
48 | "type": "metric",
49 | "name": "sql_metric",
50 | "flatten": "map"
51 | }
52 | ]
53 | }
54 | ],
55 | "sinks": [
56 | "CONSOLESink",
57 | "customSink"
58 | ]
59 | }
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/measure/MeasureService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.measure;
21 |
22 |
23 | import java.util.List;
24 |
25 | import org.apache.griffin.core.measure.entity.Measure;
26 | import org.quartz.SchedulerException;
27 |
28 | public interface MeasureService {
29 |
30 | List getAllAliveMeasures(String type);
31 |
32 | Measure getMeasureById(long id);
33 |
34 | void deleteMeasureById(Long id) throws SchedulerException;
35 |
36 | void deleteMeasures() throws SchedulerException;
37 |
38 | Measure updateMeasure(Measure measure);
39 |
40 | List getAliveMeasuresByOwner(String owner);
41 |
42 | Measure createMeasure(Measure measure);
43 | }
44 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/rule/rule.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | .table > thead > tr.success > td {
21 | background-color: #77b300;
22 | }
23 |
24 | mark {
25 | background-color: #ff8800;
26 | padding: .2em;
27 | }
28 |
29 | h5 {
30 | font-size: 20px;
31 | }
32 |
33 | .y-scrollable::-webkit-scrollbar-track {
34 | -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, 0.3);
35 | border-radius: 10px;
36 | background-color: #F5F5F5;
37 | }
38 |
39 | .y-scrollable::-webkit-scrollbar {
40 | width: 5px;
41 | border-radius: 10px;
42 | background-color: #F5F5F5;
43 | }
44 |
45 | .y-scrollable::-webkit-scrollbar-thumb {
46 | border-radius: 10px;
47 | -webkit-box-shadow: inset 0 0 6px rgba(0, 0, 0, .3);
48 | background-color: #AAAAAA;
49 | }
50 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/metastore/kafka/KafkaSchemaService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.metastore.kafka;
21 |
22 | import io.confluent.kafka.schemaregistry.client.rest.entities.Config;
23 | import io.confluent.kafka.schemaregistry.client.rest.entities.Schema;
24 | import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaString;
25 |
26 | public interface KafkaSchemaService {
27 | SchemaString getSchemaString(Integer id);
28 |
29 | Iterable getSubjects();
30 |
31 | Iterable getSubjectVersions(String subject);
32 |
33 | Schema getSubjectSchema(String subject, String version);
34 |
35 | Config getTopLevelConfig();
36 |
37 | Config getSubjectLevelConfig(String subject);
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/DQStep.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step
19 |
20 | import scala.util.Try
21 |
22 | import org.apache.griffin.measure.Loggable
23 | import org.apache.griffin.measure.context.DQContext
24 |
25 | trait DQStep extends Loggable {
26 |
27 | val name: String
28 |
29 | /**
30 | * @return execution success
31 | */
32 | def execute(context: DQContext): Try[Boolean]
33 |
34 | def getNames: Seq[String] = name :: Nil
35 |
36 | }
37 |
38 | object DQStepStatus extends Enumeration {
39 | val PENDING: DQStepStatus.Value = Value
40 | val RUNNING: DQStepStatus.Value = Value
41 | val COMPLETE: DQStepStatus.Value = Value
42 | val FAILED: DQStepStatus.Value = Value
43 | }
44 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/step2/step2.component.css:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | @import url('../../../../../../node_modules/angular2-toaster/toaster.css');
20 | @import url('../../../measure.component.css');
21 |
22 | fieldset {
23 | height: 60vh;
24 | }
25 |
26 | label.stepDesc {
27 | padding-left: 30px;
28 | padding-right: 30px;
29 | }
30 |
31 | div.btn-container {
32 | width: 100%;
33 | padding-left: 15px;
34 | padding-right: 15px;
35 | clear: both;
36 | }
37 |
38 | div.btn-container > button.prev-step {
39 | float: left;
40 | }
41 |
42 | div.btn-container > button.next-step {
43 | float: right;
44 | }
45 |
46 | div.block {
47 | margin: 20px 10px;
48 | height: 26px;
49 | }
50 |
51 | div.block input {
52 | width: 50%;
53 | display: inline-block;
54 | }
55 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/dqdefinition/reader/ParamJsonReader.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.dqdefinition.reader
19 |
20 | import scala.reflect.ClassTag
21 | import scala.util.Try
22 |
23 | import org.apache.griffin.measure.configuration.dqdefinition.Param
24 | import org.apache.griffin.measure.utils.JsonUtil
25 |
26 | /**
27 | * read params from json string directly
28 | *
29 | * @param jsonString
30 | */
31 | case class ParamJsonReader(jsonString: String) extends ParamReader {
32 |
33 | def readConfig[T <: Param](implicit m: ClassTag[T]): Try[T] = {
34 | Try {
35 | val param = JsonUtil.fromJson[T](jsonString)
36 | validate(param)
37 | }
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/read/UnionReadStep.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.read
19 |
20 | import org.apache.spark.sql._
21 |
22 | import org.apache.griffin.measure.context.DQContext
23 | import org.apache.griffin.measure.utils.DataFrameUtil._
24 |
25 | case class UnionReadStep(name: String, readSteps: Seq[ReadStep]) extends ReadStep {
26 |
27 | val config: Map[String, Any] = Map()
28 | val cache: Boolean = false
29 |
30 | def read(context: DQContext): Option[DataFrame] = {
31 | val dfOpts = readSteps.map { readStep =>
32 | readStep.read(context)
33 | }
34 | if (dfOpts.nonEmpty) {
35 | dfOpts.reduce((a, b) => unionDfOpts(a, b))
36 | } else None
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/metric/MetricStore.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.metric;
21 |
22 | import java.io.IOException;
23 | import java.util.List;
24 |
25 | import org.apache.griffin.core.metric.model.MetricValue;
26 | import org.springframework.http.ResponseEntity;
27 |
28 | public interface MetricStore {
29 |
30 | List getMetricValues(String metricName, int from, int size,
31 | long tmst) throws IOException;
32 |
33 | ResponseEntity addMetricValues(List metricValues)
34 | throws IOException;
35 |
36 | ResponseEntity deleteMetricValues(String metricName) throws IOException;
37 |
38 | MetricValue getMetric(String applicationId) throws IOException;
39 | }
40 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/metric/MetricService.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.metric;
21 |
22 |
23 | import java.util.List;
24 | import java.util.Map;
25 |
26 | import org.apache.griffin.core.metric.model.Metric;
27 | import org.apache.griffin.core.metric.model.MetricValue;
28 | import org.springframework.http.ResponseEntity;
29 |
30 | public interface MetricService {
31 |
32 | Map> getAllMetrics();
33 |
34 | List getMetricValues(String metricName, int offset, int size,
35 | long tmst);
36 |
37 | ResponseEntity addMetricValues(List values);
38 |
39 | ResponseEntity deleteMetricValues(String metricName);
40 |
41 | MetricValue findMetric(Long id);
42 | }
43 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more contributor license
2 | # agreements. See the NOTICE file distributed with this work for additional information regarding
3 | # copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
4 | # "License"); you may not use this file except in compliance with the License. You may obtain a
5 | # copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software distributed under the License
10 | # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11 | # or implied. See the License for the specific language governing permissions and limitations under
12 | # the License.
13 |
14 | language: java scala node_js
15 |
16 | services:
17 | - docker
18 |
19 | env:
20 | - COMPOSE_FILE=griffin-doc/docker/compose/docker-compose-batch.yml
21 |
22 | jdk:
23 | - openjdk8
24 | scala:
25 | - 2.10.6
26 | node_js:
27 | - "6"
28 | git:
29 | quiet: true
30 | cache:
31 | directories:
32 | - $HOME/.m2
33 |
34 | install:
35 | - npm install -g bower
36 |
37 | before_script:
38 | - docker-compose up -d
39 |
40 |
41 | # keep 30, need change according to ci logs.
42 | script:
43 | - export MAVEN_SKIP_RC=true
44 | - export MAVEN_OPTS=" -Xmx1g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
45 | - mvn -B clean test -Dlogging.level.org.springframework=WARN
46 |
47 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/context/streaming/checkpoint/offset/OffsetCheckpoint.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.context.streaming.checkpoint.offset
19 |
20 | import org.apache.griffin.measure.Loggable
21 | import org.apache.griffin.measure.context.streaming.checkpoint.lock.CheckpointLock
22 |
23 | trait OffsetCheckpoint extends Loggable with Serializable {
24 |
25 | def init(): Unit
26 | def available(): Boolean
27 | def close(): Unit
28 |
29 | def cache(kvs: Map[String, String]): Unit
30 | def read(keys: Iterable[String]): Map[String, String]
31 | def delete(keys: Iterable[String]): Unit
32 | def clear(): Unit
33 |
34 | def listKeys(path: String): List[String]
35 |
36 | def genLock(s: String): CheckpointLock
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/event/GriffinHook.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.event;
21 |
22 | import org.apache.griffin.core.exception.GriffinException;
23 |
24 | /**
25 | * The Hook interface for receiving internal events.
26 | * The class that is interested in processing an event
27 | * implements this interface, and the object created with that
28 | * class is registered to griffin, using the configuration.
29 | * When the event occurs, that object's onEvent method is
30 | * invoked.
31 | *
32 | * @author Eugene Liu
33 | * @since 0.3
34 | */
35 | public interface GriffinHook {
36 | /**
37 | * Invoked when an action occurs.
38 | *
39 | * @see GriffinEvent
40 | */
41 | void onEvent(GriffinEvent event) throws GriffinException;
42 | }
43 |
--------------------------------------------------------------------------------
/service/src/main/resources/env/env_streaming.json:
--------------------------------------------------------------------------------
1 | {
2 | "spark": {
3 | "log.level": "WARN",
4 | "checkpoint.dir": "hdfs:///griffin/checkpoint/${JOB_NAME}",
5 | "init.clear": true,
6 | "batch.interval": "1m",
7 | "process.interval": "5m",
8 | "config": {
9 | "spark.default.parallelism": 4,
10 | "spark.task.maxFailures": 5,
11 | "spark.streaming.kafkaMaxRatePerPartition": 1000,
12 | "spark.streaming.concurrentJobs": 4,
13 | "spark.yarn.maxAppAttempts": 5,
14 | "spark.yarn.am.attemptFailuresValidityInterval": "1h",
15 | "spark.yarn.max.executor.failures": 120,
16 | "spark.yarn.executor.failuresValidityInterval": "1h",
17 | "spark.hadoop.fs.hdfs.impl.disable.cache": true
18 | }
19 | },
20 | "sinks": [
21 | {
22 | "type": "CONSOLE",
23 | "config": {
24 | "max.log.lines": 100
25 | }
26 | },
27 | {
28 | "type": "HDFS",
29 | "config": {
30 | "path": "hdfs:///griffin/persist",
31 | "max.persist.lines": 10000,
32 | "max.lines.per.file": 10000
33 | }
34 | },
35 | {
36 | "type": "ELASTICSEARCH",
37 | "config": {
38 | "method": "post",
39 | "api": "http://es:9200/griffin/accuracy"
40 | }
41 | }
42 | ],
43 | "griffin.checkpoint": [
44 | {
45 | "type": "zk",
46 | "config": {
47 | "hosts": "zk:2181",
48 | "namespace": "griffin/infocache",
49 | "lock.path": "lock",
50 | "mode": "persist",
51 | "init.clear": true,
52 | "close.clear": false
53 | }
54 | }
55 | ]
56 | }
57 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/configuration/enums/WriteMode.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.configuration.enums
19 |
20 | /**
21 | * write mode when write metrics and records
22 | */
23 | sealed trait WriteMode {}
24 |
25 | object WriteMode {
26 | def defaultMode(procType: ProcessType.ProcessType): WriteMode = {
27 | procType match {
28 | case ProcessType.BatchProcessType => SimpleMode
29 | case ProcessType.StreamingProcessType => TimestampMode
30 | }
31 | }
32 | }
33 |
34 | /**
35 | * simple mode: write metrics and records directly
36 | */
37 | case object SimpleMode extends WriteMode {}
38 |
39 | /**
40 | * timestamp mode: write metrics and records with timestamp information
41 | */
42 | case object TimestampMode extends WriteMode {}
43 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/entity/StreamingJob.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.entity;
21 |
22 | import javax.persistence.DiscriminatorValue;
23 | import javax.persistence.Entity;
24 |
25 | @Entity
26 | @DiscriminatorValue("griffinStreamingJob")
27 | public class StreamingJob extends AbstractJob {
28 | private static final long serialVersionUID = 3292253488392308505L;
29 |
30 | @Override
31 | public String getType() {
32 | return JobType.STREAMING.getName();
33 | }
34 |
35 | public StreamingJob() {
36 | }
37 |
38 | public StreamingJob(Long measureId, String jobName, String name,
39 | String group, boolean deleted) {
40 | super(measureId, jobName, name, group, deleted);
41 | this.metricName = jobName;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/utils/DataFrameUtil.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.utils
19 |
20 | import org.apache.spark.sql.DataFrame
21 | import org.apache.spark.sql.functions._
22 |
23 | object DataFrameUtil {
24 |
25 | def unionDfOpts(dfOpt1: Option[DataFrame], dfOpt2: Option[DataFrame]): Option[DataFrame] = {
26 | (dfOpt1, dfOpt2) match {
27 | case (Some(df1), Some(df2)) => Some(unionByName(df1, df2))
28 | case (Some(_), _) => dfOpt1
29 | case (_, Some(_)) => dfOpt2
30 | case _ => None
31 | }
32 | }
33 |
34 | def unionByName(a: DataFrame, b: DataFrame): DataFrame = {
35 | val columns = a.columns.toSet.intersect(b.columns.toSet).map(col).toSeq
36 | a.select(columns: _*).union(b.select(columns: _*))
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/repo/JobRepo.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 |
20 | package org.apache.griffin.core.job.repo;
21 |
22 | import org.apache.griffin.core.job.entity.AbstractJob;
23 | import org.springframework.data.jpa.repository.Query;
24 |
25 | import java.util.List;
26 |
27 | public interface JobRepo extends BaseJpaRepository {
28 |
29 | @Query("select count(j) from #{#entityName} j " +
30 | "where j.jobName = ?1 and j.deleted = ?2")
31 | int countByJobNameAndDeleted(String jobName, Boolean deleted);
32 |
33 | List findByDeleted(boolean deleted);
34 |
35 | List findByJobNameAndDeleted(String jobName, boolean deleted);
36 |
37 | List findByMeasureIdAndDeleted(Long measureId, boolean deleted);
38 |
39 | T findByIdAndDeleted(Long jobId, boolean deleted);
40 | }
41 |
--------------------------------------------------------------------------------
/ui/angular/src/app/login/login.component.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
20 |
21 | import { AppModule } from '../app.module';
22 | import {LoginComponent} from './login.component';
23 |
24 | describe('LoginComponent', () => {
25 | let component: LoginComponent;
26 | let fixture: ComponentFixture;
27 |
28 | beforeEach(async(() => {
29 | TestBed.configureTestingModule({
30 | imports: [ AppModule ],
31 | declarations: []
32 | })
33 | .compileComponents();
34 | }));
35 |
36 | beforeEach(() => {
37 | fixture = TestBed.createComponent(LoginComponent);
38 | component = fixture.componentInstance;
39 | fixture.detectChanges();
40 | });
41 |
42 | it('should be created', () => {
43 | expect(component).toBeTruthy();
44 | });
45 | });
46 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/job/entity/JobHealth.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | package org.apache.griffin.core.job.entity;
20 |
21 |
22 | public class JobHealth {
23 | private int healthyJobCount;
24 | private int jobCount;
25 |
26 | public int getHealthyJobCount() {
27 | return healthyJobCount;
28 | }
29 |
30 | public void setHealthyJobCount(int healthyJobCount) {
31 | this.healthyJobCount = healthyJobCount;
32 | }
33 |
34 | public int getJobCount() {
35 | return jobCount;
36 | }
37 |
38 | public void setJobCount(int jobCount) {
39 | this.jobCount = jobCount;
40 | }
41 |
42 | public JobHealth(int healthyJobCount, int jobCount) {
43 | this.healthyJobCount = healthyJobCount;
44 | this.jobCount = jobCount;
45 | }
46 |
47 | public JobHealth() {
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/raw/raw.component.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
20 |
21 | import { AppModule } from '../../../app.module';
22 | import {RawComponent} from './raw.component';
23 |
24 | describe('RawComponent', () => {
25 | let component: RawComponent;
26 | let fixture: ComponentFixture;
27 |
28 | beforeEach(async(() => {
29 | TestBed.configureTestingModule({
30 | imports: [ AppModule ],
31 | declarations: []
32 | })
33 | .compileComponents();
34 | }));
35 |
36 | beforeEach(() => {
37 | fixture = TestBed.createComponent(RawComponent);
38 | component = fixture.componentInstance;
39 | fixture.detectChanges();
40 | });
41 |
42 | it('should be created', () => {
43 | expect(component).toBeTruthy();
44 | });
45 | });
46 |
--------------------------------------------------------------------------------
/ui/angular/src/app/measure/create-measure/pr/rule/rule.component.spec.ts:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | import {async, ComponentFixture, TestBed} from '@angular/core/testing';
20 |
21 | import { AppModule } from '../../../../app.module';
22 | import {RuleComponent} from './rule.component';
23 |
24 | describe('RuleComponent', () => {
25 | let component: RuleComponent;
26 | let fixture: ComponentFixture;
27 |
28 | beforeEach(async(() => {
29 | TestBed.configureTestingModule({
30 | imports: [ AppModule ],
31 | declarations: []
32 | })
33 | .compileComponents();
34 | }));
35 |
36 | beforeEach(() => {
37 | fixture = TestBed.createComponent(RuleComponent);
38 | component = fixture.componentInstance;
39 | fixture.detectChanges();
40 | });
41 |
42 | it('should be created', () => {
43 | expect(component).toBeTruthy();
44 | });
45 | });
46 |
--------------------------------------------------------------------------------
/ui/angular/protractor.conf.js:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | // Protractor configuration file, see link for more information
20 | // https://github.com/angular/protractor/blob/master/lib/config.ts
21 |
22 | const {SpecReporter} = require('jasmine-spec-reporter');
23 |
24 | exports.config = {
25 | allScriptsTimeout: 11000,
26 | specs: [
27 | './e2e/**/*.e2e-spec.ts'
28 | ],
29 | capabilities: {
30 | 'browserName': 'chrome'
31 | },
32 | directConnect: true,
33 | baseUrl: 'http://localhost:4200/',
34 | framework: 'jasmine',
35 | jasmineNodeOpts: {
36 | showColors: true,
37 | defaultTimeoutInterval: 30000,
38 | print: function () {
39 | }
40 | },
41 | onPrepare() {
42 | require('ts-node').register({
43 | project: 'e2e/tsconfig.e2e.json'
44 | });
45 | jasmine.getEnv().addReporter(new SpecReporter({spec: {displayStacktrace: true}}));
46 | }
47 | };
48 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/datasource/cache/StreamingCacheOrcClient.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.datasource.cache
19 |
20 | import org.apache.spark.sql._
21 |
22 | import org.apache.griffin.measure.datasource.TimestampStorage
23 |
24 | /**
25 | * data source cache in orc format
26 | */
27 | case class StreamingCacheOrcClient(
28 | sparkSession: SparkSession,
29 | param: Map[String, Any],
30 | dsName: String,
31 | index: Int,
32 | timestampStorage: TimestampStorage)
33 | extends StreamingCacheClient {
34 |
35 | protected def writeDataFrame(dfw: DataFrameWriter[Row], path: String): Unit = {
36 | info(s"write path: $path")
37 | dfw.orc(path)
38 | }
39 |
40 | protected def readDataFrame(dfr: DataFrameReader, path: String): DataFrame = {
41 | dfr.orc(path)
42 | }
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/datasource/cache/StreamingCacheJsonClient.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.datasource.cache
19 |
20 | import org.apache.spark.sql._
21 |
22 | import org.apache.griffin.measure.datasource.TimestampStorage
23 |
24 | /**
25 | * data source cache in json format
26 | */
27 | case class StreamingCacheJsonClient(
28 | sparkSession: SparkSession,
29 | param: Map[String, Any],
30 | dsName: String,
31 | index: Int,
32 | timestampStorage: TimestampStorage)
33 | extends StreamingCacheClient {
34 |
35 | protected def writeDataFrame(dfw: DataFrameWriter[Row], path: String): Unit = {
36 | info(s"write path: $path")
37 | dfw.json(path)
38 | }
39 |
40 | protected def readDataFrame(dfr: DataFrameReader, path: String): DataFrame = {
41 | dfr.json(path)
42 | }
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/measure/src/main/resources/config-batch-advanced.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "accu_batch",
3 | "process.type": "batch",
4 | "data.sources": [
5 | {
6 | "name": "source",
7 | "baseline": true,
8 | "connector": {
9 | "type": "avro",
10 | "version": "1.7",
11 | "config": {
12 | "file.name": "src/test/resources/users_info_src.avro"
13 | }
14 | }
15 | },
16 | {
17 | "name": "target",
18 | "connector": {
19 | "type": "avro",
20 | "version": "1.7",
21 | "config": {
22 | "file.name": "src/test/resources/users_info_target.avro"
23 | }
24 | }
25 | }
26 | ],
27 | "evaluate.rule": {
28 | "rules": [
29 | {
30 | "dsl.type": "griffin-dsl",
31 | "dq.type": "accuracy",
32 | "out.dataframe.name": "accu",
33 | "rule": "source.user_id = target.user_id AND upper(source.first_name) = upper(target.first_name) AND source.last_name = target.last_name AND source.address = target.address AND source.email = target.email AND source.phone = target.phone AND source.post_code = target.post_code",
34 | "details": {
35 | "source": "source",
36 | "target": "target",
37 | "miss": "miss_count",
38 | "total": "total_count",
39 | "matched": "matched_count"
40 | },
41 | "out": [
42 | {
43 | "type": "metric",
44 | "name": "accu"
45 | },
46 | {
47 | "type": "record",
48 | "name": "missRecords"
49 | }
50 | ]
51 | }
52 | ]
53 | },
54 | "sinks": [
55 | "consoleSink",
56 | "elasticSink"
57 | ]
58 | }
59 |
--------------------------------------------------------------------------------
/measure/src/main/scala/org/apache/griffin/measure/step/builder/SparkSqlDQStepBuilder.scala:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.griffin.measure.step.builder
19 |
20 | import org.apache.griffin.measure.configuration.dqdefinition.RuleParam
21 | import org.apache.griffin.measure.context.DQContext
22 | import org.apache.griffin.measure.step.DQStep
23 | import org.apache.griffin.measure.step.transform.SparkSqlTransformStep
24 |
25 | case class SparkSqlDQStepBuilder() extends RuleParamStepBuilder {
26 |
27 | def buildSteps(context: DQContext, ruleParam: RuleParam): Seq[DQStep] = {
28 | val name = getStepName(ruleParam.getOutDfName())
29 | val transformStep = SparkSqlTransformStep(
30 | name,
31 | ruleParam.getRule,
32 | ruleParam.getDetails,
33 | None,
34 | ruleParam.getCache)
35 | transformStep +: buildDirectWriteSteps(ruleParam)
36 | }
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/service/src/main/java/org/apache/griffin/core/config/CacheConfig.java:
--------------------------------------------------------------------------------
1 | /*
2 | Licensed to the Apache Software Foundation (ASF) under one
3 | or more contributor license agreements. See the NOTICE file
4 | distributed with this work for additional information
5 | regarding copyright ownership. The ASF licenses this file
6 | to you under the Apache License, Version 2.0 (the
7 | "License"); you may not use this file except in compliance
8 | with the License. You may obtain a copy of the License at
9 |
10 | http://www.apache.org/licenses/LICENSE-2.0
11 |
12 | Unless required by applicable law or agreed to in writing,
13 | software distributed under the License is distributed on an
14 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 | KIND, either express or implied. See the License for the
16 | specific language governing permissions and limitations
17 | under the License.
18 | */
19 | package org.apache.griffin.core.config;
20 |
21 | import org.apache.commons.lang.StringUtils;
22 | import org.springframework.cache.annotation.EnableCaching;
23 | import org.springframework.cache.interceptor.KeyGenerator;
24 | import org.springframework.context.annotation.Bean;
25 | import org.springframework.context.annotation.Configuration;
26 |
27 | @Configuration
28 | @EnableCaching
29 | public class CacheConfig {
30 | @Bean
31 | public KeyGenerator cacheKeyGenerator() {
32 | return (o, method, objects) -> {
33 | StringBuilder sb = new StringBuilder(method.getName());
34 | String params = StringUtils.join(objects);
35 | if (!StringUtils.isEmpty(params)) {
36 | sb.append(params);
37 | }
38 | return sb.toString();
39 | };
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/ui/angular/src/test.html:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | Griffin - Data Quality Service
29 |
30 |
31 |
32 |
36 |
37 |
38 | This is for test
39 |
40 |
41 |