Skip to content

eagleEye / eagleEye-flink_kafka

Go to a project
Commit 6c6db13e by 魏建枢
Options

simi上报增加海内外版

parent 36fe53e0
Expand all
Inline Side-by-side
Showing with 61 additions and 26 deletions
eagleEye-flink_kafka/src/main/java/com/flink/achieve/table/schema/DorisBaseSchema.java
...@@ -130,6 +130,7 @@ public class DorisBaseSchema { ...@@ -130,6 +130,7 @@ public class DorisBaseSchema {
.column("country_code", STRING()) .column("country_code", STRING())
.column("phone_number", STRING()) .column("phone_number", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -141,6 +142,7 @@ public class DorisBaseSchema { ...@@ -141,6 +142,7 @@ public class DorisBaseSchema {
.column("old_nick_name", STRING()) .column("old_nick_name", STRING())
.column("new_nick_name", STRING()) .column("new_nick_name", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -154,6 +156,7 @@ public class DorisBaseSchema { ...@@ -154,6 +156,7 @@ public class DorisBaseSchema {
.column("add_method", STRING()) .column("add_method", STRING())
.column("remark", STRING()) .column("remark", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -167,6 +170,7 @@ public class DorisBaseSchema { ...@@ -167,6 +170,7 @@ public class DorisBaseSchema {
.column("add_method", STRING()) .column("add_method", STRING())
.column("remark", STRING()) .column("remark", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -179,6 +183,7 @@ public class DorisBaseSchema { ...@@ -179,6 +183,7 @@ public class DorisBaseSchema {
.column("old_nick_name", STRING()) .column("old_nick_name", STRING())
.column("new_nick_name", STRING()) .column("new_nick_name", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -191,6 +196,7 @@ public class DorisBaseSchema { ...@@ -191,6 +196,7 @@ public class DorisBaseSchema {
.column("friend_nick_name", STRING()) .column("friend_nick_name", STRING())
.column("remark", STRING()) .column("remark", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -201,6 +207,7 @@ public class DorisBaseSchema { ...@@ -201,6 +207,7 @@ public class DorisBaseSchema {
.column("time", TIMESTAMP(3)) .column("time", TIMESTAMP(3))
.column("group_id", DataTypes.STRING()) .column("group_id", DataTypes.STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -213,6 +220,7 @@ public class DorisBaseSchema { ...@@ -213,6 +220,7 @@ public class DorisBaseSchema {
.column("group_name", STRING()) .column("group_name", STRING())
.column("join_type", STRING()) .column("join_type", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -225,6 +233,7 @@ public class DorisBaseSchema { ...@@ -225,6 +233,7 @@ public class DorisBaseSchema {
.column("old_group_name", STRING()) .column("old_group_name", STRING())
.column("new_group_name", STRING()) .column("new_group_name", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -237,6 +246,7 @@ public class DorisBaseSchema { ...@@ -237,6 +246,7 @@ public class DorisBaseSchema {
.column("group_name", STRING()) .column("group_name", STRING())
.column("exit_type", STRING()) .column("exit_type", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -248,6 +258,7 @@ public class DorisBaseSchema { ...@@ -248,6 +258,7 @@ public class DorisBaseSchema {
.column("release_type", STRING()) .column("release_type", STRING())
.column("article_id", STRING()) .column("article_id", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
...@@ -263,6 +274,7 @@ public class DorisBaseSchema { ...@@ -263,6 +274,7 @@ public class DorisBaseSchema {
.column("comment_id", STRING()) .column("comment_id", STRING())
.column("comment_level", STRING()) .column("comment_level", STRING())
.column("create_time", TIMESTAMP(3)) .column("create_time", TIMESTAMP(3))
.column("platform", STRING())
.build(); .build();
createTableDescriptor(tableEnv, schema, tempTableName,dbTableName); createTableDescriptor(tableEnv, schema, tempTableName,dbTableName);
} }
......
eagleEye-flink_kafka/src/main/java/com/flink/achieve/table/sql/OpenSimiApiTable.java
...@@ -22,9 +22,10 @@ public class OpenSimiApiTable { ...@@ -22,9 +22,10 @@ public class OpenSimiApiTable {
" t.`time` AS `time`, " + " t.`time` AS `time`, " +
" t.simi_api_info['countryCode'] AS country_code, " + " t.simi_api_info['countryCode'] AS country_code, " +
" t.simi_api_info['phoneNumber'] AS phone_number, " + " t.simi_api_info['phoneNumber'] AS phone_number, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '1'" "WHERE send_type = '1'"
); );
//2 //2
...@@ -36,9 +37,10 @@ public class OpenSimiApiTable { ...@@ -36,9 +37,10 @@ public class OpenSimiApiTable {
" t.`time` AS `time`, " + " t.`time` AS `time`, " +
" t.simi_api_info['oldNickname'] AS old_nick_name, " + " t.simi_api_info['oldNickname'] AS old_nick_name, " +
" t.simi_api_info['newNickname'] AS new_nick_name, " + " t.simi_api_info['newNickname'] AS new_nick_name, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '2'" "WHERE send_type = '2'"
); );
//3 //3
...@@ -52,9 +54,10 @@ public class OpenSimiApiTable { ...@@ -52,9 +54,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['friendNickname'] AS friend_nick_name, " + " t.simi_api_info['friendNickname'] AS friend_nick_name, " +
" t.simi_api_info['addMethod'] AS add_method, " + " t.simi_api_info['addMethod'] AS add_method, " +
" t.simi_api_info['remark'] AS remark, " + " t.simi_api_info['remark'] AS remark, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '3'" "WHERE send_type = '3'"
); );
//4 //4
...@@ -68,9 +71,10 @@ public class OpenSimiApiTable { ...@@ -68,9 +71,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['friendNickname'] AS friend_nick_name, " + " t.simi_api_info['friendNickname'] AS friend_nick_name, " +
" t.simi_api_info['addMethod'] AS add_method, " + " t.simi_api_info['addMethod'] AS add_method, " +
" t.simi_api_info['remark'] AS remark, " + " t.simi_api_info['remark'] AS remark, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '4'" "WHERE send_type = '4'"
); );
//5 //5
...@@ -83,9 +87,10 @@ public class OpenSimiApiTable { ...@@ -83,9 +87,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['friendCid'] AS friend_cid, " + " t.simi_api_info['friendCid'] AS friend_cid, " +
" t.simi_api_info['oldNickname'] AS old_nick_name, " + " t.simi_api_info['oldNickname'] AS old_nick_name, " +
" t.simi_api_info['newNickname'] AS new_nick_name, " + " t.simi_api_info['newNickname'] AS new_nick_name, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '5'" "WHERE send_type = '5'"
); );
//6 //6
...@@ -98,9 +103,10 @@ public class OpenSimiApiTable { ...@@ -98,9 +103,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['friendCid'] AS friend_cid, " + " t.simi_api_info['friendCid'] AS friend_cid, " +
" t.simi_api_info['friendNickname'] AS friend_nick_name, " + " t.simi_api_info['friendNickname'] AS friend_nick_name, " +
" t.simi_api_info['remark'] AS remark, " + " t.simi_api_info['remark'] AS remark, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '6'" "WHERE send_type = '6'"
); );
//7 //7
...@@ -111,9 +117,10 @@ public class OpenSimiApiTable { ...@@ -111,9 +117,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['cid'] AS cid, " + " t.simi_api_info['cid'] AS cid, " +
" t.`time` AS `time`, " + " t.`time` AS `time`, " +
" t.simi_api_info['groupId'] AS group_id, " + " t.simi_api_info['groupId'] AS group_id, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '7'" "WHERE send_type = '7'"
); );
//8 //8
...@@ -126,9 +133,10 @@ public class OpenSimiApiTable { ...@@ -126,9 +133,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['groupId'] AS group_id, " + " t.simi_api_info['groupId'] AS group_id, " +
" t.simi_api_info['groupName'] AS group_name, " + " t.simi_api_info['groupName'] AS group_name, " +
" t.simi_api_info['joinType'] AS join_type, " + " t.simi_api_info['joinType'] AS join_type, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '8'" "WHERE send_type = '8'"
); );
//9 //9
...@@ -141,9 +149,10 @@ public class OpenSimiApiTable { ...@@ -141,9 +149,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['groupId'] AS group_id, " + " t.simi_api_info['groupId'] AS group_id, " +
" t.simi_api_info['oldGroupName'] AS old_group_name, " + " t.simi_api_info['oldGroupName'] AS old_group_name, " +
" t.simi_api_info['newGroupName'] AS new_group_name, " + " t.simi_api_info['newGroupName'] AS new_group_name, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '9'" "WHERE send_type = '9'"
); );
//10 //10
...@@ -156,9 +165,10 @@ public class OpenSimiApiTable { ...@@ -156,9 +165,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['groupId'] AS group_id, " + " t.simi_api_info['groupId'] AS group_id, " +
" t.simi_api_info['groupName'] AS group_name, " + " t.simi_api_info['groupName'] AS group_name, " +
" t.simi_api_info['exitType'] AS exit_type, " + " t.simi_api_info['exitType'] AS exit_type, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '10'" "WHERE send_type = '10'"
); );
//11 //11
...@@ -170,9 +180,10 @@ public class OpenSimiApiTable { ...@@ -170,9 +180,10 @@ public class OpenSimiApiTable {
" t.`time` AS `time`, " + " t.`time` AS `time`, " +
" t.simi_api_info['releaseType'] AS release_type, " + " t.simi_api_info['releaseType'] AS release_type, " +
" t.simi_api_info['articleId'] AS article_id, " + " t.simi_api_info['articleId'] AS article_id, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '11'" "WHERE send_type = '11'"
); );
//12 //12
...@@ -188,9 +199,10 @@ public class OpenSimiApiTable { ...@@ -188,9 +199,10 @@ public class OpenSimiApiTable {
" t.simi_api_info['commentContent'] AS comment_content, " + " t.simi_api_info['commentContent'] AS comment_content, " +
" t.simi_api_info['commentId'] AS comment_id, " + " t.simi_api_info['commentId'] AS comment_id, " +
" t.simi_api_info['commentLevel'] AS comment_level, " + " t.simi_api_info['commentLevel'] AS comment_level, " +
" k.create_time AS create_time " + " k.create_time AS create_time, " +
" t.simi_api_info['platform'] AS platform " +
"FROM kafka_open_simi_api AS k " + "FROM kafka_open_simi_api AS k " +
"LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type)) AS t(`time`,simi_api_info) ON TRUE " + "LEFT JOIN LATERAL TABLE(ParseOpenSimiApi(req_body,send_type,flume_type)) AS t(`time`,simi_api_info) ON TRUE " +
"WHERE send_type = '12'" "WHERE send_type = '12'"
); );
} }
......
eagleEye-flink_kafka/src/main/java/com/flink/achieve/table/udf/ParseOpenSimiApiUDTF.java
...@@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory; ...@@ -17,6 +17,7 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference; import com.alibaba.fastjson.TypeReference;
import com.flink.enums.OpenSimiApiTypeEnum; import com.flink.enums.OpenSimiApiTypeEnum;
import com.flink.util.CompareUtils;
import com.flink.vo.simi.ContentInteractionReqDto; import com.flink.vo.simi.ContentInteractionReqDto;
import com.flink.vo.simi.CreateGroupReqDto; import com.flink.vo.simi.CreateGroupReqDto;
import com.flink.vo.simi.DeleteFriendReqDto; import com.flink.vo.simi.DeleteFriendReqDto;
...@@ -55,13 +56,23 @@ public class ParseOpenSimiApiUDTF extends TableFunction<Row>{ ...@@ -55,13 +56,23 @@ public class ParseOpenSimiApiUDTF extends TableFunction<Row>{
return LocalDateTime.parse(timeStr, FORMATTER); return LocalDateTime.parse(timeStr, FORMATTER);
} }
public void eval(String req_body,String send_type) throws ParseException { public void eval(String req_body,String send_type,String flume_type) throws ParseException {
if(StringUtils.isAllEmpty(send_type,req_body)) { if(StringUtils.isAllEmpty(send_type,req_body,flume_type)) {
return; return;
} }
if(!CompareUtils.stringExists(flume_type, "openSimiApi","openAbroadSimiApiTest")) {
return;
}
String platform = null;
if(StringUtils.equals(flume_type, "openSimiApi")) {
platform = "国内版";
}else if(StringUtils.equals(flume_type, "openAbroadSimiApiTest")) {
platform = "海外版";
}
Map<String, Object> params = new HashMap<>(); Map<String, Object> params = new HashMap<>();
LocalDateTime timestamp = null; LocalDateTime timestamp = null;
params.put("platform", platform);
if(StringUtils.equals(send_type, OpenSimiApiTypeEnum.USER_REGISTRATION.getCode())) {//1 if(StringUtils.equals(send_type, OpenSimiApiTypeEnum.USER_REGISTRATION.getCode())) {//1
UserRegistrationReqDto jsonReqDto = JSONObject.parseObject(req_body,new TypeReference<UserRegistrationReqDto>(){}); UserRegistrationReqDto jsonReqDto = JSONObject.parseObject(req_body,new TypeReference<UserRegistrationReqDto>(){});
params.put("id", jsonReqDto.getId()); params.put("id", jsonReqDto.getId());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment