Skip to content

eagleEye / eagleEye-flink_kafka

Go to a project
Commit 0a7239f6 by 魏建枢
Options

代码整合提交

parent d8f3af8a
Inline Side-by-side
Showing with 194 additions and 454 deletions
eagleEye-flink_kafka/src/main/java/com/flink/achieve/base/CommonConsumeBaseAchi.java
......@@ -40,10 +40,10 @@ public class CommonConsumeBaseAchi extends MultipleSourceCommonBase implements S
DataStreamSource<String> eventLogStreamSource = kafkaDataSource.getDataStreamSource();
EventLogAchi.eventLog(eventLogStreamSource);
}
// if(StringUtils.equals(kafkaDataSource.getTopic(),TopicTypeEnum.ODS_COMMUNITY_HISTORY.getTopic())) {
// DataStreamSource<String> communityHistoryStreamSource = kafkaDataSource.getDataStreamSource();
// CommunityHistoryAchi.communityHistory(communityHistoryStreamSource);
// }
if(StringUtils.equals(kafkaDataSource.getTopic(),TopicTypeEnum.ODS_COMMUNITY_HISTORY.getTopic())) {
DataStreamSource<String> communityHistoryStreamSource = kafkaDataSource.getDataStreamSource();
CommunityHistoryAchi.communityHistory(communityHistoryStreamSource);
}
if(StringUtils.equals(kafkaDataSource.getTopic(),TopicTypeEnum.ODS_NEW_COLLECT_LOG.getTopic())) {
DataStreamSource<String> collectLogStreamSource = kafkaDataSource.getDataStreamSource();
CollectLogAchi.collectLog(collectLogStreamSource);
......
eagleEye-flink_kafka/src/main/java/com/flink/achieve/base/CommunityHistoryAchi.java
package com.flink.achieve.base;
import java.io.Serializable;
import java.util.List;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author wjs
* @version 创建时间：2025-7-28 10:40:20
* 类说明
*/
public class CommunityHistoryAchi implements Serializable{
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.flink.common.DorisConnector;
import com.flink.config.TableConfig;
import com.flink.util.TimeConvertUtil;
import com.flink.vo.CommunityHistory;
import com.flink.vo.RiskInfo;
/**
* @author wjs
* @version 创建时间：2025-7-28 10:40:20 类说明
*/
public class CommunityHistoryAchi implements Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(CommunityHistoryAchi.class);
// 定义公共常量
private static final String FLUME_TYPE_FIELD = "flume_type";
private static final String DATA_FIELD = "list";
private static final String DORIS_DELETE_SIGN = "__DORIS_DELETE_SIGN__";
private static final int DELETE_SIGN_VALUE = 0;
private static final String[] COMMUNITY_HISTORY_FIELDS = { "article_id", "source_type", "article_type",
"article_state", "create_time", "article_user_cid", "article_text", "images", "label", "score", "law",
"law_image", "politics", "politics_image", "ad", "ad_image", "hot_search_title", DORIS_DELETE_SIGN };
private static final DataType[] COMMUNITY_HISTORY_TYPES = { DataTypes.STRING(), DataTypes.STRING(),
DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(),
DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(),
DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING(),
DataTypes.INT() };
public static void communityHistory(DataStreamSource<String> dataStreamSource) {
// 初始化表配置
TableConfig communityHistoryConfig = new TableConfig(COMMUNITY_HISTORY_FIELDS, COMMUNITY_HISTORY_TYPES,
"bi.simi_community_history");
// 创建Doris Sink
DorisSink<RowData> dorisCommunityHistorySink = DorisConnector.sinkDoris(communityHistoryConfig.getFields(),
communityHistoryConfig.getTypes(), communityHistoryConfig.getTableName());
processDataStream(dataStreamSource, "communityHistory", communityHistoryConfig, dorisCommunityHistorySink,
(RowMapper<CommunityHistory>) CommunityHistoryAchi::mapToCommunityHistoryRow);
processDataStream(dataStreamSource, "communityHistoryAbroad", communityHistoryConfig, dorisCommunityHistorySink,
(RowMapper<CommunityHistory>) CommunityHistoryAchi::mapToCommunityHistoryAbroadRow);
}
/**
* 通用数据处理方法
*
* @param dataStream 数据流
* @param flumeType 数据类型
* @param tableConfig 表配置
* @param dorisSink Doris Sink
* @param mapper 行数据映射函数
*/
private static <T> void processDataStream(DataStreamSource<String> dataStream, String flumeType,
TableConfig tableConfig, DorisSink<RowData> dorisSink, RowMapper<T> mapper) {
SingleOutputStreamOperator<RowData> processedStream = dataStream
.flatMap(new ElementProcessor<>(flumeType, mapper, tableConfig.getFields().length))
.returns(TypeInformation.of(RowData.class)).filter(Objects::nonNull);
processedStream.sinkTo(dorisSink).name("Doris-" + flumeType);
}
public static void communityHistory(DataStreamSource<String> communityHistoryStreamSource) {
// TODO Auto-generated method stub
private static class ElementProcessor<T> implements FlatMapFunction<String, RowData>, Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
private final String flumeType;
private final RowMapper<T> mapper;
private final int fieldCount;
public ElementProcessor(String flumeType, RowMapper<T> mapper, int fieldCount) {
this.flumeType = flumeType;
this.mapper = mapper;
this.fieldCount = fieldCount;
}
@Override
public void flatMap(String value, Collector<RowData> out) throws Exception {
try {
JSONObject jsonObj = JSON.parseObject(value);
if (!flumeType.equals(jsonObj.getString(FLUME_TYPE_FIELD))) {
return;
}
String bodyStr = jsonObj.getString(DATA_FIELD);
List<?> dataList = JSON.parseArray(bodyStr, getTargetClass(flumeType));
if (CollectionUtils.isNotEmpty(dataList)) {
for (Object item : dataList) {
RowData row = mapper.map(item, fieldCount);
if (row != null)
out.collect(row);
}
}
} catch (Exception e) {
logger.error("CommunityHistoryAchi 处理 {} 数据出错 | rawData:{} | error:{}", flumeType, value,
e.getMessage(), e);
}
}
private Class<?> getTargetClass(String type) {
switch (type) {
case "communityHistory":
return CommunityHistory.class;
case "communityHistoryAbroad":
return CommunityHistory.class;
default:
throw new IllegalArgumentException("未知类型: " + type);
}
}
}
// 国内社区记录数据映射
private static RowData mapToCommunityHistoryRow(Object item, int fieldCount) {
CommunityHistory communityHistory = (CommunityHistory) item; // 显式类型转换
RiskInfo riskInfo = communityHistory.getRisk();
GenericRowData row = new GenericRowData(fieldCount);
row.setField(0, StringData.fromString(communityHistory.getArticleId()));
row.setField(1, StringData.fromString("国内版"));
row.setField(2, StringData.fromString(communityHistory.getArticleType()));
row.setField(3, StringData.fromString(communityHistory.getArticleState()));
row.setField(4, StringData.fromString(TimeConvertUtil.parseToStringSSS(communityHistory.getCreateTime())));
row.setField(5, StringData.fromString(communityHistory.getArticleUserCid()));
row.setField(6, StringData.fromString(communityHistory.getArticleText()));
row.setField(7, StringData.fromString(communityHistory.getImages()));
row.setField(8, StringData.fromString(communityHistory.getLabel()));
row.setField(9, StringData.fromString(riskInfo == null ? null : riskInfo.getScore()));
row.setField(10, StringData.fromString(riskInfo == null ? null : riskInfo.getLaw()));
row.setField(11, StringData.fromString(riskInfo == null ? null : riskInfo.getLawImage()));
row.setField(12, StringData.fromString(riskInfo == null ? null : riskInfo.getPolitics()));
row.setField(13, StringData.fromString(riskInfo == null ? null : riskInfo.getPoliticsImage()));
row.setField(14, StringData.fromString(riskInfo == null ? null : riskInfo.getAd()));
row.setField(15, StringData.fromString(riskInfo == null ? null : riskInfo.getAdImage()));
row.setField(16, StringData.fromString(communityHistory.getHotSearchTitle()));
row.setField(17, DELETE_SIGN_VALUE);
return row;
}
// 海外社区记录数据映射
private static RowData mapToCommunityHistoryAbroadRow(Object item, int fieldCount) {
CommunityHistory communityHistory = (CommunityHistory) item; // 显式类型转换
RiskInfo riskInfo = communityHistory.getRisk();
GenericRowData row = new GenericRowData(fieldCount);
row.setField(0, StringData.fromString(communityHistory.getArticleId()));
row.setField(1, StringData.fromString("海外版"));
row.setField(2, StringData.fromString(communityHistory.getArticleType()));
row.setField(3, StringData.fromString(communityHistory.getArticleState()));
row.setField(4, StringData.fromString(TimeConvertUtil.parseToStringSSS(communityHistory.getCreateTime())));
row.setField(5, StringData.fromString(communityHistory.getArticleUserCid()));
row.setField(6, StringData.fromString(communityHistory.getArticleText()));
row.setField(7, StringData.fromString(communityHistory.getImages()));
row.setField(8, StringData.fromString(communityHistory.getLabel()));
row.setField(9, StringData.fromString(riskInfo == null ? null : riskInfo.getScore()));
row.setField(10, StringData.fromString(riskInfo == null ? null : riskInfo.getLaw()));
row.setField(11, StringData.fromString(riskInfo == null ? null : riskInfo.getLawImage()));
row.setField(12, StringData.fromString(riskInfo == null ? null : riskInfo.getPolitics()));
row.setField(13, StringData.fromString(riskInfo == null ? null : riskInfo.getPoliticsImage()));
row.setField(14, StringData.fromString(riskInfo == null ? null : riskInfo.getAd()));
row.setField(15, StringData.fromString(riskInfo == null ? null : riskInfo.getAdImage()));
row.setField(16, StringData.fromString(communityHistory.getHotSearchTitle()));
row.setField(17, DELETE_SIGN_VALUE);
return row;
}
/**
* 行数据映射接口
*
* @param <T> 数据类型
*/
@FunctionalInterface
private static interface RowMapper<T> extends Serializable {
RowData map(Object item, int fieldCount);
}
}
eagleEye-flink_kafka/src/main/java/com/flink/achieve/doris/EventLogAchi.java deleted 100644 → 0
package com.flink.achieve.doris;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Objects;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.io.ParseException;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.data.TimestampData;
import org.apache.flink.table.types.DataType;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import com.flink.common.DorisConnector;
import com.flink.common.MultipleSourceCommonBase;
import com.flink.config.TableConfig;
import com.flink.enums.TopicTypeEnum;
import com.flink.processor.function.UserPropertiesProcessor;
import com.flink.util.CompareUtils;
import com.flink.util.TimeConvertUtil;
import com.flink.vo.CommunityHistory;
import com.flink.vo.EventList;
import com.flink.vo.KafkaDataSource;
import com.flink.vo.OdsEventLog;
import com.flink.vo.Properties;
import com.flink.vo.RiskInfo;
import com.flink.vo.UserProperties;
/**
* @author wjs
* @version 创建时间：2025-6-20 23:40:33
* 类说明
*/
public class EventLogAchi extends MultipleSourceCommonBase implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(EventLogAchi.class);
@Override
public void parseSourceKafkaJson(List<KafkaDataSource> dataSourceList) throws ParseException, Exception {
// =================配置入库字段=========================================
// 事件明细表结构
// TableConfig tableConfig = new TableConfig(
// new String[]{
// "id",
// "dt",
// "send_time",
// "create_time",
// "strategy_group_id",
// "app_key",
// "app_type",
// "app_channel",
// "zone_code",
// "zone_name",
// "zone_type",
// "sdk_version",
// "user_agent",
// "device_id",
// "uid",
// "strategy_version",
// "event_list",
// "route_ip",
// "cid",
// "phone",
// "nick",
// "unique_id",
// "__DORIS_DELETE_SIGN__"
// },
// new DataType[]{
// DataTypes.STRING(),
// DataTypes.DATE(),
// DataTypes.TIMESTAMP(3),
// DataTypes.TIMESTAMP(3),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.INT()
// },
// "bi.event_log"
// );
//
// TableConfig tableErrorConfig = new TableConfig(
// new String[]{
// "id",
// "dt",
// "event_time",
// "create_time",
// "app_key",
// "app_type",
// "cid",
// "phone",
// "nick",
// "event",
// "data",
// "startTime",
// "timeDifference",
// "endTime",
// "userId",
// "__DORIS_DELETE_SIGN__"
// },
// new DataType[]{
// DataTypes.STRING(),
// DataTypes.DATE(),
// DataTypes.TIMESTAMP(3),
// DataTypes.TIMESTAMP(3),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.STRING(),
// DataTypes.TIMESTAMP(3),
// DataTypes.STRING(),
// DataTypes.TIMESTAMP(3),
// DataTypes.STRING(),
// DataTypes.INT()
// },
// "bi.event_log_error"
// );
//
TableConfig tableCommunityHistoryConfig = new TableConfig(
new String[]{
"article_id",
"source_type",
"article_type",
"article_state",
"create_time",
"article_user_cid",
"article_text",
"images",
"label",
"score",
"law",
"law_image",
"politics",
"politics_image",
"ad",
"ad_image",
"hot_search_title",
"__DORIS_DELETE_SIGN__"
},
new DataType[]{
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.STRING(),
DataTypes.INT()
},
"bi.simi_community_history"
);
//=================流式处理=========================================
// DorisSink<RowData> dorisSink = DorisConnector.sinkDoris(tableConfig.getFields(), tableConfig.getTypes(), tableConfig.getTableName());
// DorisSink<RowData> dorisErrorSink = DorisConnector.sinkDoris(tableErrorConfig.getFields(), tableErrorConfig.getTypes(), tableErrorConfig.getTableName());
DorisSink<RowData> dorisCommunityHistorySink = DorisConnector.sinkDoris(tableCommunityHistoryConfig.getFields(), tableCommunityHistoryConfig.getTypes(), tableCommunityHistoryConfig.getTableName());
//=================数据处理流水线=========================================
// DataStreamSource<String> eventLogStreamSource = null;
DataStreamSource<String> communityHistoryStreamSource = null;
if(CollectionUtils.isNotEmpty(dataSourceList)) {
for(KafkaDataSource kafkaDataSource : dataSourceList) {
// if(StringUtils.equals(kafkaDataSource.getTopic(),TopicTypeEnum.ODS_EVENT_LOG.getTopic())) {
// eventLogStreamSource = kafkaDataSource.getDataStreamSource();
// }
if(StringUtils.equals(kafkaDataSource.getTopic(),TopicTypeEnum.ODS_COMMUNITY_HISTORY.getTopic())) {
communityHistoryStreamSource = kafkaDataSource.getDataStreamSource();
}
}
}else {
return;
}
// eventLogStreamSource
// .map(value->{
// try {
// // 解析 Kafka 数据
// OdsEventLog event = JSONObject.parseObject(value, new TypeReference<OdsEventLog>() {});
// String createTime = event.getCreate_time();
// String routeIp = event.getRoute_ip();
// String appKey = event.getApp_key().trim();
// String appType = event.getApp_type();
// String sendTime = event.getSend_time();
// if(StringUtils.isEmpty(appKey) || StringUtils.equals(appKey, "C7jias27jias2")) {
// appKey = "8ooOvXJo276";
// }
// UserProperties userProperties = UserPropertiesProcessor.userPropertiesToJson(event.getUser_properties());
//
//
// // 转换为RowData
// GenericRowData row = new GenericRowData(tableConfig.getFields().length);
// row.setField(0, StringData.fromString(event.getId()));
// row.setField(1, TimeConvertUtil.convertToSqlDate(createTime.substring(0, 10)));
// row.setField(2, TimestampData.fromLocalDateTime(LocalDateTime.parse(sendTime,DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))));
// row.setField(3, TimestampData.fromLocalDateTime(LocalDateTime.parse(createTime,DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))));
// row.setField(4, StringData.fromString(event.getStrategy_group_id()));
// row.setField(5, StringData.fromString(appKey));
// row.setField(6, StringData.fromString(appType));
// row.setField(7, StringData.fromString(event.getApp_channel()));
// row.setField(8, StringData.fromString(event.getZone_code()));
// row.setField(9, StringData.fromString(event.getZone_name()));
// row.setField(10, StringData.fromString(event.getZone_type()));
// row.setField(11, StringData.fromString(event.getSdk_version()));
// row.setField(12, StringData.fromString(event.getUser_agent()));
// row.setField(13, StringData.fromString(event.getDevice_id()));
// row.setField(14, StringData.fromString(event.getUid()));
// row.setField(15, StringData.fromString(event.getStrategy_version()));
// row.setField(16, StringData.fromString(event.getEvent_list()));
// row.setField(17, StringData.fromString(routeIp));
// row.setField(18, StringData.fromString(userProperties==null ? null : userProperties.getCid()));
// row.setField(19, StringData.fromString(userProperties==null ? null : userProperties.getPhone()));
// row.setField(20, StringData.fromString(userProperties==null ? null : userProperties.getNick()));
// row.setField(21, StringData.fromString(event.getUnique_id()));
// row.setField(22, 0);
// return (RowData)row;
// } catch (Exception e) {
// System.err.println("解析失败: "+e.toString());
// return null;
// }
// })
// .filter(Objects::nonNull)
//// .print()
// .sinkTo(dorisSink)
// .name("Doris-EventLog");
//
// SingleOutputStreamOperator<RowData> rowDataStream = eventLogStreamSource.flatMap(
// new FlatMapFunction<String, RowData>() {
// private static final long serialVersionUID = 1L;
//
// @Override
// public void flatMap(String value, Collector<RowData> out) throws Exception {
// try {
// // 解析 Kafka 数据
// OdsEventLog event = JSONObject.parseObject(value, new TypeReference<OdsEventLog>() {});
// if (null == event) {
// return;
// }
//
// String id = event.getId();
// String createTime = event.getCreate_time();
// int dt = TimeConvertUtil.convertToSqlDate(createTime.substring(0, 10));
// String appKey = event.getApp_key().trim();
// String appType = event.getApp_type();
// UserProperties userProperties = UserPropertiesProcessor.userPropertiesToJson(event.getUser_properties());
// List<EventList> eventList = JSONObject.parseObject(event.getEvent_list(), new TypeReference<List<EventList>>() {});
// if(CollectionUtils.isNotEmpty(eventList)){
// for(EventList eventListInfo : eventList) {
// String r7 = eventListInfo.getR7();
// Long r9 = eventListInfo.getR9();
// Properties r8 = eventListInfo.getR8();
// if(CompareUtils.stringExists(r7,
// "socket_event",
// "socket_error",
// "socket_time",
// "refresh_token",
// "all_time")) {
// // 转换为RowData
// GenericRowData row = new GenericRowData(tableErrorConfig.getFields().length);
// row.setField(0, StringData.fromString(id)); //id
// row.setField(1, dt); //dt
// row.setField(2, TimestampData.fromLocalDateTime(LocalDateTime.parse(TimeConvertUtil.parseToStringSSS(r9),DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))));
// row.setField(3, TimestampData.fromLocalDateTime(LocalDateTime.parse(createTime,DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))));
// row.setField(4, StringData.fromString(appKey));
// row.setField(5, StringData.fromString(appType));
// row.setField(6, StringData.fromString(userProperties==null ? null : userProperties.getCid()));
// row.setField(7, StringData.fromString(userProperties==null ? null : userProperties.getPhone()));
// row.setField(8, StringData.fromString(userProperties==null ? null : userProperties.getNick()));
// row.setField(9, StringData.fromString(r7));
// row.setField(10, StringData.fromString(r8.getData()));
// row.setField(11, TimestampData.fromLocalDateTime(LocalDateTime.parse(TimeConvertUtil.parseToStringSSS(r8.getStartTime()),DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))));
// row.setField(12, StringData.fromString(r8.getTimeDifference()));
// row.setField(13, TimestampData.fromLocalDateTime(LocalDateTime.parse(TimeConvertUtil.parseToStringSSS(r8.getEndTime()),DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))));
// row.setField(14, StringData.fromString(r8.getUserId()));
// row.setField(15, 0);
// out.collect(row);
// }
// }
// }
// } catch (Exception e) {
// logger.error("EventLogAchi 处理 Kafka 消息出错 | rawData:{} | error:{}", value, e.getMessage());
// System.err.println("解析失败: "+e.toString());
// }
// }
// });
//
// rowDataStream
// .filter(Objects::nonNull)
//// .print()
// .sinkTo(dorisErrorSink)
// .name("Doris-EventLogError");
SingleOutputStreamOperator<RowData> communityHistoryRowDataStream = communityHistoryStreamSource.flatMap(new FlatMapFunction<String, RowData>(){
/**
*
*/
private static final long serialVersionUID = 1L;
@Override
public void flatMap(String value, Collector<RowData> out) throws Exception {
try {
// 解析 Kafka 数据
JSONObject jsonObj = JSON.parseObject(value);
String list = jsonObj.getString("list");
String flumeType = jsonObj.getString("flume_type");
if(StringUtils.isEmpty(list)) {
return;
}
String sourceType = null;
if(StringUtils.equals(flumeType, "communityHistory")) {
sourceType = "国内版";
}else if(StringUtils.equals(flumeType, "communityHistoryAbroad")) {
sourceType = "海外版";
}
List<CommunityHistory> records = JSONObject.parseObject(list, new TypeReference<List<CommunityHistory>>() {});
if (CollectionUtils.isEmpty(records)) {
return;
}
for(CommunityHistory communityHistory : records) {
RiskInfo riskInfo = communityHistory.getRisk();
GenericRowData row = new GenericRowData(tableCommunityHistoryConfig.getFields().length);
row.setField(0, StringData.fromString(communityHistory.getArticleId()));
row.setField(1, StringData.fromString(sourceType));
row.setField(2, StringData.fromString(communityHistory.getArticleType()));
row.setField(3, StringData.fromString(communityHistory.getArticleState()));
row.setField(4, StringData.fromString(TimeConvertUtil.parseToStringSSS(communityHistory.getCreateTime())));
row.setField(5, StringData.fromString(communityHistory.getArticleUserCid()));
row.setField(6, StringData.fromString(communityHistory.getArticleText()));
row.setField(7, StringData.fromString(communityHistory.getImages()));
row.setField(8, StringData.fromString(communityHistory.getLabel()));
row.setField(9, StringData.fromString(riskInfo == null ? null : riskInfo.getScore()));
row.setField(10, StringData.fromString(riskInfo == null ? null : riskInfo.getLaw()));
row.setField(11, StringData.fromString(riskInfo == null ? null : riskInfo.getLawImage()));
row.setField(12, StringData.fromString(riskInfo == null ? null : riskInfo.getPolitics()));
row.setField(13, StringData.fromString(riskInfo == null ? null : riskInfo.getPoliticsImage()));
row.setField(14, StringData.fromString(riskInfo == null ? null : riskInfo.getAd()));
row.setField(15, StringData.fromString(riskInfo == null ? null : riskInfo.getAdImage()));
row.setField(16, StringData.fromString(communityHistory.getHotSearchTitle()));
row.setField(17, 0);
out.collect(row);
}
} catch (Exception e) {
logger.error("EventLogAchi communityHistory处理 Kafka 消息出错 | rawData:{} | error:{}", value, e.getMessage());
System.err.println("解析失败: "+e.toString());
}
}
});
communityHistoryRowDataStream
.filter(Objects::nonNull)
// .print()
.sinkTo(dorisCommunityHistorySink)
.name("Doris-EventLogCommunityHistory");
}
}
eagleEye-flink_kafka/src/main/java/com/flink/enums/JobTypeEnum.java
......@@ -25,7 +25,6 @@ public enum JobTypeEnum {
SIMI_FRIENDS("JOB_09", "SIMI好友作业"),
SIMI_GROUPS("JOB_10", "SIMI群组作业"),
VECTOR_ANGLE_CALCULATION("JOB_11", "矢量角度计算作业"),
EVENT_LOG("JOB_12", "事件采集作业"),
REGISTRATION_CHECK("JOB_13", "注册检验采集作业"),
;
......
eagleEye-flink_kafka/src/main/java/com/flink/factory/JobProcessorFactory.java
......@@ -6,7 +6,6 @@ import com.flink.processor.impl.CommonConsumeBaseProcessor;
import com.flink.processor.impl.DeviceIdLatestProcessor;
import com.flink.processor.impl.EventIpConvertProcessor;
import com.flink.processor.impl.EventIpLatestProcessor;
import com.flink.processor.impl.EventLogProcessor;
import com.flink.processor.impl.RegistrationCheckProcessor;
import com.flink.processor.impl.SimiFriendsProcessor;
import com.flink.processor.impl.SimiGroupstProcessor;
......@@ -33,8 +32,6 @@ public class JobProcessorFactory {
return new SimiGroupstProcessor();
case VECTOR_ANGLE_CALCULATION:
return new VectorAngleCalculationProcessor();
case EVENT_LOG:
return new EventLogProcessor();
case REGISTRATION_CHECK:
return new RegistrationCheckProcessor();
case COMMON_CONSUME_BASE:
......
eagleEye-flink_kafka/src/main/java/com/flink/processor/impl/EventLogProcessor.java deleted 100644 → 0
package com.flink.processor.impl;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import com.flink.achieve.doris.EventLogAchi;
import com.flink.enums.JobTypeEnum;
import com.flink.enums.TopicTypeEnum;
import com.flink.processor.JobProcessor;
import com.flink.vo.KafkaTopic;
/**
* @author wjs
* @version 创建时间：2025-6-20 23:39:38
* 类说明
*/
public class EventLogProcessor implements JobProcessor{
@Override
public void process() throws Exception {
new EventLogAchi().handleDataStreamSource(
createTopicList(),
JobTypeEnum.EVENT_LOG
);
}
private static List<KafkaTopic> createTopicList() {
return Arrays.stream(new TopicTypeEnum[]{
// TopicTypeEnum.ODS_EVENT_LOG,
TopicTypeEnum.ODS_COMMUNITY_HISTORY
}).map(TopicTypeEnum::createKafkaTopic)
.collect(Collectors.toList());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment