Commit e691cc10 by 魏建枢

simi好友指标作业

parent 2ac13f87
......@@ -12,6 +12,7 @@ import com.flink.processor.impl.RealTransactionProcessor;
import com.flink.processor.impl.RealUsersProcessor;
import com.flink.processor.impl.SimiFriendsProcessor;
import com.flink.processor.impl.SimiGroupstProcessor;
import com.flink.processor.impl.VectorAngleCalculationProcessor;
/**
* @author wjs
......@@ -42,6 +43,8 @@ public class JobProcessorFactory {
return new SimiFriendsProcessor();
case SIMI_GROUPS:
return new SimiGroupstProcessor();
case VECTOR_ANGLE_CALCULATION:
return new VectorAngleCalculationProcessor();
default:
throw new IllegalArgumentException("未知的Job类型: " + jobType);
}
......
package com.flink.processor.function;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.state.MapState;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.co.CoProcessFunction;
import org.apache.flink.util.Collector;
import com.flink.achieve.doris.SimiFriendsAchi.TotalTemp;
import com.flink.vo.SimiUserInfo;
/**
* @author wjs
* @version 创建时间:2025-6-17 11:21:03
* 类说明
*/
public class SimiFriendsTempJoinProcessor extends CoProcessFunction<TotalTemp, SimiUserInfo, TotalTemp>{
/**
*
*/
private static final long serialVersionUID = 1L;
private ValueState<SimiUserInfo> userState;
private MapState<Long, TotalTemp> pendingTotalTemp;
@Override
public void open(Configuration parameters) {
userState = getRuntimeContext().getState(new ValueStateDescriptor<>("user-state", SimiUserInfo.class));
MapStateDescriptor<Long, TotalTemp> totalTempDesc = new MapStateDescriptor<>("pendingDevices", Long.class,
TotalTemp.class);
pendingTotalTemp = getRuntimeContext().getMapState(totalTempDesc);
}
@Override
public void processElement1(TotalTemp totalTemp, CoProcessFunction<TotalTemp, SimiUserInfo, TotalTemp>.Context ctx,
Collector<TotalTemp> out) throws Exception {
SimiUserInfo user = userState.value();
if (user != null && StringUtils.equals(user.getCid(), totalTemp.getCid()) && StringUtils.isEmpty(user.getThird_id())) {
out.collect(buildResult(user, totalTemp));
} else {
pendingTotalTemp.put(totalTemp.getCollectTime(), totalTemp);
ctx.timerService().registerEventTimeTimer(totalTemp.getCollectTime() + 60000); // 1分钟超时
}
}
@Override
public void processElement2(SimiUserInfo user, CoProcessFunction<TotalTemp, SimiUserInfo, TotalTemp>.Context ctx,
Collector<TotalTemp> out) throws Exception {
userState.update(user); // 更新最新用户状态
for (TotalTemp totalTemp : pendingTotalTemp.values()) {
if (user != null && StringUtils.equals(user.getCid(), totalTemp.getCid()) && StringUtils.isEmpty(user.getThird_id())) {
out.collect(buildResult(user, totalTemp));
}
}
pendingTotalTemp.clear();
}
@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<TotalTemp> out) throws Exception {
pendingTotalTemp.remove(timestamp - 60000); // 清理超时事件
}
private TotalTemp buildResult(SimiUserInfo user, TotalTemp totalTemp) {
return new TotalTemp(
totalTemp.getCid(),
totalTemp.getOverlapCidTotal(),
totalTemp.getIp(),
totalTemp.getAreaName(),
totalTemp.getAddMethod(),
totalTemp.getCreateTime(),
totalTemp.getCidsList(),
totalTemp.getFriendsList(),
totalTemp.getCollectTime()
);
}
}
package com.flink.processor.impl;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import com.flink.achieve.doris.SimiFriendsAchi;
import com.flink.enums.JobTypeEnum;
import com.flink.enums.TopicTypeEnum;
import com.flink.processor.JobProcessor;
import com.flink.vo.KafkaTopic;
/**
* @author wjs
......@@ -15,9 +20,18 @@ public class SimiFriendsProcessor implements JobProcessor{
@Override
public void process() throws Exception {
new SimiFriendsAchi().handleDataStreamSource(
JobTypeEnum.SIMI_FRIENDS,
TopicTypeEnum.OPEN_SIMI_API
createTopicList(),
JobTypeEnum.SIMI_FRIENDS
);
}
private static List<KafkaTopic> createTopicList() {
return Arrays.stream(new TopicTypeEnum[]{
TopicTypeEnum.OPEN_SIMI_API,
TopicTypeEnum.ODS_EVENT_LOG,
TopicTypeEnum.SIMI_USER_LIST_TOPIC
}).map(TopicTypeEnum::createKafkaTopic)
.collect(Collectors.toList());
}
}
......@@ -2,16 +2,19 @@ package com.flink.util;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.TimeZone;
import org.apache.flink.table.data.TimestampData;
/**
* @author wjs
* @version 创建时间:2025-5-27 14:33:05
* 类说明
*/
* @author wjs
* @version 创建时间:2025-5-27 14:33:05 类说明
*/
public class TimeConvertUtil {
public static long convertToTimestamp(String timeStr) throws Exception {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
......@@ -19,7 +22,26 @@ public class TimeConvertUtil {
}
public static int convertToSqlDate(String datetime) {
return (int) LocalDate.parse(datetime.split(" ")[0])
.toEpochDay(); // 转换为天数偏移量
return (int) LocalDate.parse(datetime.split(" ")[0]).toEpochDay(); // 转换为天数偏移量
}
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
private static final ZoneId DEFAULT_ZONE = ZoneId.systemDefault();
// 获取当前TimestampData(毫秒级)
public static TimestampData currentTimestamp() {
return TimestampData.fromEpochMillis(System.currentTimeMillis());
}
// 带时区的字符串转时间戳
public static TimestampData parseToTimestamp(String timeStr, ZoneId zone) {
LocalDateTime ldt = LocalDateTime.parse(timeStr, FORMATTER);
long epochMillis = ldt.atZone(zone).toInstant().toEpochMilli();
return TimestampData.fromEpochMillis(epochMillis);
}
// 格式化输出(用于日志/导出)
public static String format(TimestampData timestamp) {
return FORMATTER.format(timestamp.toLocalDateTime());
}
}
......@@ -19,6 +19,7 @@ public class SimiUserInfo implements Serializable {
private static final long serialVersionUID = 1L;
private String nick;
private String third_id;
private String country_code;
private String user_head_url;
private String create_time;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment