Skip to content

H
huazheng-project-flink
提交 a5252897 作者: guofeng
操作

增量更新逻辑

上级 e3c0e580
正在显示 包含 216 行增加 和 67 行删除
src/main/java/com/huazheng/project/greenplum/service/impl/CheckDeleteServiceImpl.java
package com.huazheng.project.greenplum.service.impl;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -10,12 +8,16 @@ import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Service;
import com.huazheng.project.greenplum.mapper.GPMapper;
import com.huazheng.project.greenplum.utils.SomeUtils;
import com.huazheng.project.hana.mapper.SapMapper;
import com.huazheng.project.hana.model.Afko;
import com.huazheng.project.hana.model.Afpo;
import com.huazheng.project.hana.model.Aufk;
import com.huazheng.project.hana.model.Bkpf;
import com.huazheng.project.hana.model.Kna1;
import com.huazheng.project.hana.model.Likp;
import com.huazheng.project.hana.model.Lips;
import com.huazheng.project.hana.model.Pa0002;
import cn.hutool.core.thread.ThreadUtil;
import lombok.extern.log4j.Log4j2;
......@@ -26,24 +28,11 @@ public class CheckDeleteServiceImpl {
@Autowired
private RedisTemplate<String, String> redis1Template;
@Autowired
private GPMapper gpMapper;
@Autowired
private SapMapper sapMapper;
public String getErrorInfoFromException(Exception e) {
try {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
return "\r\n" + sw.toString() + "\r\n";
} catch (Exception e2) {
return "bad getErrorInfoFromException";
}
}
public void selectAufkCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
......@@ -70,10 +59,9 @@ public class CheckDeleteServiceImpl {
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Aufk:rowNum", getErrorInfoFromException(e));
redis1Template.opsForValue().set("huazheng:checkDeleteError:Aufk:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectAfkoCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
......@@ -100,10 +88,9 @@ public class CheckDeleteServiceImpl {
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Afko:rowNum", getErrorInfoFromException(e));
redis1Template.opsForValue().set("huazheng:checkDeleteError:Afko:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectAfpoCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
......@@ -131,10 +118,9 @@ public class CheckDeleteServiceImpl {
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Afpo:rowNum", getErrorInfoFromException(e));
redis1Template.opsForValue().set("huazheng:checkDeleteError:Afpo:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectBkpfCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
......@@ -162,10 +148,9 @@ public class CheckDeleteServiceImpl {
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Bkpf:rowNum", getErrorInfoFromException(e));
redis1Template.opsForValue().set("huazheng:checkDeleteError:Bkpf:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectKna1CheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
......@@ -193,7 +178,97 @@ public class CheckDeleteServiceImpl {
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Kna1:rowNum", getErrorInfoFromException(e));
redis1Template.opsForValue().set("huazheng:checkDeleteError:Kna1:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectLikpCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
opsForValue.setIfAbsent("huazheng:checkDelete:Likp:rowNum", "0");
String rowNum = opsForValue.get("huazheng:checkDelete:Likp:rowNum");
Likp build = Likp.builder().rowNum(rowNum).build();
List<Likp> list = gpMapper.selectLikpCheck(build); // 从数仓中查询一组数据
if (list.size() == 0) { // rowid已完成，则转为由更新时间字段开始同步
redis1Template.opsForValue().set("huazheng:checkDelete:Likp:rowNum", "0"); // 计数器复位
ThreadUtil.sleep(1000); // 没有数据了，休眠一下
}
list.forEach(target -> { // 遍历要检查的数据
Likp source = sapMapper.selectLikpById(target); // 根据主键查询源库中的数据
String operator = "none";
if (source == null) { // 如果源库中没有数据
gpMapper.deleteLikp(target); // 删除数仓中的数据
operator = "delete";
}
redis1Template.opsForValue().set("huazheng:checkDelete:Likp:rowNum", target.getRowNum());
if (!operator.equals("none")) {
log.info(String.format("selectLikpcheckDelete --> rowNum:%s, operator:%s", target.getRowNum(), operator));
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Likp:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectLipsCheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
opsForValue.setIfAbsent("huazheng:checkDelete:Lips:rowNum", "0");
String rowNum = opsForValue.get("huazheng:checkDelete:Lips:rowNum");
Lips build = Lips.builder().rowNum(rowNum).build();
List<Lips> list = gpMapper.selectLipsCheck(build); // 从数仓中查询一组数据
if (list.size() == 0) { // rowid已完成，则转为由更新时间字段开始同步
redis1Template.opsForValue().set("huazheng:checkDelete:Lips:rowNum", "0"); // 计数器复位
ThreadUtil.sleep(1000); // 没有数据了，休眠一下
}
list.forEach(target -> { // 遍历要检查的数据
Lips source = sapMapper.selectLipsById(target); // 根据主键查询源库中的数据
String operator = "none";
if (source == null) { // 如果源库中没有数据
gpMapper.deleteLips(target); // 删除数仓中的数据
operator = "delete";
}
redis1Template.opsForValue().set("huazheng:checkDelete:Lips:rowNum", target.getRowNum());
if (!operator.equals("none")) {
log.info(String.format("selectLipscheckDelete --> rowNum:%s, operator:%s", target.getRowNum(), operator));
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Lips:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
public void selectPa0002CheckByDelete() {
try {
ValueOperations<String, String> opsForValue = redis1Template.opsForValue();
opsForValue.setIfAbsent("huazheng:checkDelete:Pa0002:rowNum", "0");
String rowNum = opsForValue.get("huazheng:checkDelete:Pa0002:rowNum");
Pa0002 build = Pa0002.builder().rowNum(rowNum).build();
List<Pa0002> list = gpMapper.selectPa0002Check(build); // 从数仓中查询一组数据
if (list.size() == 0) { // rowid已完成，则转为由更新时间字段开始同步
redis1Template.opsForValue().set("huazheng:checkDelete:Pa0002:rowNum", "0"); // 计数器复位
ThreadUtil.sleep(1000); // 没有数据了，休眠一下
}
list.forEach(target -> { // 遍历要检查的数据
Pa0002 source = sapMapper.selectPa0002ById(target); // 根据主键查询源库中的数据
String operator = "none";
if (source == null) { // 如果源库中没有数据
gpMapper.deletePa0002(target); // 删除数仓中的数据
operator = "delete";
}
redis1Template.opsForValue().set("huazheng:checkDelete:Pa0002:rowNum", target.getRowNum());
if (!operator.equals("none")) {
log.info(String.format("selectPa0002checkDelete --> rowNum:%s, operator:%s", target.getRowNum(), operator));
}
});
} catch (Exception e) {
redis1Template.opsForValue().set("huazheng:checkDeleteError:Pa0002:rowNum", SomeUtils.getErrorInfoFromException(e));
}
}
......
src/main/java/com/huazheng/project/greenplum/utils/SomeUtils.java 0 → 100644
package com.huazheng.project.greenplum.utils;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Date;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
public class SomeUtils {
public static String getErrorInfoFromException(Exception e) {
try {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
return "\r\n" + sw.toString() + "\r\n";
} catch (Exception e2) {
return "bad getErrorInfoFromException";
}
}
public static Date caDate(String datestr) {
try {
if (datestr == null || datestr.equals("00000000") || StrUtil.isBlank(datestr)) {
return null;
} else {
Date date = DateUtil.parse(datestr, "yyyyMMdd");
return date;
}
} catch (Exception e) {
return null;
}
}
}
src/main/resources/mapper/hana/SapMapper_hana.xml
......@@ -329,28 +329,9 @@
where "$rowid$" &gt; #{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLikpCheckByUpdate" parameterType="Likp" resultType="Likp">
select top 20 "$rowid$" as rowids,
vbeln, mandt, erdat, wadat_ist, ctlpc, kunnr, aedat,lfuhr
from ${hana_user}.likp
where "$rowid$" &gt;#{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLipsCheckByUpdate" parameterType="Lips" resultType="Lips">
select top 20 "$rowid$" as rowids,
vbeln, posnr, vgbel, vgpos, mandt, matnr, matkl, arktx, werks, lgort,
charg, lfimg, uebto, untto, erdat, erzet, aedat
from ${hana_user}.lips
where "$rowid$" &gt; #{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectPa0002CheckByUpdate" parameterType="Pa0002" resultType="Pa0002">
select top 20 "$rowid$" as rowids,
mandt,pernr,subty,objps,sprps,endda,begda,seqnr,nachn,vorna,aedtm
from ${hana_user}.Pa0002
where "$rowid$" &gt; #{rowids} and aedtm != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectVbakCheckByUpdate" parameterType="Vbak" resultType="Vbak">
select top 20 a."$rowid$" as rowids,
a.vbeln, a.mandt, a.vbtyp, a.audat, a.waerk, a.kalsm, a.ctlpc, a.kunnr, a.bukrs_vf, a.bstnk,
......@@ -430,9 +411,28 @@
where "$rowid$" &gt; #{rowids} and updat != '00000000' and updat = CURRENT_DATE ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLikpCheckByUpdate" parameterType="Likp" resultType="Likp">
select top 20 "$rowid$" as rowids,
vbeln, mandt, erdat, wadat_ist, ctlpc, kunnr, aedat,lfuhr
from ${hana_user}.likp
where "$rowid$" &gt;#{rowids} and aedat != '00000000' and aedat = CURRENT_DATE ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLipsCheckByUpdate" parameterType="Lips" resultType="Lips">
select top 20 "$rowid$" as rowids,
vbeln, posnr, vgbel, vgpos, mandt, matnr, matkl, arktx, werks, lgort,
charg, lfimg, uebto, untto, erdat, erzet, aedat
from ${hana_user}.lips
where "$rowid$" &gt; #{rowids} and aedat != '00000000' and aedat = CURRENT_DATE ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectPa0002CheckByUpdate" parameterType="Pa0002" resultType="Pa0002">
select top 20 "$rowid$" as rowids,
mandt,pernr,subty,objps,sprps,endda,begda,seqnr,nachn,vorna,aedtm
from ${hana_user}.Pa0002
where "$rowid$" &gt; #{rowids} and aedtm != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectAfkoById" parameterType="Afko" resultType="Afko">
......
src/main/resources/spring-init.xml
......@@ -705,24 +705,8 @@
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectLikpCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectLipsCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
......@@ -809,6 +793,33 @@
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkUpdateServiceImpl" />
<property name="targetMethod" value="selectLikpCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkUpdateServiceImpl" />
<property name="targetMethod" value="selectLipsCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkUpdateServiceImpl" />
<property name="targetMethod" value="selectPa0002CheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<!-- 删除流程 -->
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
......@@ -856,6 +867,33 @@
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkDeleteServiceImpl" />
<property name="targetMethod" value="selectLikpCheckByDelete" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkDeleteServiceImpl" />
<property name="targetMethod" value="selectLipsCheckByDelete" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="checkDeleteServiceImpl" />
<property name="targetMethod" value="selectPa0002CheckByDelete" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
</list>
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论