Skip to content

H
huazheng-project-flink
提交 d01dd345 作者: guofeng
操作

修改了增量更新的流程逻辑

上级 2c8d9d43
正在显示 包含 176 行增加 和 34 行删除
src/main/java/com/huazheng/project/greenplum/mapper/GPMapper.java
......@@ -223,7 +223,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectLikp'+':'+#p0.vbeln+','+#p0.mandt")
public void updateLikp(Likp element);
public List<Likp> selectLikpCheck(Likp build);
public List<Likp> selectLikpCheckByUpdate(Likp build);
@Cacheable(key = "#root.method.name+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt", unless="#result == null")
public Lips selectLips(Lips lips); // 查询替代删除
......@@ -233,7 +232,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectLips'+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt")
public void updateLips(Lips item);
public List<Lips> selectLipsCheck(Lips build);
public List<Lips> selectLipsCheckByUpdate(Lips build);
@Cacheable(key = "#root.method.name+':'+#p0.matnr+','+#p0.mandt+','+#p0.spras", unless="#result == null")
public Makt selectMakt(Makt makt); // 查询替代删除
......@@ -270,7 +268,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectPa0002'+':'+#p0.mandt+','+#p0.pernr+','+#p0.subty+','+#p0.objps+','+#p0.sprps+','+#p0.endda+','+#p0.begda+','+#p0.seqnr")
public void deletePa0002(Pa0002 pa0002);
public List<Pa0002> selectPa0002Check(Pa0002 build);
public List<Pa0002> selectPa0002CheckByUpdate(Pa0002 build);
@Cacheable(key = "#root.method.name+':'+#p0.mandt+','+#p0.ssour+','+#p0.vrsio+','+#p0.kkber+','+#p0.knkli+','+#p0.spmon+','+#p0.sptags+','+#p0.spwocs+','+#p0.spbups", unless="#result == null")
public S066 selectS066(S066 so66);
......@@ -316,7 +313,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectVbak'+':'+#p0.vbeln+','+#p0.mandt")
public void deleteVbak(Vbak item);
public List<Vbak> selectVbakCheck(Vbak build);
public List<Vbak> selectVbakCheckByUpdate(Vbak build);
@Cacheable(key = "#root.method.name+':'+#p0.vbeln+','+#p0.posnr+','+#p0.etenr+','+#p0.mandt", unless="#result == null")
public Vbep selectVbep(Vbep vbep); // 查询替代删除
......@@ -344,7 +340,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectVbrk'+':'+#p0.vbeln+','+#p0.mandt")
public void deleteVbrk(Vbrk item);
public List<Vbrk> selectVbrkCheck(Vbrk build);
public List<Vbrk> selectVbrkCheckByUpdate(Vbrk build);
@Cacheable(key = "#root.method.name+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt", unless="#result == null")
public Vbrp selectVbrp(Vbrp item); // 查询替代删除
......@@ -381,8 +376,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectZsd06'+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt")
public void deleteZsd06(Zsd06 item);
public List<Zsd06> selectZsd06Check(Zsd06 build);
public List<Zsd06> selectZsd06CheckByUpdate(Zsd06 build);
@Cacheable(key = "#root.method.name+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt", unless="#result == null")
public Zsdfhzl selectZsdfhzl(Zsdfhzl zsdfhzl); // 查询替代删除
......@@ -392,7 +385,6 @@ public interface GPMapper {
@CacheEvict(key = "'selectZsdfhzl'+':'+#p0.vbeln+','+#p0.posnr+','+#p0.mandt")
public void deleteZsdfhzl(Zsdfhzl item);
public List<Zsdfhzl> selectZsdfhzlCheck(Zsdfhzl build);
public List<Zsdfhzl> selectZsdfhzlCheckByUpdate(Zsdfhzl build);
@Cacheable(key = "#root.method.name+':'+#p0.mandt+','+#p0.bukrs+','+#p0.kunnr+','+#p0.umsks+','+#p0.umskz+','+#p0.augdt+','+#p0.augbl+','+#p0.zuonr+','+#p0.gjahr+','+#p0.belnr+','+#p0.buzei", unless="#result == null")
public Bsad selectBsad(Bsad bsad);
......
src/main/java/com/huazheng/project/hana/mapper/SapMapper.java
......@@ -85,6 +85,13 @@ public interface SapMapper {
public List<Knvp> selectKnvpNew(Knvp knvp);
public List<Knkk> selectKnkkCheckByUpdate(Knkk knkk);
public List<Likp> selectLikpCheckByUpdate(Likp likp);
public List<Lips> selectLipsCheckByUpdate(Lips lips);
public List<Pa0002> selectPa0002CheckByUpdate(Pa0002 pa0002);
public List<Vbak> selectVbakCheckByUpdate(Vbak vbak);
public List<Vbrk> selectVbrkCheckByUpdate(Vbrk vbrk);
public List<Zsd06> selectZsd06CheckByUpdate(Zsd06 zsd06);
public List<Zsdfhzl> selectZsdfhzlCheckByUpdate(Zsdfhzl zsdfhzl);
public Knvp selectKnvpById(Knvp target);
public Ausp selectAuspById(Ausp target);
......
src/main/resources/application.properties
......@@ -18,7 +18,7 @@ druid.hana.password: Hzxc888888
druid.hana.validationQuery: select 'hello world' from dummy
druid.hana.initialSize: 5
druid.hana.minIdle: 10
druid.hana.maxActive: 50
druid.hana.maxActive: 100
druid.hana.maxWait: 60000
druid.hana.timeBetweenEvictionRunsMillis: 60000
druid.hana.minEvictableIdleTimeMillis: 300000
......
src/main/resources/devtools/dev/hana2.sql
......@@ -2,4 +2,6 @@ select top 20 "$rowid$" as rowids,
mandt,kunnr,kkber,klimk,skfor,ssobl,aedat
from sapabap1.Knkk
where aedat != '00000000' and mandt = '800'
order by "$rowid$";
\ No newline at end of file
order by "$rowid$";
select "$rowid$" as rowids,lfimg,aedat from sapabap1.lips where vgbel = '0010073086' and vgpos = '000100' limit 10;
src/main/resources/devtools/doc/问题/20201115问题整理.txt 0 → 100644
20201115问题整理
20201115问题整理
1、全量、增量和删除任务的问题
问题：
1、全量更新表：数据量增多周期越来越慢。aufm、konv已停止执行更新删除任务，因为数据量增长速度过快，循环周期过长。
2、增量更新表：数据量增多周期越来越慢。增量更新首先是一个时间范围去取源表数据0-20rowid，然后取21-40rowid，取到最后的rowid，然后又循环到0-20的rowid。成为一个周期
随着数据量的增多，rowid的总量会越来越多，所以也会越来越慢。
3、由于新的表不断的加入，每个表都需要新增更新删除任务，相当于每个表需要占用3个数据库连接，1个新增数据、1个更新任务、1个删除任务。
目前有64个表，需要数仓占用250个连接，同时相对的sap那边也需要占用250个连接，这种查询压力是很大的，如果减小连接数，那么查询回排队，速度也会降低。
目前的设计是，一些任务被控制再一个连接内，所以速度进一步降低，如果展开一个表一个任务一个连接，那么数据库有超载查询的风险。
目前的思路：
1、全量更新和增量更新都，都取一部分的数据更新，比如取最新N条数据进行更新。周期缩短，最新的数据能得到最快的更新。抛弃历史数据的更新
会导致的问题：
1、N条以外的数据，删除不干净
2、N条以外的数据，更新不到（如果N=500此时有550条数据需要更新，那么50条数据没有更新到，因为并不知道多少数据需要更新）
全量更新和增量更新逻辑上处理是相同的，不同的是，增量更新加了时间条件，过滤了大部分数据，缩短了更新周期。
src/main/resources/mapper/greenplum/GPMapper_greenplum.xml
......@@ -803,9 +803,6 @@
<select id="selectLikpCheck" parameterType="com.huazheng.project.hana.model.Likp" resultType="com.huazheng.project.hana.model.Likp">
select * from Likp where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectLikpCheckByUpdate" parameterType="com.huazheng.project.hana.model.Likp" resultType="com.huazheng.project.hana.model.Likp">
select * from Likp where aedat &gt;= #{aedat} order by rownum
</select>
<select id="selectLips" parameterType="com.huazheng.project.hana.model.Lips" resultType="com.huazheng.project.hana.model.Lips">
select * from lips where vbeln = #{vbeln} and posnr = #{posnr} and mandt = #{mandt}
......@@ -829,9 +826,6 @@
<select id="selectLipsCheck" parameterType="com.huazheng.project.hana.model.Lips" resultType="com.huazheng.project.hana.model.Lips">
select * from Lips where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectLipsCheckByUpdate" parameterType="com.huazheng.project.hana.model.Lips" resultType="com.huazheng.project.hana.model.Lips">
select * from Lips where aedat &gt;= #{aedat} order by rownum
</select>
<select id="selectMakt" parameterType="com.huazheng.project.hana.model.Makt" resultType="com.huazheng.project.hana.model.Makt">
select * from makt where matnr = #{matnr} and mandt = #{mandt} and spras = #{spras}
......@@ -925,9 +919,6 @@
<select id="selectPa0002Check" parameterType="com.huazheng.project.hana.model.Pa0002" resultType="com.huazheng.project.hana.model.Pa0002">
select * from Pa0002 where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectPa0002CheckByUpdate" parameterType="com.huazheng.project.hana.model.Pa0002" resultType="com.huazheng.project.hana.model.Pa0002">
select * from Pa0002 where aedtm &gt;= #{aedtm} order by rownum
</select>
<select id="selectS066" parameterType="com.huazheng.project.hana.model.S066" resultType="com.huazheng.project.hana.model.S066">
select * from S066
......@@ -1051,9 +1042,6 @@
<select id="selectVbakCheck" parameterType="com.huazheng.project.hana.model.Vbak" resultType="com.huazheng.project.hana.model.Vbak">
select * from Vbak where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectVbakCheckByUpdate" parameterType="com.huazheng.project.hana.model.Vbak" resultType="com.huazheng.project.hana.model.Vbak">
select * from Vbak where aedat &gt;= #{aedat} order by rownum
</select>
<select id="selectVbep" parameterType="com.huazheng.project.hana.model.Vbep" resultType="com.huazheng.project.hana.model.Vbep">
select * from vbep
......@@ -1123,9 +1111,6 @@
<select id="selectVbrkCheck" parameterType="com.huazheng.project.hana.model.Vbrk" resultType="com.huazheng.project.hana.model.Vbrk">
select * from Vbrk where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectVbrkCheckByUpdate" parameterType="com.huazheng.project.hana.model.Vbrk" resultType="com.huazheng.project.hana.model.Vbrk">
select * from Vbrk where aedat &gt;= #{aedat} order by rownum
</select>
<select id="selectVbrp" parameterType="com.huazheng.project.hana.model.Vbrp" resultType="com.huazheng.project.hana.model.Vbrp">
select * from Vbrp
......@@ -1218,9 +1203,6 @@
<select id="selectZsd06Check" parameterType="com.huazheng.project.hana.model.Zsd06" resultType="com.huazheng.project.hana.model.Zsd06">
select * from Zsd06 where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectZsd06CheckByUpdate" parameterType="com.huazheng.project.hana.model.Zsd06" resultType="com.huazheng.project.hana.model.Zsd06">
select * from Zsd06 where cha_datum &gt;= #{cha_datum} order by rownum
</select>
<select id="selectZsdfhzl" parameterType="com.huazheng.project.hana.model.Zsdfhzl" resultType="com.huazheng.project.hana.model.Zsdfhzl">
select * from Zsdfhzl
......@@ -1246,9 +1228,6 @@
<select id="selectZsdfhzlCheck" parameterType="com.huazheng.project.hana.model.Zsdfhzl" resultType="com.huazheng.project.hana.model.Zsdfhzl">
select * from Zsdfhzl where rownum &gt; #{rowNum} order by rownum limit 20
</select>
<select id="selectZsdfhzlCheckByUpdate" parameterType="com.huazheng.project.hana.model.Zsdfhzl" resultType="com.huazheng.project.hana.model.Zsdfhzl">
select * from Zsdfhzl where cha_datum &gt;= #{cha_datum} order by rownum
</select>
<select id="selectBsad" parameterType="com.huazheng.project.hana.model.Bsad" resultType="com.huazheng.project.hana.model.Bsad">
select * from Bsad
......
src/main/resources/mapper/hana/SapMapper_hana.xml
......@@ -313,13 +313,84 @@
</select>
<select id="selectKnkkCheckByUpdate" parameterType="Knkk" resultType="Knkk">
select top 20
select top 20 "$rowid$" as rowids,
mandt,kunnr,kkber,klimk,skfor,ssobl,aedat
from ${hana_user}.Knkk
where "$rowid$" &gt; #{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLikpCheckByUpdate" parameterType="Likp" resultType="Likp">
select top 20 "$rowid$" as rowids,
vbeln, mandt, erdat, wadat_ist, ctlpc, kunnr, aedat,lfuhr,
case wadat_ist when '00000000' then null else (to_date(wadat_ist)||' '||to_time(lfuhr)) end as wadat_ist2lfuhr
from ${hana_user}.likp
where "$rowid$" &gt;#{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectLipsCheckByUpdate" parameterType="Lips" resultType="Lips">
select top 20 "$rowid$" as rowids,
vbeln, posnr, vgbel, vgpos, mandt, matnr, matkl, arktx, werks, lgort,
charg, lfimg, uebto, untto, erdat, erzet, aedat
from ${hana_user}.lips
where "$rowid$" &gt; #{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectPa0002CheckByUpdate" parameterType="Pa0002" resultType="Pa0002">
select top 20 "$rowid$" as rowids,
mandt,pernr,subty,objps,sprps,endda,begda,seqnr,nachn,vorna,aedtm
from ${hana_user}.Pa0002
where "$rowid$" &gt; #{rowids} and aedtm != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectVbakCheckByUpdate" parameterType="Vbak" resultType="Vbak">
select top 20 a."$rowid$" as rowids,
a.vbeln, a.mandt, a.vbtyp, a.audat, a.waerk, a.kalsm, a.ctlpc, a.kunnr, a.bukrs_vf, a.bstnk,
a.bname, a.telf1, a.netwr, a.vkbur, a.knumv, a.vkorg, a.vtweg, a.kkber, a.auart, a.aedat, b.pernr,
case a.erdat when '00000000' then null else (to_date(a.erdat)||' '||to_time(a.erzet)) end as erdat2erzet,
a.erdat, a.erzet, a.spart
from ${hana_user}.vbak a
left join (
select q.vbeln,r.pernr from ${hana_user}.vbap q
left join (
select vbeln,posnr,pernr from ${hana_user}.vbpa where mandt = #{mandt} and pernr != '00000000' group by vbeln,posnr,pernr
) r on q.vbeln = r.vbeln and q.posnr = r.posnr
where q.mandt = #{mandt} and r.pernr is not null
group by q.vbeln,r.pernr
) b on a.vbeln = b.vbeln
where "$rowid$" &gt; #{rowids} and a.aedat != '00000000' and a.mandt = '800'
order by "$rowid$"
</select>
<select id="selectVbrkCheckByUpdate" parameterType="com.huazheng.project.hana.model.Vbrk" resultType="com.huazheng.project.hana.model.Vbrk">
select top 20 "$rowid$" as rowids,
vbeln,fkart,waerk,erdat,erzet,zterm,kunrg,bstnk_vf,
zbillty,zbz01,zbz02,zdc_flg,bukrs,ktgrd,fkdat,inco1,
inco2,netwr,mwsbk,aedat,mandt
from ${hana_user}.Vbrk
where "$rowid$" &gt; #{rowids} and aedat != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectZsd06CheckByUpdate" parameterType="Zsd06" resultType="Zsd06">
select top 20 "$rowid$" as rowids,
vbeln, posnr, mandt, datum, uname, crt_uname, crt_datum,
crt_time, cha_name, cha_datum, cha_time
from ${hana_user}.Zsd_06
where "$rowid$" &gt; #{rowids} and cha_datum != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectZsdfhzlCheckByUpdate" parameterType="com.huazheng.project.hana.model.Zsdfhzl" resultType="com.huazheng.project.hana.model.Zsdfhzl">
select top 20 "$rowid$" as rowids,
vbeln, posnr, mandt, vgbel, vgpos, uname, matnr, fmenge, cdate, ctime, fh_date, werks, cha_datum
from ${hana_user}.zsd_fhzl
where "$rowid$" &gt; #{rowids} and cha_datum != '00000000' ${hana_mandt}
order by "$rowid$"
</select>
<select id="selectAfkoById" parameterType="Afko" resultType="Afko">
......
src/main/resources/quartz.properties
......@@ -11,7 +11,7 @@ org.quartz.scheduler.wrapJobExecutionInUserTransaction=false
# ThreadPool
#============================================================================
org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount=20
org.quartz.threadPool.threadCount=30
org.quartz.threadPool.threadPriority=5
org.quartz.threadPool.threadsInheritContextClassLoaderOfInitializingThread=true
......
src/main/resources/spring-init.xml
......@@ -695,6 +695,79 @@
<property name="cronExpression" value="* * * * * ?" />
</bean>
<!-- ========================== 独立的任务 ========================== -->
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectKnkkCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectLikpCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectLipsCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectPa0002CheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectVbakCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectVbrkCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectZsd06CheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean parent="methodJobDetail">
<property name="targetObject" ref="deleteUpdateJobServiceImpl" />
<property name="targetMethod" value="selectZsdfhzlCheckUpdate" />
</bean>
</property>
<property name="cronExpression" value="* * * * * ?" />
</bean>
</list>
</constructor-arg>
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论