ctripcorp/zeus
1
1
Fork 0
mirror of https://github.com/ctripcorp/zeus.git synced 2024-09-20 23:56:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

add batch insert

Browse source
This commit is contained in:
fanqq 2015-05-12 19:33:46 +08:00
parent 6c0381e27d
commit 685a3d343f
6 changed files with 50 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pom.xml
View file

@ -12,7 +12,7 @@
<dependency>
<groupId>org.unidal.framework</groupId>
<artifactId>dal-jdbc</artifactId>
<version>2.1.3</version>
<version>2.2.4</version>
</dependency>
<dependency>
<groupId>com.dianping.cat</groupId>

45
src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java
View file

@ -222,30 +222,6 @@ public class NginxConfServiceImpl implements NginxConfService {
apps.add(appSlb.getAppName());
}
// List<AppSlb> appSlbList = slbClusterRepository.listAppSlbsBySlb(slbName);
//
// if (appSlbList==null)
// {
// appSlbList = new ArrayList<>();
// }
//
// for (AppSlb appslb : appSlbList)
// {
// VirtualServer vs = appslb.getVirtualServer();
// String vsstr = vs.getName();
//
// Set<String> apps = appNamesMap.get(vsstr);
// if (apps==null)
// {
// apps = new HashSet<>();
// appNamesMap.put(vsstr,apps);
// }
//
// apps.add(appslb.getAppName());
// }
Map<String, List<App>> appsMap = new HashMap<>();
for (String vs : appNamesMap.keySet()) {
List<String> l = activeConfService.getConfAppActiveContentByAppNames(appNamesMap.get(vs).toArray(new String[]{}));
@ -278,6 +254,12 @@ public class NginxConfServiceImpl implements NginxConfService {
Set<String> allDownServers = statusService.findAllDownServers();
Set<String> allDownAppServers = statusService.findAllDownAppServersBySlbName(slbName);
int length = slb.getVirtualServers().size();
NginxConfServerDo[] nginxConfServerDos = new NginxConfServerDo[length];
NginxConfUpstreamDo[] nginxConfUpstreamDos = new NginxConfUpstreamDo[length];
int index = 0 ;
for (VirtualServer vs : slb.getVirtualServers()) {
List<App> apps = appsMap.get(vs.getName());
if (apps == null) {
@ -287,21 +269,22 @@ public class NginxConfServiceImpl implements NginxConfService {
String serverConf = nginxConfigBuilder.generateServerConf(slb, vs, apps);
String upstreamConf = nginxConfigBuilder.generateUpstreamsConf(slb, vs, apps, allDownServers, allDownAppServers);
nginxConfServerDao.insert(new NginxConfServerDo().setCreatedTime(new Date())
nginxConfServerDos[index] = new NginxConfServerDo().setCreatedTime(new Date())
.setSlbName(slb.getName())
.setName(vs.getName())
.setContent(serverConf)
.setVersion(version));
.setVersion(version);
logger.debug("Nginx Server Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version);
nginxConfUpstreamDao.insert(new NginxConfUpstreamDo().setCreatedTime(new Date())
nginxConfUpstreamDos[index] = new NginxConfUpstreamDo().setCreatedTime(new Date())
.setSlbName(slb.getName())
.setName(vs.getName())
.setContent(upstreamConf)
.setVersion(version));
.setVersion(version);
index++;
logger.debug("Nginx Server Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version);
logger.debug("Nginx Upstream Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version);
}
nginxConfServerDao.insert(nginxConfServerDos);
nginxConfUpstreamDao.insert(nginxConfUpstreamDos);
}
}

8
src/main/resources/META-INF/dal/jdbc/core-nginx.xml
View file

@ -39,8 +39,10 @@
</query-defs>
</entity>
<entity name="nginx-conf-server" table="nginx_conf_server" alias="ncs">
<member name="created-time" field="created_time" value-type="Date" insert-expr="now()"/>
<member name="last-modified" field="last_modified" value-type="Date" nullable="false" insert-expr="now()" update-expr="now()"/>
<query-defs>
<query name="insert" type="INSERT">
<query name="insert" type="INSERT" batch="true">
<statement>
<![CDATA[
INSERT INTO <TABLE/>(<FIELDS/>)
@ -77,8 +79,10 @@
</query-defs>
</entity>
<entity name="nginx-conf-upstream" table="nginx_conf_upstream" alias="ncs">
<member name="created-time" field="created_time" value-type="Date" insert-expr="now()"/>
<member name="last-modified" field="last_modified" value-type="Date" nullable="false" insert-expr="now()" update-expr="now()"/>
<query-defs>
<query name="insert" type="INSERT">
<query name="insert" type="INSERT" batch="true">
<statement>
<![CDATA[
INSERT INTO <TABLE/>(<FIELDS/>)

29
src/main/resources/META-INF/plexus/components.xml
View file

@ -13,7 +13,6 @@
<role-hint>app</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>app</logical-table-name>
<physical-table-name>app</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -23,7 +22,6 @@
<role-hint>app-health-check</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>app-health-check</logical-table-name>
<physical-table-name>app_health_check</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -33,7 +31,6 @@
<role-hint>app-load-balancing-method</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>app-load-balancing-method</logical-table-name>
<physical-table-name>app_load_balancing_method</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -43,7 +40,6 @@
<role-hint>app-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>app-server</logical-table-name>
<physical-table-name>app_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -53,7 +49,6 @@
<role-hint>app-slb</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>app-slb</logical-table-name>
<physical-table-name>app_slb</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -63,7 +58,6 @@
<role-hint>archive-app</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>archive-app</logical-table-name>
<physical-table-name>archive_app</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -73,7 +67,6 @@
<role-hint>archive-slb</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>archive-slb</logical-table-name>
<physical-table-name>archive_slb</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -83,7 +76,6 @@
<role-hint>auth-resource</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>auth-resource</logical-table-name>
<physical-table-name>auth_resource</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -93,7 +85,6 @@
<role-hint>auth-resource-role</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>auth-resource-role</logical-table-name>
<physical-table-name>auth_resource_role</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -103,7 +94,6 @@
<role-hint>auth-role</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>auth-role</logical-table-name>
<physical-table-name>auth_role</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -113,7 +103,6 @@
<role-hint>auth-user</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>auth-user</logical-table-name>
<physical-table-name>auth_user</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -123,7 +112,6 @@
<role-hint>auth-user-role</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>auth-user-role</logical-table-name>
<physical-table-name>auth_user_role</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -133,7 +121,6 @@
<role-hint>build-info</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>build-info</logical-table-name>
<physical-table-name>build_info</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -143,7 +130,6 @@
<role-hint>conf-app-active</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>conf-app-active</logical-table-name>
<physical-table-name>conf_app_active</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -153,7 +139,6 @@
<role-hint>conf-app-slb-active</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>conf-app-slb-active</logical-table-name>
<physical-table-name>conf_app_slb_active</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -163,7 +148,6 @@
<role-hint>conf-slb-active</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>conf-slb-active</logical-table-name>
<physical-table-name>conf_slb_active</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -173,7 +157,6 @@
<role-hint>dist-lock</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>dist-lock</logical-table-name>
<physical-table-name>dist_lock</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -183,7 +166,6 @@
<role-hint>nginx-conf</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>nginx-conf</logical-table-name>
<physical-table-name>nginx_conf</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -193,7 +175,6 @@
<role-hint>nginx-conf-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>nginx-conf-server</logical-table-name>
<physical-table-name>nginx_conf_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -203,7 +184,6 @@
<role-hint>nginx-conf-upstream</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>nginx-conf-upstream</logical-table-name>
<physical-table-name>nginx_conf_upstream</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -213,7 +193,6 @@
<role-hint>nginx-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>nginx-server</logical-table-name>
<physical-table-name>nginx_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -223,7 +202,6 @@
<role-hint>server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>server</logical-table-name>
<physical-table-name>server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -233,7 +211,6 @@
<role-hint>slb</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>slb</logical-table-name>
<physical-table-name>slb</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -243,7 +220,6 @@
<role-hint>slb-domain</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>slb-domain</logical-table-name>
<physical-table-name>slb_domain</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -253,7 +229,6 @@
<role-hint>slb-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>slb-server</logical-table-name>
<physical-table-name>slb_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -263,7 +238,6 @@
<role-hint>slb-vip</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>slb-vip</logical-table-name>
<physical-table-name>slb_vip</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -273,7 +247,6 @@
<role-hint>slb-virtual-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>slb-virtual-server</logical-table-name>
<physical-table-name>slb_virtual_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -283,7 +256,6 @@
<role-hint>status-app-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>status-app-server</logical-table-name>
<physical-table-name>status_app_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>
@ -293,7 +265,6 @@
<role-hint>status-server</role-hint>
<implementation>org.unidal.dal.jdbc.mapping.SimpleTableProvider</implementation>
<configuration>
<logical-table-name>status-server</logical-table-name>
<physical-table-name>status_server</physical-table-name>
<data-source-name>zeus</data-source-name>
</configuration>

28
src/test/java/com/ctrip/zeus/service/DalPerfTest.java
View file

@ -40,7 +40,8 @@ public class DalPerfTest extends AbstractSpringTest {
}
@Test
public void perfTest()throws Exception{
NginxConfUpstreamDo tmpdata = new NginxConfUpstreamDo().setCreatedTime(new Date())
NginxConfUpstreamDo tmpdata = new NginxConfUpstreamDo()
.setContent("upstream backend_VS_1007_App_1007 {\n" +
" server 10.2.25.83:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" server 10.2.25.93:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
@ -53,12 +54,31 @@ public class DalPerfTest extends AbstractSpringTest {
" check_http_expect_alive http_2xx http_3xx;\n" +
" }\n");
tmpdata.setSlbName("slbname2_").setVersion(0).setName("vs_"+5);
nginxConfUpstreamDao.insert(tmpdata);
long start = new Date().getTime();
for (int i = 0 ; i < 1000 ; i ++)
NginxConfUpstreamDo[] dos = new NginxConfUpstreamDo[10000];
for (int i = 0 ; i < 10000 ; i ++)
{
tmpdata.setSlbName("slbname_"+i).setVersion(i).setName("vs_"+i);
nginxConfUpstreamDao.insert(tmpdata);
tmpdata = new NginxConfUpstreamDo()
.setContent("upstream backend_VS_1007_App_1007 {\n" +
" server 10.2.25.83:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" server 10.2.25.93:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" server 10.2.25.94:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" server 10.2.25.95:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" server 10.2.25.96:20004 weight=1 max_fails=10 fail_timeout=30;\n" +
" check interval=2000 rise=1 fall=1 timeout=1000 type=http;\n" +
" check_keepalive_requests 100;\n" +
" check_http_send \"GET /checkHealth HTTP/1.0\\r\\n Connection: keep-alive\\r\\n Host: 1007.ctrip.com\\r\\n\\r\\n\";\n" +
" check_http_expect_alive http_2xx http_3xx;\n" +
" }\n");
tmpdata.setSlbName("slbname5_"+i).setVersion(i).setName("vs_"+i);
dos[i]=tmpdata;
}
nginxConfUpstreamDao.insert(dos);
long end = new Date().getTime();
System.out.println("Time cost :"+(end-start));
}

10
src/test/java/test/EnvFactory.java
View file

@ -11,11 +11,11 @@ import java.util.List;
* Created by fanqq on 2015/5/8.
*/
public class EnvFactory {
private static final String host = "http://127.0.0.1:8099";
// private static final String host = "http://10.2.27.21:8099";
// private static final String host = "http://127.0.0.1:8099";
private static final String host = "http://10.2.27.21:8099";
private static final String vip="10.2.27.21";
private static final int vsNum=10000;
private static final int appNum=10000;
private static final int vsNum=1000;
private static final int appNum=5000;
private static final String slbName="test-env";
private static final String checkHealthPath="/checkHealth";
private static ReqClient reqClient = new ReqClient(host);
@ -80,7 +80,7 @@ public class EnvFactory {
for ( int i = 0 ; i < appNum ; i++)
{
apptmp = new App().setName("App_" + i).setAppId(String.valueOf(100000 + i)).setVersion(1).setHealthCheck(new HealthCheck().setFails(1)
.setIntervals(2000).setPasses(1).setUri(checkHealthPath)).setLoadBalancingMethod(new LoadBalancingMethod().setType("roundrobin")
.setIntervals(2000*appNum/100).setPasses(1).setUri(checkHealthPath)).setLoadBalancingMethod(new LoadBalancingMethod().setType("roundrobin")
.setValue("test"))
.addAppSlb(new AppSlb().setSlbName(slbName).setPath("/App" + i)
.setVirtualServer(vsList.get(i % vsNum)));