From 685a3d343fde19052ad5d089e110ac0ac7522b37 Mon Sep 17 00:00:00 2001 From: fanqq Date: Tue, 12 May 2015 19:33:46 +0800 Subject: [PATCH] add batch insert --- pom.xml | 2 +- .../build/impl/NginxConfServiceImpl.java | 45 ++++++------------- .../META-INF/dal/jdbc/core-nginx.xml | 8 +++- .../resources/META-INF/plexus/components.xml | 29 ------------ .../com/ctrip/zeus/service/DalPerfTest.java | 28 ++++++++++-- src/test/java/test/EnvFactory.java | 10 ++--- 6 files changed, 50 insertions(+), 72 deletions(-) diff --git a/pom.xml b/pom.xml index 4e45cc70..4299b15e 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.unidal.framework dal-jdbc - 2.1.3 + 2.2.4 com.dianping.cat diff --git a/src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java b/src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java index 5784d003..8cf6e2fc 100644 --- a/src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java +++ b/src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java @@ -222,30 +222,6 @@ public class NginxConfServiceImpl implements NginxConfService { apps.add(appSlb.getAppName()); } - -// List appSlbList = slbClusterRepository.listAppSlbsBySlb(slbName); -// -// if (appSlbList==null) -// { -// appSlbList = new ArrayList<>(); -// } -// -// for (AppSlb appslb : appSlbList) -// { -// VirtualServer vs = appslb.getVirtualServer(); -// String vsstr = vs.getName(); -// -// Set apps = appNamesMap.get(vsstr); -// if (apps==null) -// { -// apps = new HashSet<>(); -// appNamesMap.put(vsstr,apps); -// } -// -// apps.add(appslb.getAppName()); -// } - - Map> appsMap = new HashMap<>(); for (String vs : appNamesMap.keySet()) { List l = activeConfService.getConfAppActiveContentByAppNames(appNamesMap.get(vs).toArray(new String[]{})); @@ -278,6 +254,12 @@ public class NginxConfServiceImpl implements NginxConfService { Set allDownServers = statusService.findAllDownServers(); Set allDownAppServers = statusService.findAllDownAppServersBySlbName(slbName); + + int length = slb.getVirtualServers().size(); + NginxConfServerDo[] nginxConfServerDos = new NginxConfServerDo[length]; + NginxConfUpstreamDo[] nginxConfUpstreamDos = new NginxConfUpstreamDo[length]; + int index = 0 ; + for (VirtualServer vs : slb.getVirtualServers()) { List apps = appsMap.get(vs.getName()); if (apps == null) { @@ -287,21 +269,22 @@ public class NginxConfServiceImpl implements NginxConfService { String serverConf = nginxConfigBuilder.generateServerConf(slb, vs, apps); String upstreamConf = nginxConfigBuilder.generateUpstreamsConf(slb, vs, apps, allDownServers, allDownAppServers); - nginxConfServerDao.insert(new NginxConfServerDo().setCreatedTime(new Date()) + nginxConfServerDos[index] = new NginxConfServerDo().setCreatedTime(new Date()) .setSlbName(slb.getName()) .setName(vs.getName()) .setContent(serverConf) - .setVersion(version)); + .setVersion(version); - logger.debug("Nginx Server Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version); - - nginxConfUpstreamDao.insert(new NginxConfUpstreamDo().setCreatedTime(new Date()) + nginxConfUpstreamDos[index] = new NginxConfUpstreamDo().setCreatedTime(new Date()) .setSlbName(slb.getName()) .setName(vs.getName()) .setContent(upstreamConf) - .setVersion(version)); + .setVersion(version); + index++; + logger.debug("Nginx Server Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version); logger.debug("Nginx Upstream Conf build sucess! slbName: "+slb+",virtualserver: "+vs.getName()+",version: "+version); } - + nginxConfServerDao.insert(nginxConfServerDos); + nginxConfUpstreamDao.insert(nginxConfUpstreamDos); } } diff --git a/src/main/resources/META-INF/dal/jdbc/core-nginx.xml b/src/main/resources/META-INF/dal/jdbc/core-nginx.xml index 30917634..223606fb 100644 --- a/src/main/resources/META-INF/dal/jdbc/core-nginx.xml +++ b/src/main/resources/META-INF/dal/jdbc/core-nginx.xml @@ -39,8 +39,10 @@ + + - + () @@ -77,8 +79,10 @@ + + - + () diff --git a/src/main/resources/META-INF/plexus/components.xml b/src/main/resources/META-INF/plexus/components.xml index 39214aca..f0ee7ccb 100644 --- a/src/main/resources/META-INF/plexus/components.xml +++ b/src/main/resources/META-INF/plexus/components.xml @@ -13,7 +13,6 @@ app org.unidal.dal.jdbc.mapping.SimpleTableProvider - app app zeus @@ -23,7 +22,6 @@ app-health-check org.unidal.dal.jdbc.mapping.SimpleTableProvider - app-health-check app_health_check zeus @@ -33,7 +31,6 @@ app-load-balancing-method org.unidal.dal.jdbc.mapping.SimpleTableProvider - app-load-balancing-method app_load_balancing_method zeus @@ -43,7 +40,6 @@ app-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - app-server app_server zeus @@ -53,7 +49,6 @@ app-slb org.unidal.dal.jdbc.mapping.SimpleTableProvider - app-slb app_slb zeus @@ -63,7 +58,6 @@ archive-app org.unidal.dal.jdbc.mapping.SimpleTableProvider - archive-app archive_app zeus @@ -73,7 +67,6 @@ archive-slb org.unidal.dal.jdbc.mapping.SimpleTableProvider - archive-slb archive_slb zeus @@ -83,7 +76,6 @@ auth-resource org.unidal.dal.jdbc.mapping.SimpleTableProvider - auth-resource auth_resource zeus @@ -93,7 +85,6 @@ auth-resource-role org.unidal.dal.jdbc.mapping.SimpleTableProvider - auth-resource-role auth_resource_role zeus @@ -103,7 +94,6 @@ auth-role org.unidal.dal.jdbc.mapping.SimpleTableProvider - auth-role auth_role zeus @@ -113,7 +103,6 @@ auth-user org.unidal.dal.jdbc.mapping.SimpleTableProvider - auth-user auth_user zeus @@ -123,7 +112,6 @@ auth-user-role org.unidal.dal.jdbc.mapping.SimpleTableProvider - auth-user-role auth_user_role zeus @@ -133,7 +121,6 @@ build-info org.unidal.dal.jdbc.mapping.SimpleTableProvider - build-info build_info zeus @@ -143,7 +130,6 @@ conf-app-active org.unidal.dal.jdbc.mapping.SimpleTableProvider - conf-app-active conf_app_active zeus @@ -153,7 +139,6 @@ conf-app-slb-active org.unidal.dal.jdbc.mapping.SimpleTableProvider - conf-app-slb-active conf_app_slb_active zeus @@ -163,7 +148,6 @@ conf-slb-active org.unidal.dal.jdbc.mapping.SimpleTableProvider - conf-slb-active conf_slb_active zeus @@ -173,7 +157,6 @@ dist-lock org.unidal.dal.jdbc.mapping.SimpleTableProvider - dist-lock dist_lock zeus @@ -183,7 +166,6 @@ nginx-conf org.unidal.dal.jdbc.mapping.SimpleTableProvider - nginx-conf nginx_conf zeus @@ -193,7 +175,6 @@ nginx-conf-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - nginx-conf-server nginx_conf_server zeus @@ -203,7 +184,6 @@ nginx-conf-upstream org.unidal.dal.jdbc.mapping.SimpleTableProvider - nginx-conf-upstream nginx_conf_upstream zeus @@ -213,7 +193,6 @@ nginx-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - nginx-server nginx_server zeus @@ -223,7 +202,6 @@ server org.unidal.dal.jdbc.mapping.SimpleTableProvider - server server zeus @@ -233,7 +211,6 @@ slb org.unidal.dal.jdbc.mapping.SimpleTableProvider - slb slb zeus @@ -243,7 +220,6 @@ slb-domain org.unidal.dal.jdbc.mapping.SimpleTableProvider - slb-domain slb_domain zeus @@ -253,7 +229,6 @@ slb-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - slb-server slb_server zeus @@ -263,7 +238,6 @@ slb-vip org.unidal.dal.jdbc.mapping.SimpleTableProvider - slb-vip slb_vip zeus @@ -273,7 +247,6 @@ slb-virtual-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - slb-virtual-server slb_virtual_server zeus @@ -283,7 +256,6 @@ status-app-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - status-app-server status_app_server zeus @@ -293,7 +265,6 @@ status-server org.unidal.dal.jdbc.mapping.SimpleTableProvider - status-server status_server zeus diff --git a/src/test/java/com/ctrip/zeus/service/DalPerfTest.java b/src/test/java/com/ctrip/zeus/service/DalPerfTest.java index 1acdaea8..5a0abe58 100644 --- a/src/test/java/com/ctrip/zeus/service/DalPerfTest.java +++ b/src/test/java/com/ctrip/zeus/service/DalPerfTest.java @@ -40,7 +40,8 @@ public class DalPerfTest extends AbstractSpringTest { } @Test public void perfTest()throws Exception{ - NginxConfUpstreamDo tmpdata = new NginxConfUpstreamDo().setCreatedTime(new Date()) + + NginxConfUpstreamDo tmpdata = new NginxConfUpstreamDo() .setContent("upstream backend_VS_1007_App_1007 {\n" + " server 10.2.25.83:20004 weight=1 max_fails=10 fail_timeout=30;\n" + " server 10.2.25.93:20004 weight=1 max_fails=10 fail_timeout=30;\n" + @@ -53,12 +54,31 @@ public class DalPerfTest extends AbstractSpringTest { " check_http_expect_alive http_2xx http_3xx;\n" + " }\n"); + tmpdata.setSlbName("slbname2_").setVersion(0).setName("vs_"+5); + nginxConfUpstreamDao.insert(tmpdata); + long start = new Date().getTime(); - for (int i = 0 ; i < 1000 ; i ++) + NginxConfUpstreamDo[] dos = new NginxConfUpstreamDo[10000]; + + for (int i = 0 ; i < 10000 ; i ++) { - tmpdata.setSlbName("slbname_"+i).setVersion(i).setName("vs_"+i); - nginxConfUpstreamDao.insert(tmpdata); + tmpdata = new NginxConfUpstreamDo() + .setContent("upstream backend_VS_1007_App_1007 {\n" + + " server 10.2.25.83:20004 weight=1 max_fails=10 fail_timeout=30;\n" + + " server 10.2.25.93:20004 weight=1 max_fails=10 fail_timeout=30;\n" + + " server 10.2.25.94:20004 weight=1 max_fails=10 fail_timeout=30;\n" + + " server 10.2.25.95:20004 weight=1 max_fails=10 fail_timeout=30;\n" + + " server 10.2.25.96:20004 weight=1 max_fails=10 fail_timeout=30;\n" + + " check interval=2000 rise=1 fall=1 timeout=1000 type=http;\n" + + " check_keepalive_requests 100;\n" + + " check_http_send \"GET /checkHealth HTTP/1.0\\r\\n Connection: keep-alive\\r\\n Host: 1007.ctrip.com\\r\\n\\r\\n\";\n" + + " check_http_expect_alive http_2xx http_3xx;\n" + + " }\n"); + + tmpdata.setSlbName("slbname5_"+i).setVersion(i).setName("vs_"+i); + dos[i]=tmpdata; } + nginxConfUpstreamDao.insert(dos); long end = new Date().getTime(); System.out.println("Time cost :"+(end-start)); } diff --git a/src/test/java/test/EnvFactory.java b/src/test/java/test/EnvFactory.java index f4bd7a3a..c354709e 100644 --- a/src/test/java/test/EnvFactory.java +++ b/src/test/java/test/EnvFactory.java @@ -11,11 +11,11 @@ import java.util.List; * Created by fanqq on 2015/5/8. */ public class EnvFactory { - private static final String host = "http://127.0.0.1:8099"; -// private static final String host = "http://10.2.27.21:8099"; +// private static final String host = "http://127.0.0.1:8099"; + private static final String host = "http://10.2.27.21:8099"; private static final String vip="10.2.27.21"; - private static final int vsNum=10000; - private static final int appNum=10000; + private static final int vsNum=1000; + private static final int appNum=5000; private static final String slbName="test-env"; private static final String checkHealthPath="/checkHealth"; private static ReqClient reqClient = new ReqClient(host); @@ -80,7 +80,7 @@ public class EnvFactory { for ( int i = 0 ; i < appNum ; i++) { apptmp = new App().setName("App_" + i).setAppId(String.valueOf(100000 + i)).setVersion(1).setHealthCheck(new HealthCheck().setFails(1) - .setIntervals(2000).setPasses(1).setUri(checkHealthPath)).setLoadBalancingMethod(new LoadBalancingMethod().setType("roundrobin") + .setIntervals(2000*appNum/100).setPasses(1).setUri(checkHealthPath)).setLoadBalancingMethod(new LoadBalancingMethod().setType("roundrobin") .setValue("test")) .addAppSlb(new AppSlb().setSlbName(slbName).setPath("/App" + i) .setVirtualServer(vsList.get(i % vsNum)));