manage conf read

src/main/java/com/ctrip/zeus/service/build/NginxConfService.java

@@ -1,14 +1,9 @@
 package com.ctrip.zeus.service.build;
 
-import com.ctrip.zeus.model.entity.DyUpstreamOpsData;
-import com.ctrip.zeus.model.entity.NginxConfServerData;
-import com.ctrip.zeus.model.entity.NginxConfUpstreamData;
-import com.ctrip.zeus.model.entity.Slb;
-import com.ctrip.zeus.nginx.entity.VsConfData;
+import com.ctrip.zeus.nginx.entity.NginxConfEntry;
 import com.ctrip.zeus.service.Repository;
 
 import java.util.List;
-import java.util.Map;
 
 /**
  * @author:xingchaowang
@@ -27,45 +22,24 @@ public interface NginxConfService extends Repository {
     public String getNginxConf(Long slbId, Long version) throws Exception;
 
     /**
-     * get nginx server config by slb name and version number
-     *
-     * @param slbId   the slb name
-     * @param version the version number
-     * @return nginx server config data list
+     * Get nginx conf under dir upstreams/ and vhosts/
+     * @param slbId
+     * @param version
+     * @return nginx conf entry which contains upstream and vhost conf files
      * @throws Exception
      */
-    public List<NginxConfServerData> getNginxConfServer(Long slbId, Long version) throws Exception;
-
-    /**
-     * get nginx upstream config by slb name and version number
-     *
-     * @param slbId   the slb name
-     * @param version the version number
-     * @return nginx upstream config data list
-     * @throws Exception
-     */
-    public List<NginxConfUpstreamData> getNginxConfUpstream(Long slbId, Long version) throws Exception;
-
-    /**
-     * get nginx upstream&server config by slb id and version number
-     *
-     * @param slbId   the slb name
-     * @param version the version number
-     * @return nginx upstream config data list
-     * @throws Exception
-     */
-    public Map<Long,VsConfData> getVsConfBySlbId(Long slbId, Long version) throws Exception;
+    public NginxConfEntry getUpstreamsAndVhosts(Long slbId, Long version) throws Exception;
 
       /**
      * get nginx upstream&server config by slb id , vsIds and version number
      *
-     * @param slbId   the slb name
-     * @param version the version number
-     * @param vsIds the vsIds
-     * @return nginx upstream config data list
+     * @param slbId
+     * @param version selected slb conf version
+     * @param vsIds selected vs ids
+     * @return nginx conf entry which contains upstream and vhost conf files of selected vs ids
      * @throws Exception
      */
-    public Map<Long,VsConfData> getVsConfByVsIds(Long slbId, List<Long> vsIds, Long version) throws Exception;
+    public NginxConfEntry getUpstreamsAndVhosts(Long slbId, Long version, List<Long> vsIds) throws Exception;
 
     /**
      * get current building version by slb name

src/main/java/com/ctrip/zeus/service/build/impl/NginxConfServiceImpl.java

@@ -4,7 +4,8 @@ import com.ctrip.zeus.dal.core.*;
 import com.ctrip.zeus.exceptions.ValidationException;
 import com.ctrip.zeus.model.entity.NginxConfServerData;
 import com.ctrip.zeus.model.entity.NginxConfUpstreamData;
-import com.ctrip.zeus.nginx.entity.VsConfData;
+import com.ctrip.zeus.nginx.entity.*;
+import com.ctrip.zeus.nginx.transform.DefaultJsonParser;
 import com.ctrip.zeus.service.build.BuildInfoService;
 import com.ctrip.zeus.service.build.NginxConfBuilder;
 import com.ctrip.zeus.service.build.NginxConfService;
@@ -25,24 +26,18 @@ import java.util.Map;
  */
 @Service("nginxConfService")
 public class NginxConfServiceImpl implements NginxConfService {
-
     @Resource
     private NginxConfDao nginxConfDao;
     @Resource
-    private NginxConfServerDao nginxConfServerDao;
-    @Resource
-    private NginxConfUpstreamDao nginxConfUpstreamDao;
+    private NginxConfSlbDao nginxConfSlbDao;
     @Resource
     private BuildInfoService buildInfoService;
-    @Resource
-    ConfGroupSlbActiveDao confGroupSlbActiveDao;
-
 
     private Logger logger = LoggerFactory.getLogger(NginxConfServiceImpl.class);
 
     @Override
     public String getNginxConf(Long slbId, Long version) throws Exception {
-        NginxConfDo nginxConfDo = nginxConfDao.findBySlbIdAndVersion(slbId, (int) (long) version, NginxConfEntity.READSET_FULL);
+        NginxConfDo nginxConfDo = nginxConfDao.findBySlbIdAndVersion(slbId, (int) version.longValue(), NginxConfEntity.READSET_FULL);
         if (nginxConfDo == null) {
             throw new ValidationException("Not found nginx conf by slbId and version. slbId:" + slbId + " version:" + version);
         }
@@ -50,83 +45,57 @@ public class NginxConfServiceImpl implements NginxConfService {
     }
 
     @Override
-    public List<NginxConfServerData> getNginxConfServer(Long slbId, Long version) throws Exception {
-        List<NginxConfServerData> result = new ArrayList<>();
-        List<NginxConfServerDo> d = nginxConfServerDao.findAllBySlbIdAndVersion(slbId, (int) (long) version, NginxConfServerEntity.READSET_FULL);
-        if (d == null || d.size() == 0) {
-            logger.warn("Not found nginx server conf by slbId and version. slbId:" + slbId + " version:" + version);
-            return result;
+    public NginxConfEntry getUpstreamsAndVhosts(Long slbId, Long version) throws Exception {
+        NginxConfSlbDo d = nginxConfSlbDao.findBySlbAndVersion(slbId, version, NginxConfSlbEntity.READSET_FULL);
+        if (d == null) return null;
+
+        NginxConfEntry entry = DefaultJsonParser.parse(NginxConfEntry.class, d.getContent());
+        if (entry.getUpstreams().getFiles().size() == 0 || entry.getVhosts().getFiles().size() == 0) {
+            logger.warn("No vhost or upstream files exists. "
+                    + "Upstream size: " + entry.getUpstreams().getFiles().size()
+                    + ", Vhost size: " + entry.getVhosts().getFiles().size() + ".");
         }
-        for (NginxConfServerDo t : d) {
-            result.add(new NginxConfServerData().setVsId(t.getSlbVirtualServerId()).setContent(t.getContent()));
-        }
-        return result;
+        return entry;
     }
 
     @Override
-    public List<NginxConfUpstreamData> getNginxConfUpstream(Long slbId, Long version) throws Exception {
-        List<NginxConfUpstreamData> result = new ArrayList<>();
-        List<NginxConfUpstreamDo> d = nginxConfUpstreamDao.findAllBySlbIdAndVersion(slbId, (int) (long) version, NginxConfUpstreamEntity.READSET_FULL);
-        if (d == null || d.size() == 0) {
-            logger.warn("Not found nginx upstream conf by slbId and version. slbId:" + slbId + " version:" + version);
-            return result;
-        }
-        for (NginxConfUpstreamDo t : d) {
-            result.add(new NginxConfUpstreamData().setVsId(t.getSlbVirtualServerId()).setContent(t.getContent()));
-        }
-        return result;
-    }
+    public NginxConfEntry getUpstreamsAndVhosts(Long slbId, Long version, List<Long> vsIds) throws Exception {
+        NginxConfEntry entry = new NginxConfEntry().setUpstreams(new Upstreams()).setVhosts(new Vhosts());
+        NginxConfEntry completeEntry = getUpstreamsAndVhosts(slbId, version);
+        if (completeEntry == null) return null;
 
-    @Override
-    public Map<Long, VsConfData> getVsConfBySlbId(Long slbId, Long version) throws Exception {
-        List<NginxConfServerData> confServerDataList = getNginxConfServer(slbId, version);
-        List<NginxConfUpstreamData> confUpstreamDataList = getNginxConfUpstream(slbId, version);
-        if (confServerDataList.size() != confUpstreamDataList.size()) {
-            throw new ValidationException("Count of server conf and count of upstream conf is different.");
-        }
-        Map<Long, VsConfData> dataMap = new HashMap<>();
-        for (NginxConfUpstreamData upstream : confUpstreamDataList) {
-            VsConfData vsConfData = new VsConfData();
-            Long vsId = upstream.getVsId();
-            vsConfData.setUpstreamConf(upstream.getContent());
-            dataMap.put(vsId, vsConfData);
-        }
-        for (NginxConfServerData sd : confServerDataList) {
-            if (dataMap.containsKey(sd.getVsId())) {
-                dataMap.get(sd.getVsId()).setVhostConf(sd.getContent());
-            } else {
-                throw new ValidationException("Not found upstream conf for server conf.");
+        for (ConfFile cf : completeEntry.getVhosts().getFiles()) {
+            try {
+                Long vsId = Long.parseLong(cf.getName());
+                if (vsIds.contains(vsId)) {
+                    entry.getVhosts().addConfFile(cf);
+                }
+            } catch (NumberFormatException ex) {
+                logger.error("Unable to extract vs id information from vhost file: " + cf.getName() + ".");
             }
         }
-        return dataMap;
-    }
+        for (ConfFile cf : completeEntry.getUpstreams().getFiles()) {
+            String[] fn = cf.getName().split("_");
+            boolean add = false;
+            for (String relatedVsId : fn) {
+                if (relatedVsId.isEmpty()) continue;
 
-    @Override
-    public Map<Long, VsConfData> getVsConfByVsIds(Long slbId, List<Long> vsIds, Long version) throws Exception {
-        List<NginxConfServerData> confServerDataList = getNginxConfServer(slbId, version);
-        List<NginxConfUpstreamData> confUpstreamDataList = getNginxConfUpstream(slbId, version);
-        if (confServerDataList.size() != confUpstreamDataList.size()) {
-            throw new ValidationException("Count of server conf and count of upstream conf is different.");
-        }
-        Map<Long, VsConfData> dataMap = new HashMap<>();
-        for (NginxConfUpstreamData upstream : confUpstreamDataList) {
-            if (vsIds.contains(upstream.getVsId())) {
-                VsConfData vsConfData = new VsConfData();
-                Long vsId = upstream.getVsId();
-                vsConfData.setUpstreamConf(upstream.getContent());
-                dataMap.put(vsId, vsConfData);
-            }
-        }
-        for (NginxConfServerData sd : confServerDataList) {
-            if (vsIds.contains(sd.getVsId())) {
-                if (dataMap.containsKey(sd.getVsId())) {
-                    dataMap.get(sd.getVsId()).setVhostConf(sd.getContent());
-                } else {
-                    throw new ValidationException("Not found upstream conf for server conf.");
+                Long vsId = 0L;
+                try {
+                    vsId = Long.parseLong(relatedVsId);
+                } catch (NumberFormatException ex) {
+                    logger.warn("Unable to extract vs id information from upstream file: " + cf.getName() + ".");
+                    break;
+                }
+                if (vsIds.contains(vsId)) {
+                    if (!add) add = true;
                 }
             }
+            if (add) {
+                entry.getUpstreams().addConfFile(cf);
+            }
         }
-        return dataMap;
+        return entry;
     }
 
     @Override

src/main/java/com/ctrip/zeus/service/nginx/NginxService.java

@@ -3,14 +3,10 @@ package com.ctrip.zeus.service.nginx;
 import com.ctrip.zeus.model.entity.DyUpstreamOpsData;
 import com.ctrip.zeus.model.entity.Slb;
 import com.ctrip.zeus.model.entity.SlbServer;
-import com.ctrip.zeus.nginx.entity.NginxResponse;
-import com.ctrip.zeus.nginx.entity.NginxServerStatus;
-import com.ctrip.zeus.nginx.entity.ReqStatus;
-import com.ctrip.zeus.nginx.entity.VsConfData;
+import com.ctrip.zeus.nginx.entity.*;
 
 import java.util.Date;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -23,7 +19,7 @@ public interface NginxService {
      * update nginx configs
      *
      * @param nginxConf
-     * @param vsConfDataMap
+     * @param entry
      * @param cleanVsIds
      * @param dyups
      * @param needReload
@@ -32,7 +28,8 @@ public interface NginxService {
      * @throws Exception
      */
     List<NginxResponse> update(String nginxConf,
-                               Map<Long, VsConfData> vsConfDataMap,
+                               NginxConfEntry entry,
+                               Set<Long> updateVsIds,
                                Set<Long> cleanVsIds,
                                DyUpstreamOpsData[] dyups,
                                boolean needReload,
@@ -43,11 +40,11 @@ public interface NginxService {
      * refresh nginx configs
      *
      * @param nginxConf
-     * @param vsConfDataMap
+     * @param entry
      * @return List of nginx Response
      * @throws Exception
      */
-    NginxResponse refresh(String nginxConf, Map<Long, VsConfData> vsConfDataMap, boolean reload) throws Exception;
+    NginxResponse refresh(String nginxConf, NginxConfEntry entry, boolean reload) throws Exception;
 
     /**
      * push config to slb servers

src/main/java/com/ctrip/zeus/service/nginx/handler/NginxConfOpsService.java

@@ -1,10 +1,8 @@
 package com.ctrip.zeus.service.nginx.handler;
 
-import com.ctrip.zeus.nginx.entity.NginxResponse;
-import com.ctrip.zeus.nginx.entity.VsConfData;
+import com.ctrip.zeus.nginx.entity.NginxConfEntry;
+import com.ctrip.zeus.service.nginx.impl.FileOpRecord;
 
-import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -19,7 +17,7 @@ public interface NginxConfOpsService {
      * @return Long
      * @throws Exception
      */
-    Long updateAll(Map<Long, VsConfData> vsConfs) throws Exception;
+    Long updateAll(NginxConfEntry entry) throws Exception;
 
     /**
      * undo write all to dist ;
@@ -55,7 +53,7 @@ public interface NginxConfOpsService {
      * @return Long time flag
      * @throws Exception
      */
-    Long cleanAndUpdateConf(Set<Long> cleanVsIds, Map<Long, VsConfData> vsConfs) throws Exception;
+    FileOpRecord cleanAndUpdateConf(Set<Long> cleanVsIds, Set<Long> updateVsIds, NginxConfEntry entry) throws Exception;
 
     /**
      * undo clean and update nginx conf  ;
@@ -64,5 +62,5 @@ public interface NginxConfOpsService {
      * @return Long time flag
      * @throws Exception
      */
-    void undoCleanAndUpdateConf(Set<Long> cleanVsIds, Map<Long, VsConfData> vsConfs, Long timeFlag) throws Exception;
+    void undoCleanAndUpdateConf(Set<Long> cleanVsIds, NginxConfEntry entry, FileOpRecord record) throws Exception;
 }

src/main/java/com/ctrip/zeus/service/nginx/handler/impl/NginxConfOpsServiceImpl.java

@@ -1,9 +1,12 @@
 package com.ctrip.zeus.service.nginx.handler.impl;
 
 import com.ctrip.zeus.exceptions.NginxProcessingException;
+import com.ctrip.zeus.nginx.entity.ConfFile;
+import com.ctrip.zeus.nginx.entity.NginxConfEntry;
 import com.ctrip.zeus.nginx.entity.NginxResponse;
-import com.ctrip.zeus.nginx.entity.VsConfData;
 import com.ctrip.zeus.service.nginx.handler.NginxConfOpsService;
+import com.ctrip.zeus.service.nginx.impl.FileOpRecord;
+import com.google.common.base.Joiner;
 import org.apache.commons.exec.CommandLine;
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.exec.PumpStreamHandler;
@@ -30,65 +33,71 @@ public class NginxConfOpsServiceImpl implements NginxConfOpsService {
 
 
     @Override
-    public Long updateAll(Map<Long, VsConfData> vsConfs) throws Exception {
+    public Long updateAll(NginxConfEntry entry) throws Exception {
         String vhostDir = DEFAULT_NGINX_CONF_DIR + File.separator + "vhosts";
         String upstreamDir = DEFAULT_NGINX_CONF_DIR + File.separator + "upstreams";
         File vhostFile = new File(vhostDir);
-        String[] vhostFileNames = vhostFile.list();
         File upstreamFile = new File(upstreamDir);
-        String[] upstreamFileNames = upstreamFile.list();
-        Date now = new Date();
 
+        Date now = new Date();
         backupAll(now, BACKUP_TYPE_REFRESH);
 
-        List<String> cleanFileNames = new ArrayList<>();
-        List<String> deleteFileNames = new ArrayList<>();
-        for (String name : vhostFileNames) {
+        LOGGER.info("[NginxConfUpdateAll] Cleanse and backup files under dir upstreams and vhosts on disk.");
+        List<String> cleansingFilenames = new ArrayList<>();
+        List<String> trashFilenames = new ArrayList<>();
+        for (String name : vhostFile.list()) {
             if (name.endsWith(CONF_SUFFIX)) {
-                cleanFileNames.add(vhostDir + File.separator + name);
+                cleansingFilenames.add(vhostDir + File.separator + name);
             } else {
-                deleteFileNames.add(vhostDir + File.separator + name);
+                trashFilenames.add(vhostDir + File.separator + name);
             }
         }
-        for (String name : upstreamFileNames) {
+        for (String name : upstreamFile.list()) {
             if (name.endsWith(CONF_SUFFIX)) {
-                cleanFileNames.add(upstreamDir + File.separator + name);
+                cleansingFilenames.add(upstreamDir + File.separator + name);
             } else {
-                deleteFileNames.add(upstreamDir + File.separator + name);
+                trashFilenames.add(upstreamDir + File.separator + name);
             }
         }
         try {
-            cleanFile(cleanFileNames, now);
-            deleteFile(deleteFileNames);
+            cleanFile(cleansingFilenames, now);
+            deleteFile(trashFilenames);
         } catch (Exception e) {
-            undoCleanFile(cleanFileNames, now);
+            undoCleanFile(cleansingFilenames, now);
             throw e;
         }
 
-        List<Long> sucIds = new ArrayList<>();
+        LOGGER.info("[NginxConfUpdateAll] Start to overwrite complete confs under dir upstreams and vhosts.");
+        List<String> writingVhostFiles = new ArrayList<>();
+        List<String> writingUpstreamFiles = new ArrayList<>();
         try {
-            for (Long vsId : vsConfs.keySet()) {
-                writeFile(vhostDir, vsId + CONF_SUFFIX, vsConfs.get(vsId).getVhostConf());
-                writeFile(upstreamDir, vsId + CONF_SUFFIX, vsConfs.get(vsId).getUpstreamConf());
-                sucIds.add(vsId);
+            for (ConfFile cf : entry.getVhosts().getFiles()) {
+                writeFile(vhostDir, cf.getName() + CONF_SUFFIX, cf.getContent());
+                writingVhostFiles.add(vhostDir + File.separator + cf.getName() + CONF_SUFFIX);
+            }
+            for (ConfFile cf : entry.getUpstreams().getFiles()) {
+                writeFile(upstreamDir, cf.getName() + CONF_SUFFIX, cf.getContent());
+                writingUpstreamFiles.add(upstreamDir + File.separator + cf.getName() + CONF_SUFFIX);
             }
         } catch (Exception e) {
-            List<String> deleteFile = new ArrayList<>();
-            for (Long vsId : vsConfs.keySet()) {
-                deleteFile.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
-                deleteFile.add(upstreamDir + File.separator + vsId + CONF_SUFFIX);
-            }
+            LOGGER.error("Updating upstream/vhost conf failed. Proceeding files stops at \n"
+                    + Joiner.on(",").join(writingVhostFiles) + "\n"
+                    + Joiner.on(",").join(writingUpstreamFiles) + ".", e);
             try {
-                deleteFile(deleteFile);
-                undoCleanFile(cleanFileNames, now);
+                deleteFile(writingVhostFiles);
+                deleteFile(writingUpstreamFiles);
+                undoCleanFile(cleansingFilenames, now);
             } catch (Exception e1) {
-                LOGGER.error("[NginxConfOpsUpdateAll]delete file or undo clean file failed. Nginx Config on disk is uncontrollable. Need refresh config files.");
-                throw new NginxProcessingException("[NginxConfOpsUpdateAll]delete file or undo clean file failed.", e1);
+                String err = "[NginxConfUpdateAll] Delete or revert conf files failed. Broken confs on disk. Refreshing is required.";
+                LOGGER.error(err);
+                throw new NginxProcessingException(err, e1);
             }
-            LOGGER.error("Vs config updated. But some file write failed. Success updated vsIds: " + sucIds.toString(), e);
             throw e;
         }
-        LOGGER.info("Vs config updated. vsId: " + sucIds.toString());
+
+        LOGGER.info("[NginxConfUpdateAll] Successfuly updated conf files: \n"
+                + Joiner.on(",").join(writingVhostFiles) + "\n"
+                + Joiner.on(",").join(writingUpstreamFiles) + ".");
         return now.getTime();
     }
 
@@ -123,8 +132,8 @@ public class NginxConfOpsServiceImpl implements NginxConfOpsService {
             deleteFile(deleteFileNames);
             undoCleanFile(undoFileNames, time);
         } catch (Exception e) {
-            LOGGER.error("[UndoUpdateAll]Failed to delete and undo file.", e);
-            throw new NginxProcessingException("[UndoUpdateAll]Failed to delete and undo file.", e);
+            LOGGER.error("[UndoUpdateAll] Failed to delete and revert files on disk.", e);
+            throw new NginxProcessingException("[UndoUpdateAll] Failed to delete and revert files on disk.", e);
         }
     }
 
@@ -134,7 +143,7 @@ public class NginxConfOpsServiceImpl implements NginxConfOpsService {
             file = new File(name);
             if (file.exists()) {
                 if (!file.delete()) {
-                    LOGGER.error("[DeleteConfFile] Delete conf file failed. File name:" + name);
+                    LOGGER.error("[DeleteConfFile] Delete conf file failed. Filename:" + name);
                 }
             }
         }
@@ -142,7 +151,6 @@ public class NginxConfOpsServiceImpl implements NginxConfOpsService {
 
     @Override
     public void updateNginxConf(String nginxConf) throws Exception {
-
         String fileName = DEFAULT_NGINX_CONF_DIR + File.separator + NGINX_CONF_FILE;
         List<String> files = new ArrayList<>();
         files.add(fileName);
@@ -178,98 +186,151 @@ public class NginxConfOpsServiceImpl implements NginxConfOpsService {
     }
 
     @Override
-    public Long cleanAndUpdateConf(Set<Long> cleanVsIds, Map<Long, VsConfData> vsConfs) throws Exception {
-        List<String> cleanFileNames = new ArrayList<>();
-        Date now = new Date();
+    public FileOpRecord cleanAndUpdateConf(Set<Long> cleanVsIds, Set<Long> updateVsIds, NginxConfEntry entry) throws Exception {
+        FileOpRecord record = new FileOpRecord();
         String vhostDir = DEFAULT_NGINX_CONF_DIR + File.separator + "vhosts";
         String upstreamDir = DEFAULT_NGINX_CONF_DIR + File.separator + "upstreams";
-        List<String> deleteFileNames = new ArrayList<>();
+
         File vhostFile = new File(vhostDir);
         String[] vhostFileNames = vhostFile.list();
         File upstreamFile = new File(upstreamDir);
         String[] upstreamFileNames = upstreamFile.list();
+
+        LOGGER.info("[NginxConfPartialUpdate] Cleanse trash files under dir upstreams and vhosts.");
+        List<String> trashFilenames = new ArrayList<>();
         for (String name : vhostFileNames) {
             if (!name.endsWith(CONF_SUFFIX)) {
-                deleteFileNames.add(vhostDir + File.separator + name);
+                trashFilenames.add(vhostDir + File.separator + name);
             }
         }
         for (String name : upstreamFileNames) {
             if (!name.endsWith(CONF_SUFFIX)) {
-                deleteFileNames.add(upstreamDir + File.separator + name);
+                trashFilenames.add(upstreamDir + File.separator + name);
             }
         }
         try {
-            deleteFile(deleteFileNames);
+            deleteFile(trashFilenames);
         } catch (Exception e) {
-            LOGGER.error("[CleanAndUpdateConf]Delete backup files failed.", e);
+            LOGGER.error("[NginxConfPartialUpdate] Cleansing trash files failed.", e);
         }
-        for (Long vsId : cleanVsIds) {
-            cleanFileNames.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
-            cleanFileNames.add(upstreamDir + File.separator + vsId + CONF_SUFFIX);
-        }
-        try {
-            cleanFile(cleanFileNames, now);
-        } catch (Exception e) {
-            undoCleanFile(cleanFileNames, now);
-            throw e;
-        }
-        List<Long> sucIds = new ArrayList<>();
-        List<String> copyFiles = new ArrayList<>();
-        try {
-            for (Long vsId : vsConfs.keySet()) {
-                copyFiles.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
-                copyFiles.add(upstreamDir + File.separator + vsId + CONF_SUFFIX);
-            }
-            copyFile(copyFiles, now);
 
-            for (Long vsId : vsConfs.keySet()) {
-                writeFile(vhostDir, vsId + CONF_SUFFIX, vsConfs.get(vsId).getVhostConf());
-                writeFile(upstreamDir, vsId + CONF_SUFFIX, vsConfs.get(vsId).getUpstreamConf());
-                sucIds.add(vsId);
+        // mark files to be cleansed(.clean/.bak)
+        // extract filename if contains related vs ids
+        Set<String> nextRelatedUpstreams = new HashSet<>();
+        for (ConfFile cf : entry.getUpstreams().getFiles()) {
+            nextRelatedUpstreams.add(cf.getName());
+        }
+        Set<String> currentRelatedUpstreams = new HashSet<>();
+        for (String upfn : upstreamFile.list()) {
+            upfn = upfn.substring(0, upfn.indexOf(CONF_SUFFIX));
+            String[] fn = upfn.split("_");
+            boolean add = false;
+            for (String relatedVsId : fn) {
+                if (relatedVsId.isEmpty()) continue;
+
+                Long vsId = 0L;
+                try {
+                    vsId = Long.parseLong(relatedVsId);
+                } catch (NumberFormatException ex) {
+                    LOGGER.warn("[NginxConfPartialUpdate] Unable to extract vs id information from upstream file: " + upfn + ".");
+                    break;
+                }
+                if (updateVsIds.contains(vsId) || cleanVsIds.contains(vsId)) {
+                    if (!add) add = true;
+                }
+            }
+            if (add) {
+                currentRelatedUpstreams.add(upfn);
+            }
+        }
+
+        LOGGER.info("[NginxConfPartialUpdate] Backup related files under dir upstreams and vhosts on disk.");
+        Set<String> cleansingUpstreams = new HashSet<>(currentRelatedUpstreams);
+        cleansingUpstreams.removeAll(nextRelatedUpstreams);
+
+        Set<String> backupUpstreams = new HashSet<>(currentRelatedUpstreams);
+        backupUpstreams.retainAll(nextRelatedUpstreams);
+
+        List<String> cleansingFilenames = new ArrayList<>();
+        List<String> backingupFilenames = new ArrayList<>();
+        for (Long vsId : cleanVsIds) {
+            cleansingFilenames.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
+        }
+        for (String fn : cleansingUpstreams) {
+            cleansingFilenames.add(upstreamDir + File.separator + fn + CONF_SUFFIX);
+        }
+
+        try {
+            cleanFile(cleansingFilenames, record.getTimeStamp());
+        } catch (Exception e) {
+            undoCleanFile(cleansingFilenames, record.getTimeStamp());
+            throw e;
+        }
+
+        for (ConfFile cf : entry.getVhosts().getFiles()) {
+            backingupFilenames.add(vhostDir + File.separator + cf.getName() + CONF_SUFFIX);
+        }
+        for (String fn : backupUpstreams) {
+            backingupFilenames.add(upstreamDir + File.separator + fn + CONF_SUFFIX);
+        }
+
+        try {
+            copyFile(backingupFilenames, record.getTimeStamp());
+        } catch (Exception e) {
+            undoCopyFile(backingupFilenames, record.getTimeStamp());
+            throw e;
+        }
+
+        LOGGER.info("[NginxConfPartialUpdate] Start to write confs of related vses under dir upstreams and vhosts.");
+        List<String> writingVhostFiles = new ArrayList<>();
+        List<String> writingUpstreamFiles = new ArrayList<>();
+        try {
+            for (ConfFile cf : entry.getVhosts().getFiles()) {
+                writeFile(vhostDir, cf.getName() + CONF_SUFFIX, cf.getContent());
+                writingVhostFiles.add(vhostDir + File.separator + cf.getName() + CONF_SUFFIX);
+            }
+            for (ConfFile cf : entry.getUpstreams().getFiles()) {
+                writeFile(upstreamDir, cf.getName() + CONF_SUFFIX, cf.getContent());
+                writingUpstreamFiles.add(upstreamDir + File.separator + cf.getName() + CONF_SUFFIX);
             }
         } catch (Exception e) {
-            LOGGER.error("[CleanAndUpdateConf]Vs config updated. But some file write failed.Going to undo ops. Suc Ids:" + sucIds.toString(), e);
+            LOGGER.error("Updating upstream/vhost conf failed. Proceeding files stops at \n"
+                    + Joiner.on(",").join(writingVhostFiles) + "\n"
+                    + Joiner.on(",").join(writingUpstreamFiles) + ".", e);
             try {
-                undoCleanFile(cleanFileNames, now);
-                undoCopyFile(copyFiles, now);
+                deleteFile(writingVhostFiles);
+                deleteFile(writingUpstreamFiles);
+                undoCleanFile(cleansingFilenames, record.getTimeStamp());
+                undoCopyFile(backingupFilenames, record.getTimeStamp());
             } catch (Exception e1) {
-                LOGGER.error("[CleanAndUpdateConf] undo clean file or undo copy file failed. config is Uncontrollable. Need refresh configs.Date:" + now.toString());
-                throw new NginxProcessingException("[CleanAndUpdateConf] undo clean file or undo copy file failed. config is Uncontrollable. Need refresh configs.Date:" + now.toString(), e);
+                String err = "[NginxConfPartialUpdate] Delete or revert conf files failed. Broken confs on disk. Refreshing is required. Maker - " + record.getTimeStamp().toString();
+                LOGGER.error(err);
+                throw new NginxProcessingException(err, e1);
             }
             throw e;
         }
-        LOGGER.info("[CleanAndUpdateConf]Vs config updated. vsId: " + sucIds.toString());
-        return now.getTime();
+        LOGGER.info("Successfuly updated conf files: \n"
+                + Joiner.on(",").join(writingVhostFiles) + "\n"
+                + Joiner.on(",").join(writingUpstreamFiles) + ".");
+
+        record.setCleansedFilename(cleansingFilenames);
+        record.setCopiedFilename(backingupFilenames);
+        return record;
     }
 
     @Override
-    public void undoCleanAndUpdateConf(Set<Long> cleanVsIds, Map<Long, VsConfData> vsConfs, Long timeFlag) throws Exception {
-        Date flag = new Date(timeFlag);
+    public void undoCleanAndUpdateConf(Set<Long> cleanVsIds, NginxConfEntry entry, FileOpRecord record) throws Exception {
         try {
-            LOGGER.info("[UndoCleanAndUpdateConf] Start undo clean and update conf.Time flag:" + flag.toString());
-            List<String> copyFiles = new ArrayList<>();
-            String vhostDir = DEFAULT_NGINX_CONF_DIR + File.separator + "vhosts";
-            String upstreamDir = DEFAULT_NGINX_CONF_DIR + File.separator + "upstreams";
-            List<String> cleanFileNames = new ArrayList<>();
-
-            backupAll(flag, BACKUP_TYPE_ROLLBACK);
-
-            for (Long vsId : vsConfs.keySet()) {
-                copyFiles.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
-                copyFiles.add(upstreamDir + File.separator + vsId + CONF_SUFFIX);
-            }
-            undoCopyFile(copyFiles, flag);
-
-            for (Long vsId : cleanVsIds) {
-                cleanFileNames.add(vhostDir + File.separator + vsId + CONF_SUFFIX);
-                cleanFileNames.add(upstreamDir + File.separator + vsId + CONF_SUFFIX);
-            }
-            undoCleanFile(cleanFileNames, flag);
+            LOGGER.info("[UndoNginxConfPartialUpdate] Start to revert related conf files to the previous version. Maker - " + record.getTimeStamp().toString());
+            backupAll(record.getTimeStamp(), BACKUP_TYPE_ROLLBACK);
+            undoCopyFile(record.getCopiedFilename(), record.getTimeStamp());
+            undoCleanFile(record.getCleansedFilename(), record.getTimeStamp());
         } catch (Exception e) {
-            LOGGER.info("[UndoCleanAndUpdateConf] Failed to undo clean and update conf.Time flag:" + flag.toString(), e);
-            throw new NginxProcessingException("[UndoCleanAndUpdateConf] Failed to undo clean and update conf.Time flag:" + flag.toString(), e);
+            String err = "[UndoNginxConfPartialUpdate] Failed to revert related conf files to the previous version. Maker - " + record.getTimeStamp().toString();
+            LOGGER.info(err, e);
+            throw new NginxProcessingException(err, e);
         } finally {
-            LOGGER.info("[UndoCleanAndUpdateConf] End undo clean and update conf.Time flag:" + flag.toString());
+            LOGGER.info("[UndoNginxConfPartialUpdate] Successfully reverted related conf files to the previous version.");
         }
 
     }

src/main/java/com/ctrip/zeus/service/nginx/impl/FileOpRecord.java

@@ -0,0 +1,37 @@
+package com.ctrip.zeus.service.nginx.impl;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/4/21.
+ */
+public class FileOpRecord {
+    private Date timeStamp;
+    private List<String> cleansedFilename;
+    private List<String> copiedFilename;
+
+    public FileOpRecord() {
+        this.timeStamp = new Date();
+    }
+
+    public Date getTimeStamp() {
+        return timeStamp;
+    }
+
+    public List<String> getCleansedFilename() {
+        return cleansedFilename;
+    }
+
+    public void setCleansedFilename(List<String> cleansedFilename) {
+        this.cleansedFilename = cleansedFilename;
+    }
+
+    public List<String> getCopiedFilename() {
+        return copiedFilename;
+    }
+
+    public void setCopiedFilename(List<String> copiedFilename) {
+        this.copiedFilename = copiedFilename;
+    }
+}

src/main/java/com/ctrip/zeus/service/nginx/impl/NginxServiceImpl.java

@@ -57,14 +57,13 @@ public class NginxServiceImpl implements NginxService {
 
     private final Logger logger = LoggerFactory.getLogger(NginxServiceImpl.class);
 
-
     @Override
-    public List<NginxResponse> update(String nginxConf, Map<Long, VsConfData> vsConfDataMap, Set<Long> cleanVsIds, DyUpstreamOpsData[] dyups, boolean needReload, boolean needTest, boolean needDyups) throws Exception {
+    public List<NginxResponse> update(String nginxConf, NginxConfEntry entry, Set<Long> updateVsIds, Set<Long> cleanVsIds, DyUpstreamOpsData[] dyups, boolean needReload, boolean needTest, boolean needDyups) throws Exception {
         List<NginxResponse> response = new ArrayList<>();
         try {
             //1. update config
             nginxConfOpsService.updateNginxConf(nginxConf);
-            Long cleanConfFlag = nginxConfOpsService.cleanAndUpdateConf(cleanVsIds, vsConfDataMap);
+            FileOpRecord fileOpRecord = nginxConfOpsService.cleanAndUpdateConf(cleanVsIds, updateVsIds, entry);
             //2. test reload or dyups, if needed. undo config if failed.
             if (needTest) {
                 NginxResponse res = nginxOpsService.test();
@@ -72,7 +71,7 @@ public class NginxServiceImpl implements NginxService {
                 if (!res.getSucceed()) {
                     logger.error("[NginxService]update failed.Test conf failed.");
                     nginxConfOpsService.undoUpdateNginxConf();
-                    nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, vsConfDataMap, cleanConfFlag);
+                    nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, entry, fileOpRecord);
                 }
             }
             if (needReload) {
@@ -81,7 +80,7 @@ public class NginxServiceImpl implements NginxService {
                 if (!res.getSucceed()) {
                     logger.error("[NginxService]update failed.Reload conf failed.");
                     nginxConfOpsService.undoUpdateNginxConf();
-                    nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, vsConfDataMap, cleanConfFlag);
+                    nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, entry, fileOpRecord);
                 }
             }
             if (needDyups) {
@@ -91,7 +90,7 @@ public class NginxServiceImpl implements NginxService {
                     if (!res.getSucceed()) {
                         logger.error("[NginxService]update failed.Dyups conf failed.");
                         nginxConfOpsService.undoUpdateNginxConf();
-                        nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, vsConfDataMap, cleanConfFlag);
+                        nginxConfOpsService.undoCleanAndUpdateConf(cleanVsIds, entry, fileOpRecord);
                         break;
                     }
                 }
@@ -103,9 +102,9 @@ public class NginxServiceImpl implements NginxService {
     }
 
     @Override
-    public NginxResponse refresh(String nginxConf, Map<Long, VsConfData> vsConfDataMap, boolean reload) throws Exception {
+    public NginxResponse refresh(String nginxConf, NginxConfEntry entry, boolean reload) throws Exception {
         nginxConfOpsService.updateNginxConf(nginxConf);
-        Long updateAllFlag = nginxConfOpsService.updateAll(vsConfDataMap);
+        Long updateAllFlag = nginxConfOpsService.updateAll(entry);
         if (reload) {
             NginxResponse res = nginxOpsService.reload();
             if (!res.getSucceed()) {

src/main/java/com/ctrip/zeus/service/nginx/util/UpstreamConfPicker.java

@@ -2,6 +2,8 @@ package com.ctrip.zeus.service.nginx.util;
 
 import com.ctrip.zeus.exceptions.NginxProcessingException;
 import com.ctrip.zeus.model.entity.DyUpstreamOpsData;
+import com.ctrip.zeus.nginx.entity.ConfFile;
+import com.ctrip.zeus.nginx.entity.NginxConfEntry;
 import com.ctrip.zeus.nginx.entity.VsConfData;
 import com.google.common.base.Joiner;
 import org.slf4j.Logger;
@@ -19,11 +21,11 @@ public class UpstreamConfPicker {
 
     private static final String Identifier = "upstream";
 
-    public DyUpstreamOpsData[] pickByGroupIds(Map<Long, VsConfData> vsConf, final Set<Long> groupIds) throws NginxProcessingException {
+    public DyUpstreamOpsData[] pickByGroupIds(NginxConfEntry entry, final Set<Long> groupIds) throws NginxProcessingException {
         final List<DyUpstreamOpsData> result = new ArrayList<>();
         final Set<Long> selected = new HashSet<>();
-        for (VsConfData data : vsConf.values()) {
-            parse(data.getUpstreamConf(), new UpstreamDirectiveDelegate() {
+        for (ConfFile cf : entry.getUpstreams().getFiles()) {
+            parse(cf.getContent(), new UpstreamDirectiveDelegate() {
                 @Override
                 public void delegate(String upstreamName, String content) {
                     try {

src/main/java/com/ctrip/zeus/service/update/impl/SlbServerConfManagerImpl.java

@@ -2,9 +2,10 @@ package com.ctrip.zeus.service.update.impl;
 
 import com.ctrip.zeus.commit.entity.Commit;
 import com.ctrip.zeus.exceptions.NginxProcessingException;
+import com.ctrip.zeus.exceptions.ValidationException;
 import com.ctrip.zeus.model.entity.DyUpstreamOpsData;
+import com.ctrip.zeus.nginx.entity.NginxConfEntry;
 import com.ctrip.zeus.nginx.entity.NginxResponse;
-import com.ctrip.zeus.nginx.entity.VsConfData;
 import com.ctrip.zeus.service.build.NginxConfService;
 import com.ctrip.zeus.service.commit.CommitMergeService;
 import com.ctrip.zeus.service.commit.CommitService;
@@ -25,7 +26,6 @@ import org.springframework.stereotype.Service;
 import javax.annotation.Resource;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -82,10 +82,19 @@ public class SlbServerConfManagerImpl implements SlbServerConfManager {
                     needReload = true;
                 }
                 //3.2 get nginx config
-                String nginxConf = nginxConfService.getNginxConf(slbId, slbVersion);
-                Map<Long, VsConfData> vsConfDataMap = nginxConfService.getVsConfBySlbId(slbId, slbVersion);
+                String nginxConf = null;
+                try {
+                    nginxConf = nginxConfService.getNginxConf(slbId, slbVersion);
+                } catch (ValidationException ex) {
+                }
+                NginxConfEntry entry = nginxConfService.getUpstreamsAndVhosts(slbId, slbVersion);
+                if (nginxConf == null || entry == null) {
+                    String err = "Nginx conf file records are missing of slb-version " + slbId +  "-" + slbVersion + ".";
+                    logger.error(err);
+                    throw new ValidationException(err);
+                }
                 //3.3 refresh config
-                NginxResponse res = nginxService.refresh(nginxConf, vsConfDataMap, needReload);
+                NginxResponse res = nginxService.refresh(nginxConf, entry, needReload);
                 //3.3.1 if failed, set need refresh flag and return result.
                 if (!res.getSucceed()) {
                     logger.error("[SlbServerUpdate] Refresh conf failed. SlbId:" + slbId + ";Version:" + slbVersion + response.toString());
@@ -105,9 +114,18 @@ public class SlbServerConfManagerImpl implements SlbServerConfManager {
                     logger.info("[SlbServerUpdate] Not found commit. return success instead.");
                     return new NginxResponse().setSucceed(true).setServerIp(ip);
                 }
-                //4.2 get all configs
-                String nginxConf = nginxConfService.getNginxConf(slbId, slbVersion);
-                Map<Long, VsConfData> dataMap = nginxConfService.getVsConfByVsIds(slbId, commit.getVsIds(), slbVersion);
+                //4.2 get all configs by vs ids in commit
+                String nginxConf = null;
+                try {
+                    nginxConf = nginxConfService.getNginxConf(slbId, slbVersion);
+                } catch (ValidationException ex) {
+                }
+                NginxConfEntry entry = nginxConfService.getUpstreamsAndVhosts(slbId, slbVersion, commit.getVsIds());
+                if (nginxConf == null || entry == null) {
+                    String err = "Nginx conf file records are missing of slb-version " + slbId +  "-" + slbVersion + ".";
+                    logger.error(err);
+                    throw new ValidationException(err);
+                }
                 //4.3 get clean vs ids
                 Set<Long> cleanSet = new HashSet<>();
                 cleanSet.addAll(commit.getCleanvsIds());
@@ -127,10 +145,10 @@ public class SlbServerConfManagerImpl implements SlbServerConfManager {
                 if (dyups) {
                     Set<Long> gids = new HashSet<>();
                     gids.addAll(commit.getGroupIds());
-                    dyUpstreamOpsDatas = upstreamConfPicker.pickByGroupIds(dataMap, gids);
+                    dyUpstreamOpsDatas = upstreamConfPicker.pickByGroupIds(entry, gids);
                 }
                 //4.5 update nginx configs
-                List<NginxResponse> responses = nginxService.update(nginxConf, dataMap, cleanSet, dyUpstreamOpsDatas, reload, test, dyups);
+                List<NginxResponse> responses = nginxService.update(nginxConf, entry, new HashSet<>(commit.getVsIds()), cleanSet, dyUpstreamOpsDatas, reload, test, dyups);
                 //4.6 check response, if failed. set need refresh flag.
                 for (NginxResponse r : responses) {
                     if (!r.getSucceed()) {

src/test/java/com/ctrip/zeus/service/nginx/util/UpstreamConfPickerTest.java

@@ -2,7 +2,7 @@ package com.ctrip.zeus.service.nginx.util;
 
 import com.ctrip.zeus.exceptions.NginxProcessingException;
 import com.ctrip.zeus.model.entity.DyUpstreamOpsData;
-import com.ctrip.zeus.nginx.entity.VsConfData;
+import com.ctrip.zeus.nginx.entity.*;
 import com.ctrip.zeus.util.IOUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -27,13 +27,13 @@ public class UpstreamConfPickerTest {
         String conf_1109 = "";
 
         InputStream confFile = this.getClass().getClassLoader().getResourceAsStream("com.ctrip.zeus.service/782.conf");
-        Map<Long, VsConfData> vsConf = new HashMap<>();
-        vsConf.put(782L, new VsConfData().setUpstreamConf(IOUtils.inputStreamStringify(confFile)));
+        NginxConfEntry entry = new NginxConfEntry().setVhosts(new Vhosts()).setUpstreams(new Upstreams());
+        entry.getUpstreams().addConfFile(new ConfFile().setName("782").setContent(IOUtils.inputStreamStringify(confFile)));
         Set<Long> groupIds = new HashSet<>();
         groupIds.add(893L);
         groupIds.add(1109L);
 
-        DyUpstreamOpsData[] result = upstreamConfPicker.pickByGroupIds(vsConf, groupIds);
+        DyUpstreamOpsData[] result = upstreamConfPicker.pickByGroupIds(entry, groupIds);
         Assert.assertEquals(2, result.length);
         for (int i = 0; i < result.length; i++) {
             DyUpstreamOpsData d = result[i];
@@ -50,8 +50,8 @@ public class UpstreamConfPickerTest {
     @Test(expected = NginxProcessingException.class)
     public void testNginxProcessingException() throws IOException, NginxProcessingException {
         InputStream confFile = this.getClass().getClassLoader().getResourceAsStream("com.ctrip.zeus.service/782.conf");
-        Map<Long, VsConfData> vsConf = new HashMap<>();
-        vsConf.put(782L, new VsConfData().setUpstreamConf(IOUtils.inputStreamStringify(confFile)));
+        NginxConfEntry entry = new NginxConfEntry().setVhosts(new Vhosts()).setUpstreams(new Upstreams());
+        entry.getUpstreams().addConfFile(new ConfFile().setName("782").setContent(IOUtils.inputStreamStringify(confFile)));
         Set<Long> groupIds = new HashSet<>();
         groupIds.add(893L);
         groupIds.add(1109L);
@@ -59,7 +59,7 @@ public class UpstreamConfPickerTest {
         groupIds.add(1077L);
 
         try {
-            upstreamConfPicker.pickByGroupIds(vsConf, groupIds);
+            upstreamConfPicker.pickByGroupIds(entry, groupIds);
         } catch (NginxProcessingException ex) {
             String message = ex.getMessage();
             Assert.assertTrue(message.contains("1076"));