mirror of
https://github.com/getrebuild/rebuild.git
synced 2025-02-24 22:35:04 +08:00
imports
This commit is contained in:
devezhao
parent
e0641f5b94
commit
82a7578de0
9 changed files with 427 additions and 6 deletions
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
rebuild - Building your system freely.
|
||||
Copyright (C) 2019 devezhao <zhaofang123@gmail.com>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.rebuild.server.business.dataio;
|
||||
|
||||
import java.io.File;
|
||||
import java.net.URL;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.rebuild.server.Application;
|
||||
import com.rebuild.server.RebuildException;
|
||||
import com.rebuild.server.helper.QiniuCloud;
|
||||
import com.rebuild.server.helper.SystemConfig;
|
||||
import com.rebuild.server.helper.task.BulkTask;
|
||||
import com.rebuild.server.metadata.EntityHelper;
|
||||
import com.rebuild.server.metadata.MetadataHelper;
|
||||
|
||||
import cn.devezhao.commons.CodecUtils;
|
||||
import cn.devezhao.persist4j.Record;
|
||||
import cn.devezhao.persist4j.engine.ID;
|
||||
|
||||
/**
|
||||
* 导入分类数据
|
||||
*
|
||||
* @author devezhao zhaofang123@gmail.com
|
||||
* @since 2019/04/08
|
||||
*/
|
||||
public class ClassificationImporter extends BulkTask {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(ClassificationImporter.class);
|
||||
|
||||
private ID user;
|
||||
private ID dest;
|
||||
private String fileUrl;
|
||||
|
||||
/**
|
||||
* @param user
|
||||
* @param dest
|
||||
* @param fileUrl
|
||||
*/
|
||||
public ClassificationImporter(ID user, ID dest, String fileUrl) {
|
||||
this.user = user;
|
||||
this.dest = dest;
|
||||
this.fileUrl = fileUrl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
JSONArray data = null;
|
||||
try {
|
||||
data = (JSONArray) fetchRemoteJson(fileUrl);
|
||||
} catch (RebuildException e) {
|
||||
this.completedAfter();
|
||||
return;
|
||||
}
|
||||
|
||||
String delSql = String.format("delete from `%s` where `DATA_ID` = '%s'",
|
||||
MetadataHelper.getEntity(EntityHelper.ClassificationData).getPhysicalName(), dest);
|
||||
Application.getSQLExecutor().execute(delSql);
|
||||
|
||||
this.setThreadUser(user);
|
||||
|
||||
this.setTotal(data.size());
|
||||
try {
|
||||
for (Object o : data) {
|
||||
addNOne((JSONObject) o, null);
|
||||
this.setCompleteOne();
|
||||
}
|
||||
} finally {
|
||||
this.completedAfter();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param node
|
||||
* @param parent
|
||||
*/
|
||||
private void addNOne(JSONObject node, ID parent) {
|
||||
String code = node.getString("code");
|
||||
String name = node.getString("name");
|
||||
|
||||
Record item = EntityHelper.forNew(EntityHelper.ClassificationData, Application.getCurrentUser());
|
||||
item.setString("name", name);
|
||||
if (StringUtils.isNotBlank(code)) {
|
||||
item.setString("code", code);
|
||||
}
|
||||
item.setID("dataId", this.dest);
|
||||
if (parent != null) {
|
||||
item.setID("parent", parent);
|
||||
}
|
||||
item = Application.getCommonService().create(item);
|
||||
|
||||
JSONArray children = node.getJSONArray("children");
|
||||
if (children != null) {
|
||||
for (Object o : children) {
|
||||
addNOne((JSONObject) o, item.getPrimary());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -- Helper
|
||||
|
||||
/**
|
||||
* 远程仓库
|
||||
*/
|
||||
public static final String DATA_REPO = "https://raw.githubusercontent.com/getrebuild/rebuild-datas/master/classifications/";
|
||||
|
||||
/**
|
||||
* 远程读取 JSON 文件
|
||||
*
|
||||
* @param fileUrl
|
||||
* @return
|
||||
* @throws RebuildException 如果读取失败
|
||||
*/
|
||||
public static JSON fetchRemoteJson(String fileUrl) throws RebuildException {
|
||||
if (!fileUrl.startsWith("http")) {
|
||||
fileUrl = DATA_REPO + fileUrl;
|
||||
}
|
||||
|
||||
File tmp = SystemConfig.getFileOfTemp("classifications.tmp." + CodecUtils.randomCode(6));
|
||||
JSON d = null;
|
||||
try {
|
||||
if (QiniuCloud.instance().download(new URL(fileUrl), tmp)) {
|
||||
String t2str = FileUtils.readFileToString(tmp, "utf-8");
|
||||
d = (JSON) JSON.parse(t2str);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.error("Fetch failure from URL : " + fileUrl, e);
|
||||
}
|
||||
|
||||
if (d == null) {
|
||||
throw new RebuildException("无法读取远程数据");
|
||||
}
|
||||
return d;
|
||||
}
|
||||
}
|
||||
|
|
@ -23,7 +23,12 @@ import java.util.Date;
|
|||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.rebuild.server.Application;
|
||||
import com.rebuild.server.service.bizz.CurrentCaller;
|
||||
import com.rebuild.web.OnlineSessionStore;
|
||||
|
||||
import cn.devezhao.commons.CalendarUtils;
|
||||
import cn.devezhao.persist4j.engine.ID;
|
||||
|
||||
/**
|
||||
* 耗时操作可通过此类进行，例如大批量删除/修改等。此类提供了进度相关的约定，如总计执行条目，已完成条目/百分比。
|
||||
|
|
@ -45,12 +50,27 @@ public abstract class BulkTask implements Runnable {
|
|||
private Date beginTime;
|
||||
private Date completedTime;
|
||||
|
||||
private ID userInThread;
|
||||
|
||||
/**
|
||||
*/
|
||||
protected BulkTask() {
|
||||
this.beginTime = CalendarUtils.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置线程用户
|
||||
*
|
||||
* @param user
|
||||
* @see CurrentCaller
|
||||
* @see OnlineSessionStore
|
||||
* @see #completedAfter()
|
||||
*/
|
||||
protected void setThreadUser(ID user) {
|
||||
this.userInThread = user;
|
||||
Application.getSessionStore().set(user);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param total
|
||||
*/
|
||||
|
|
@ -72,10 +92,13 @@ public abstract class BulkTask implements Runnable {
|
|||
}
|
||||
|
||||
/**
|
||||
* 子类应该在执行完毕后调用此方法
|
||||
* 子类应该在执行完毕后调用此方法。任何清空下，都应保证此方法被调用！
|
||||
*/
|
||||
protected void completedAfter() {
|
||||
this.completedTime = CalendarUtils.now();
|
||||
if (this.userInThread != null) {
|
||||
Application.getSessionStore().clean();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -163,12 +163,12 @@ public class ClassificationControll extends BasePageControll {
|
|||
Object[][] child = null;
|
||||
if (parent != null) {
|
||||
child = Application.createQuery(
|
||||
"select itemId,name,code from ClassificationData where parent = ? order by name")
|
||||
"select itemId,name,code from ClassificationData where parent = ? order by code,name")
|
||||
.setParameter(1, parent)
|
||||
.array();
|
||||
} else if (dataId != null) {
|
||||
child = Application.createQuery(
|
||||
"select itemId,name,code from ClassificationData where dataId = ? and parent is null order by name")
|
||||
"select itemId,name,code from ClassificationData where dataId = ? and parent is null order by code,name")
|
||||
.setParameter(1, dataId)
|
||||
.array();
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
rebuild - Building your system freely.
|
||||
Copyright (C) 2019 devezhao <zhaofang123@gmail.com>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.rebuild.web.admin.entityhub;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.rebuild.server.business.dataio.ClassificationImporter;
|
||||
import com.rebuild.web.BaseControll;
|
||||
|
||||
import cn.devezhao.persist4j.engine.ID;
|
||||
|
||||
/**
|
||||
* 分类数据导入
|
||||
*
|
||||
* @author devezhao zhaofang123@gmail.com
|
||||
* @since 2019/04/08
|
||||
*
|
||||
* @see ClassificationImporter
|
||||
*/
|
||||
@RequestMapping("/admin/classification/")
|
||||
@Controller
|
||||
public class ClassificationImportControll extends BaseControll {
|
||||
|
||||
@RequestMapping("/imports/load-index")
|
||||
public void loadDataIndex(HttpServletRequest request, HttpServletResponse response) throws IOException {
|
||||
JSON index = ClassificationImporter.fetchRemoteJson("index.json");
|
||||
if (index == null) {
|
||||
writeSuccess(response, "无法获取索引数据");
|
||||
} else {
|
||||
writeSuccess(response, index);
|
||||
}
|
||||
}
|
||||
|
||||
@RequestMapping("/imports/starts")
|
||||
public void starts(HttpServletRequest request, HttpServletResponse response) throws IOException {
|
||||
ID user = getRequestUser(request);
|
||||
ID dest = getIdParameterNotNull(request, "dest");
|
||||
String fileUrl = getParameterNotNull(request, "file");
|
||||
|
||||
ClassificationImporter importer = new ClassificationImporter(user, dest, fileUrl);
|
||||
importer.run(); // No thread
|
||||
writeSuccess(response);
|
||||
}
|
||||
}
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
</nav>
|
||||
</div>
|
||||
<div class="float-right" style="margin-top:-3px">
|
||||
<button class="btn btn-secondary" type="button"><i class="zmdi zmdi-cloud-download"></i> 公共数据</button>
|
||||
<button class="btn btn-secondary J_imports" type="button"><i class="zmdi zmdi-cloud-download"></i> 导入</button>
|
||||
</div>
|
||||
<div class="clearfix"></div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -105,4 +105,25 @@
|
|||
|
||||
.dd-list:empty {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Imports */
|
||||
.indexes>div {
|
||||
border: 2px solid #eee;
|
||||
margin: 15px;
|
||||
padding: 15px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.indexes>div h5 {
|
||||
font-size: 1rem;
|
||||
font-weight: bold;
|
||||
margin: 0;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.indexes>div .text-muted,
|
||||
.indexes>div .text-muted a {
|
||||
font-size: 12px;
|
||||
color: #777 !important;
|
||||
}
|
||||
|
|
@ -2,6 +2,9 @@
|
|||
/* eslint-disable no-undef */
|
||||
$(document).ready(function () {
|
||||
renderRbcomp(<LevelBoxes id={dataId} />, 'boxes')
|
||||
$('.J_imports').click(() => {
|
||||
renderRbcomp(<DlgImports id={dataId} />)
|
||||
})
|
||||
})
|
||||
|
||||
class LevelBoxes extends React.Component {
|
||||
|
|
@ -210,4 +213,60 @@ var saveOpenLevel = function () {
|
|||
saveOpenLevel_last = level
|
||||
})
|
||||
}, 500, 'saveOpenLevel')
|
||||
}
|
||||
|
||||
let import_mprogress
|
||||
class DlgImports extends RbModalHandler {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
}
|
||||
render() {
|
||||
return <RbModal title="导入公共数据" ref={(c) => this._dlg = c}>
|
||||
{this.state.indexes ? <div className="indexes">{this.state.indexes.map((item) => {
|
||||
return (<div key={'index' + item.file}>
|
||||
<div className="float-left">
|
||||
<h5>{item.name}</h5>
|
||||
<div className="text-muted">数据来源 <a target="_blank" rel="noopener noreferrer" href={item.source}>{item.source}</a></div>
|
||||
</div>
|
||||
<div className="float-right pt-1">
|
||||
<button className="btn btn-sm btn-primary" data-file={item.file} data-name={item.name} onClick={this.imports}>导入</button>
|
||||
</div>
|
||||
<div className="clearfix"></div>
|
||||
</div>)
|
||||
})}</div>
|
||||
: <RbSpinner fully={true} />}
|
||||
</RbModal>
|
||||
}
|
||||
componentDidMount() {
|
||||
$.get(`${rb.baseUrl}/admin/classification/imports/load-index`, (res) => {
|
||||
this.setState({
|
||||
indexes: res.data
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
imports = (e) => {
|
||||
let file = e.currentTarget.dataset.file
|
||||
let name = e.currentTarget.dataset.name
|
||||
let url = `${rb.baseUrl}/admin/classification/imports/starts?dest=${this.props.id}&file=${$encode(file)}`
|
||||
let that = this
|
||||
rb.alert(`<strong>${name}</strong><br>导入将导致现有数据被清空。立即开始导入吗？`, {
|
||||
html: true,
|
||||
confirm: function () {
|
||||
this.hide()
|
||||
that.hide()
|
||||
import_mprogress = new Mprogress({ template: 3 })
|
||||
import_mprogress.start()
|
||||
|
||||
$.post(url, (res) => {
|
||||
if (res.error_code === 0) rb.hbsuccess('导入完成')
|
||||
else rb.hbsuccess(res.error_msg || '导入失败')
|
||||
|
||||
import_mprogress.end()
|
||||
setTimeout(() => { location.reload() }, 1500)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -187,12 +187,14 @@ class RbHighbar extends React.Component {
|
|||
}
|
||||
|
||||
// ~~ 加载条
|
||||
function RbSpinner() {
|
||||
return <div className="rb-spinner">
|
||||
function RbSpinner(props) {
|
||||
let spinner = <div className="rb-spinner">
|
||||
<svg width="40px" height="40px" viewBox="0 0 66 66" xmlns="http://www.w3.org/2000/svg">
|
||||
<circle fill="none" strokeWidth="4" strokeLinecap="round" cx="33" cy="33" r="30" className="circle" />
|
||||
</svg>
|
||||
</div>
|
||||
if (props && props.fully === true) return <div className="rb-loading rb-loading-active">{spinner}</div>
|
||||
return spinner
|
||||
}
|
||||
|
||||
let renderRbcomp__counter = new Date().getTime()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
/*
|
||||
rebuild - Building your system freely.
|
||||
Copyright (C) 2019 devezhao <zhaofang123@gmail.com>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.rebuild.server.business.dataio;
|
||||
|
||||
import org.junit.FixMethodOrder;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.MethodSorters;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
|
||||
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
|
||||
|
||||
import com.rebuild.server.Application;
|
||||
import com.rebuild.server.metadata.EntityHelper;
|
||||
import com.rebuild.server.service.bizz.UserService;
|
||||
import com.rebuild.web.MvcTestSupport;
|
||||
|
||||
import cn.devezhao.commons.web.WebUtils;
|
||||
import cn.devezhao.persist4j.Record;
|
||||
import cn.devezhao.persist4j.engine.ID;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
*
|
||||
* @author devezhao zhaofang123@gmail.com
|
||||
* @since 2019/04/08
|
||||
*/
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
||||
public class ClassificationImporterTest extends MvcTestSupport {
|
||||
|
||||
@Test
|
||||
public void test0ListPage() throws Exception {
|
||||
MockHttpServletRequestBuilder builder = MockMvcRequestBuilders
|
||||
.get("/admin/classifications")
|
||||
.sessionAttr(WebUtils.KEY_PREFIX + "-AdminVerified", "Mock");
|
||||
System.out.println(perform(builder, UserService.ADMIN_USER));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test0EditorPage() throws Exception {
|
||||
MockHttpServletRequestBuilder builder = MockMvcRequestBuilders
|
||||
.get("/admin/classification/" + getClassification())
|
||||
.sessionAttr(WebUtils.KEY_PREFIX + "-AdminVerified", "Mock");
|
||||
System.out.println(perform(builder, UserService.ADMIN_USER));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test1Import() throws Exception {
|
||||
ClassificationImporter importer = new ClassificationImporter(
|
||||
UserService.ADMIN_USER, getClassification(), "CHINA-ICNEA.json");
|
||||
importer.run();
|
||||
System.out.println("ClassificationImporter : " + importer.getTotal());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test9Delete() throws Exception {
|
||||
MockHttpServletRequestBuilder builder = MockMvcRequestBuilders
|
||||
.post("/admin/classification/delete?id=" + getClassification())
|
||||
.sessionAttr(WebUtils.KEY_PREFIX + "-AdminVerified", "Mock");
|
||||
System.out.println(perform(builder, UserService.ADMIN_USER));
|
||||
}
|
||||
|
||||
private static ID lastAdded = null;
|
||||
private static ID getClassification() {
|
||||
if (lastAdded == null) {
|
||||
Record record = EntityHelper.forNew(EntityHelper.Classification, UserService.ADMIN_USER);
|
||||
record.setString("name", "测试" + System.currentTimeMillis());
|
||||
record = Application.getCommonService().create(record);
|
||||
lastAdded = record.getPrimary();
|
||||
}
|
||||
System.out.println("Mock Classification : " + lastAdded);
|
||||
return lastAdded;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue