Imports logs (#446)

* imports trace logs
2025-03-13 15:44:26 +08:00 · 2022-03-28 21:59:01 +08:00 · 2022-03-28 21:59:01 +08:00 · 1e5717610a
commit 1e5717610a
parent 1b165b1e4c
5 changed files with 196 additions and 78 deletions
--- a/src/main/java/com/rebuild/core/service/dataimport/DataImporter.java
+++ b/src/main/java/com/rebuild/core/service/dataimport/DataImporter.java
@ -22,6 +22,7 @@ import com.rebuild.core.service.general.GeneralEntityServiceContextHolder;
 import com.rebuild.core.support.task.HeavyTask;
 import com.rebuild.utils.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;

 import java.util.*;

@ -38,8 +39,8 @@ public class DataImporter extends HeavyTask<Integer> {
    final private ImportRule rule;
    private ID owningUser;

-    // 记录每一行的错误日志
-    private Map<Integer, Object> eachLogs = new LinkedHashMap<>();
+    final private List<Object[]> traceLogs = new ArrayList<>();
+    private String cellTraces = null;

    /**
     * @param rule
@ -62,25 +63,31 @@ public class DataImporter extends HeavyTask<Integer> {
                break;
            }

-            Cell firstCell = row == null || row.length == 0 ? null : row[0];
-            if (firstCell == null || firstCell.getRowNo() == 0) {
+            Cell fc = row == null || row.length == 0 ? null : row[0];
+            if (fc == null || fc.getRowNo() == 0) {
                continue;
            }

            try {
                Record record = checkoutRecord(row);
-                if (record != null) {
+                if (record == null) {
+                    traceLogs.add(new Object[] { fc.getRowNo(), "SKIP" });
+                } else {
+                    boolean isNew = record.getPrimary() == null;
                    record = Application.getEntityService(rule.getToEntity().getEntityCode()).createOrUpdate(record);
                    this.addSucceeded();
-                    eachLogs.put(firstCell.getRowNo(), record.getPrimary());
+
+                    traceLogs.add(new Object[] { fc.getRowNo(),
+                            isNew ? "CREATED" : "UPDATED", record.getPrimary(), cellTraces });
                }
+
            } catch (Exception ex) {
-                eachLogs.put(firstCell.getRowNo(), ex.getLocalizedMessage());
-                log.error("ROW#{} > {}", firstCell.getRowNo(), ex);
+                traceLogs.add(new Object[] { fc.getRowNo(), "ERROR", ex.getLocalizedMessage() });
+                log.error("ROW#{} > {}", fc.getRowNo(), ex.getLocalizedMessage());
            }
+
            this.addCompleted();
        }
-
        return this.getSucceeded();
    }

@ -90,28 +97,26 @@ public class DataImporter extends HeavyTask<Integer> {
        GeneralEntityServiceContextHolder.isSkipSeriesValue(true);
    }

-    /**
-     * 获取错误日志（按错误行）
-     *
-     * @return
-     */
-    public Map<Integer, Object> getEachLogs() {
-        return eachLogs;
-    }
-
    /**
     * @param row
     * @return
     */
    protected Record checkoutRecord(Cell[] row) {
-        Record recordNew = EntityHelper.forNew(rule.getToEntity().getEntityCode(), this.owningUser);
+        Record recordHub = EntityHelper.forNew(rule.getToEntity().getEntityCode(), this.owningUser);

-        // 新建
-        Record record = new RecordCheckout(rule.getFiledsMapping()).checkout(recordNew, row);
+        // 解析数据
+        RecordCheckout recordCheckout = new RecordCheckout(rule.getFiledsMapping());
+        Record checkout = recordCheckout.checkout(recordHub, row);
+
+        if (recordCheckout.getTraceLogs().isEmpty()) {
+            cellTraces = null;
+        } else {
+            cellTraces = StringUtils.join(recordCheckout.getTraceLogs(), ", ");
+        }

        // 检查重复
        if (rule.getRepeatOpt() < ImportRule.REPEAT_OPT_IGNORE) {
-            final ID repeat = findRepeatedRecordId(rule.getRepeatFields(), recordNew);
+            final ID repeat = findRepeatedRecordId(rule.getRepeatFields(), recordHub);

            if (repeat != null && rule.getRepeatOpt() == ImportRule.REPEAT_OPT_SKIP) {
                return null;
@ -119,24 +124,24 @@ public class DataImporter extends HeavyTask<Integer> {

            if (repeat != null && rule.getRepeatOpt() == ImportRule.REPEAT_OPT_UPDATE) {
                // 更新
-                record = EntityHelper.forUpdate(repeat, this.owningUser);
-                for (Iterator<String> iter = recordNew.getAvailableFieldIterator(); iter.hasNext(); ) {
+                checkout = EntityHelper.forUpdate(repeat, this.owningUser);
+                for (Iterator<String> iter = recordHub.getAvailableFieldIterator(); iter.hasNext(); ) {
                    String field = iter.next();
-                    if (MetadataHelper.isCommonsField(field)) {
-                        continue;
-                    }
-                    record.setObjectValue(field, recordNew.getObjectValue(field));
+                    if (MetadataHelper.isCommonsField(field)) continue;
+
+                    checkout.setObjectValue(field, recordHub.getObjectValue(field));
                }
            }
        }

        // Verify new record
-        if (record.getPrimary() == null) {
-            EntityRecordCreator verifier = new EntityRecordCreator(rule.getToEntity(), JSONUtils.EMPTY_OBJECT, null);
-            verifier.verify(record);
+        // Throws DataSpecificationException
+        if (checkout.getPrimary() == null) {
+            new EntityRecordCreator(rule.getToEntity(), JSONUtils.EMPTY_OBJECT, null)
+                    .verify(checkout);
        }

-        return record;
+        return checkout;
    }

    /**
@ -171,4 +176,12 @@ public class DataImporter extends HeavyTask<Integer> {
        Object[] exists = query.unique();
        return exists == null ? null : (ID) exists[0];
    }
+
+    /**
+     * 错误日志
+     * @return
+     */
+    public List<Object[]> getTraceLogs() {
+        return traceLogs;
+    }
 }
--- a/src/main/java/com/rebuild/core/service/dataimport/RecordCheckout.java
+++ b/src/main/java/com/rebuild/core/service/dataimport/RecordCheckout.java
@ -27,10 +27,10 @@ import com.rebuild.core.metadata.EntityHelper;
 import com.rebuild.core.metadata.MetadataSorter;
 import com.rebuild.core.metadata.easymeta.*;
 import com.rebuild.core.metadata.impl.MetadataModificationException;
+import com.rebuild.core.support.i18n.Language;
 import com.rebuild.core.support.state.StateManager;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.NumberUtils;

 import java.text.MessageFormat;
@ -46,6 +46,8 @@ import java.util.*;
@Slf4j
 public class RecordCheckout {

+    final private List<String> traceLogs = new ArrayList<>();
+
    final private Map<Field, Integer> fieldsMapping;

    protected RecordCheckout(Map<Field, Integer> fieldsMapping) {
@ -62,23 +64,30 @@ public class RecordCheckout {
            int cellIndex = e.getValue();
            if (cellIndex >= row.length) continue;

-            Field field = e.getKey();
            Cell cellValue = row[cellIndex];
+            if (cellValue == Cell.NULL || cellValue.isEmpty()) {
+                continue;
+            }
+
+            Field field = e.getKey();
            Object value = checkoutFieldValue(field, cellValue, true);

            if (value != null) {
                record.setObjectValue(field.getName(), value);
-            } else if (cellValue != Cell.NULL && !cellValue.isEmpty()) {
-                log.warn("Invalid value of cell : " + cellValue + " > " + field.getName());
+            } else {
+                putTraceLog(cellValue, Language.L(EasyMetaFactory.getDisplayType(field)));
            }
        }
+
        return record;
    }

    /**
+     * 验证格式，如邮箱/URL等
+     *
     * @param field
     * @param cell
-     * @param validate 验证格式，如邮箱/URL等
+     * @param validate
     * @return
     */
    protected Object checkoutFieldValue(Field field, Cell cell, boolean validate) {
@ -127,8 +136,7 @@ public class RecordCheckout {
    }

    protected ID checkoutPickListValue(Field field, Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
+        final String val = cell.asString();

        // 支持ID
        if (ID.isId(val) && ID.valueOf(val).getEntityCode() == EntityHelper.PickList) {
@ -146,25 +154,16 @@ public class RecordCheckout {

    protected Integer checkoutStateValue(Field field, Cell cell) {
        final String val = cell.asString();
-        if (StringUtils.isBlank(val)) {
-            return null;
-        }

        try {
            return StateManager.instance.findState(field, val).getState();
        } catch (MetadataModificationException ignored) {
        }
-
-        // 兼容状态值
-        if (NumberUtils.isNumber(val)) {
-            return NumberUtils.toInt(val);
-        }
        return null;
    }

    protected ID checkoutClassificationValue(Field field, Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
+        final String val = cell.asString();

        // 支持ID
        if (ID.isId(val) && ID.valueOf(val).getEntityCode() == EntityHelper.ClassificationData) {
@ -181,12 +180,10 @@ public class RecordCheckout {
    }

    protected ID checkoutReferenceValue(Field field, Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
-
+        final String val = cell.asString();
        final Entity refEntity = field.getReferenceEntity();

-        // 支持ID导入
+        // 支持ID
        if (ID.isId(val) && ID.valueOf(val).getEntityCode().intValue() == refEntity.getEntityCode()) {
            ID checkId = ID.valueOf(val);
            Object exists = Application.getQueryFactory().uniqueNoFilter(checkId, refEntity.getPrimaryField().getName());
@ -199,9 +196,7 @@ public class RecordCheckout {
        }

        Object val2Text = checkoutFieldValue(refEntity.getNameField(), cell, false);
-        if (val2Text == null) {
-            return null;
-        }
+        if (val2Text == null) return null;

        Query query;
        // 用户特殊处理
@ -222,8 +217,8 @@ public class RecordCheckout {
                    String.format("select %s from %s where ",
                            refEntity.getPrimaryField().getName(), refEntity.getName()));
            for (String qf : queryFields) {
-                sql.append(
-                        String.format("%s = '%s' or ", qf, StringEscapeUtils.escapeSql((String) val2Text)));
+                sql.append(String.format("%s = '%s' or ",
+                        qf, StringEscapeUtils.escapeSql((String) val2Text)));
            }
            sql = new StringBuilder(sql.substring(0, sql.length() - 4));

@ -235,8 +230,7 @@ public class RecordCheckout {
    }

    protected ID[] checkoutN2NReferenceValue(Field field, Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
+        final String val = cell.asString();

        Set<ID> ids = new LinkedHashSet<>();
        for (String s : val.split("[,，;；]")) {
@ -250,10 +244,8 @@ public class RecordCheckout {
        Date date = cell.asDate();
        if (date != null) return date;

-        if (cell.isEmpty()) return null;
-
        String date2str = cell.asString();
-        
+
        try {
            DateTime dt = DateUtil.parse(date2str);
            if (dt != null) date = dt.toJdkDate();
@ -264,26 +256,19 @@ public class RecordCheckout {
        if (date == null && date2str.contains("/")) {
            return cell.asDate(new String[]{"yyyy/M/d H:m:s", "yyyy/M/d H:m", "yyyy/M/d"});
        }
-
-        return null;
+        return date;
    }

    protected LocalTime checkoutTimeValue(Cell cell) {
-        if (cell.isEmpty()) return null;
-
-        String time2str = cell.asString();
-
        try {
-            return RecordVisitor.tryParseTime(time2str);
+            return RecordVisitor.tryParseTime(cell.asString());
        } catch (FieldValueException ignored) {
        }
-
        return null;
    }

    protected Long checkoutMultiSelectValue(Field field, Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
+        final String val = cell.asString();

        long mVal = 0;
        for (String s : val.split("[,，;；]")) {
@ -293,8 +278,7 @@ public class RecordCheckout {
    }

    protected String checkoutFileOrImage(Cell cell) {
-        String val = cell.asString();
-        if (StringUtils.isBlank(val)) return null;
+        final String val = cell.asString();

        List<String> urls = new ArrayList<>();
        for (String s : val.split("[,，;；]")) {
@ -302,4 +286,26 @@ public class RecordCheckout {
        }
        return urls.isEmpty() ? null : JSON.toJSON(urls).toString();
    }
+
+    /**
+     * @return
+     */
+    public List<String> getTraceLogs() {
+        return traceLogs;
+    }
+
+    private void putTraceLog(Cell cell, String log) {
+        // A1 A2 ...
+        int num = cell.getColumnNo();
+        StringBuilder name = new StringBuilder();
+        while (num >= 0) {
+            int remainder = num % 26;
+            name.insert(0, (char) (remainder + 65));
+            //noinspection IntegerDivisionInFloatingPointContext
+            num = (int) Math.floor(num / 26) - 1;
+        }
+        name.append(cell.getRowNo() + 1);
+
+        traceLogs.add(name + ":" + log);
+    }
 }
--- a/src/main/java/com/rebuild/web/admin/data/DataImportController.java
+++ b/src/main/java/com/rebuild/web/admin/data/DataImportController.java
@ -28,6 +28,7 @@ import com.rebuild.core.service.dataimport.DataImporter;
 import com.rebuild.core.service.dataimport.ImportRule;
 import com.rebuild.core.support.RebuildConfiguration;
 import com.rebuild.core.support.i18n.Language;
+import com.rebuild.core.support.task.HeavyTask;
 import com.rebuild.core.support.task.TaskExecutors;
 import com.rebuild.utils.JSONUtils;
 import com.rebuild.web.BaseController;
@ -168,4 +169,16 @@ public class DataImportController extends BaseController {
        File tmp = RebuildConfiguration.getFileOfTemp(file);
        return (!tmp.exists() || tmp.isDirectory()) ? null : tmp;
    }
+
+    @GetMapping("/data-imports/import-trace")
+    public RespBody importTrace(HttpServletRequest request) {
+        String taskid = getParameterNotNull(request, "taskid");
+        HeavyTask<?> task = TaskExecutors.get(taskid);
+
+        if (task == null) {
+            return RespBody.error();
+        } else {
+            return RespBody.ok(((DataImporter) task).getTraceLogs());
+        }
+    }
 }
--- a/src/main/resources/web/admin/data/data-imports.html
+++ b/src/main/resources/web/admin/data/data-imports.html
@ -168,6 +168,7 @@
                        </div>
                        <div class="mt-3">
                          <button class="btn btn-danger btn-space J_step3-cancel" type="button">[[${bundle.L('终止导入')}]]</button>
+                          <button class="btn btn-secondary btn-space J_step3-trace" type="button">[[${bundle.L('导入详情')}]]</button>
                          <a class="btn btn-link btn-space J_step3-next hide" href="data-imports">[[${bundle.L('继续导入')}]]</a>
                        </div>
                      </form>
--- a/src/main/resources/web/assets/js/admin/data-imports.js
+++ b/src/main/resources/web/assets/js/admin/data-imports.js
@ -20,9 +20,6 @@ let import_inprogress = false
 let import_taskid

 const entity = $urlp('entity')
-if (entity) {
-  $('.J_step3-next').attr('href', `${$('.J_step3-next').attr('href')}?entity=${entity}`)
-}

 $(document).ready(() => {
  $.get('/commons/metadata/entities?detail=true', (res) => {
@ -79,6 +76,10 @@ $(document).ready(() => {
  window.onbeforeunload = function () {
    if (import_inprogress === true) return false
  }
+
+  $('.J_step3-trace').on('click', () => {
+    renderRbcomp(<ImportsTraceViewer width="681" taskid={import_taskid} />)
+  })
 })

 // 1. 初始导入
@ -167,6 +168,11 @@ const step3_import_show = () => {
  $('.steps li, .step-content .step-pane').removeClass('active complete')
  $('.steps li[data-step=1], .steps li[data-step=2]').addClass('complete')
  $('.steps li[data-step=3], .step-content .step-pane[data-step=3]').addClass('active')
+
+  // Next
+  if (_Config.entity || entity) {
+    $('.J_step3-next').attr('href', `${$('.J_step3-next').attr('href')}?entity=${_Config.entity || entity}`)
+  }
 }

 // 3.2. 导入状态
@ -338,3 +344,82 @@ function _renderRepeatFields(entity) {
    _Config.entity = entity
  })
 }
+
+// ~ 导入详情
+class ImportsTraceViewer extends RbAlert {
+  renderContent() {
+    return (
+      <table className="table table-fixed">
+        <thead>
+          <tr>
+            <th width="50" className="pr-0">
+              {$L('行号')}
+            </th>
+            <th width="120">{$L('状态')}</th>
+            <th>{$L('详情')}</th>
+          </tr>
+        </thead>
+        <tbody>
+          {(this.state.trace || []).map((item) => {
+            return (
+              <tr key={item[0]}>
+                <th className="pr-0">{item[0] + 1}</th>
+                <td>
+                  {item[1] === 'CREATED' && (
+                    <a target="_blank" title={$L('查看')} href={`${rb.baseUrl}/app/list-and-view?id=${item[2]}`}>
+                      {$L('新建成功')}
+                    </a>
+                  )}
+                  {item[1] === 'UPDATED' && (
+                    <a target="_blank" title={$L('查看')} href={`${rb.baseUrl}/app/list-and-view?id=${item[2]}`}>
+                      {$L('更新成功')}
+                    </a>
+                  )}
+                  {item[1] === 'SKIP' && <span className="text-muted">{$L('跳过')}</span>}
+                  {item[1] === 'ERROR' && <span className="text-danger">{$L('错误')}</span>}
+                </td>
+                <td>{this._formatDetail(item)}</td>
+              </tr>
+            )
+          })}
+        </tbody>
+      </table>
+    )
+  }
+
+  _formatDetail(item) {
+    if (item[1] === 'CREATED' || item[1] === 'UPDATED') {
+      return item[3] ? `${$L('单元格值错误')} ${item[3]}` : <span className="text-muted">-</span>
+    } else if (item[1] === 'ERROR') {
+      return item[2]
+    } else {
+      return <span className="text-muted">-</span>
+    }
+  }
+
+  componentDidMount() {
+    super.componentDidMount()
+    this.load()
+  }
+
+  componentWillUnmount() {
+    if (this._timer) {
+      clearTimeout(this._timer)
+      this._timer = null
+    }
+    console.log('componentWillUnmount')
+  }
+
+  load() {
+    $.get(`/admin/data/data-imports/import-trace?taskid=${this.props.taskid}`, (res) => {
+      if (res.error_code === 0) {
+        this.setState({ trace: res.data })
+
+        // reload
+        if (import_inprogress === true) this._timer = setTimeout(() => this.load(), 1500)
+      } else {
+        RbHighbar.error(res.error_msg)
+      }
+    })
+  }
+}