feat: 适配器解析引擎，用 CSS 选择器从提交的 HTML 中自动提取数据进行校验

2026-07-01 01:25:16 +08:00
parent e27a5b59ed
commit e56f1d4b92
9 changed files with 439 additions and 0 deletions
--- a/par-api/src/main/java/com/par/api/controller/AdapterController.java
+++ b/par-api/src/main/java/com/par/api/controller/AdapterController.java
@@ -0,0 +1,87 @@
+package com.par.api.controller;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.par.core.adapter.AdapterConfig;
+import com.par.core.adapter.AdapterEngine;
+import com.par.core.dto.ApiResponse;
+import com.par.core.entity.AccessResult;
+import com.par.core.entity.SiteConfig;
+import com.par.core.mapper.AccessResultMapper;
+import com.par.core.mapper.SiteConfigMapper;
+import com.par.core.storage.CachedStorageService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.Map;
+
+/**
+ * 适配器解析控制器
+ */
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+public class AdapterController {
+
+    private final AdapterEngine adapterEngine;
+    private final SiteConfigMapper siteConfigMapper;
+    private final AccessResultMapper accessResultMapper;
+    private final CachedStorageService storageService;
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    /**
+     * 用指定适配配置解析指定访问结果
+     */
+    @PostMapping("/api/v1/admin/adapter/parse")
+    public ApiResponse<Map<String, Object>> parse(
+            @RequestBody Map<String, Object> payload) {
+
+        Long configId = toLong(payload.get("configId"));
+        Long accessResultId = toLong(payload.get("accessResultId"));
+
+        if (configId == null || accessResultId == null) {
+            return ApiResponse.error(400, "configId and accessResultId are required");
+        }
+
+        // 加载适配配置
+        SiteConfig siteConfig = siteConfigMapper.selectById(configId);
+        if (siteConfig == null) return ApiResponse.error(404, "配置不存在");
+
+        String configJson;
+        try { configJson = storageService.read(siteConfig.getStoragePath()); }
+        catch (Exception e) { return ApiResponse.error(500, "读取配置失败: " + e.getMessage()); }
+        if (configJson == null) return ApiResponse.error(404, "配置内容为空");
+
+        AdapterConfig adapterConfig;
+        try { adapterConfig = objectMapper.readValue(configJson, AdapterConfig.class); }
+        catch (Exception e) { return ApiResponse.error(400, "配置格式无效: " + e.getMessage()); }
+
+        // 加载访问结果 HTML
+        AccessResult accessResult = accessResultMapper.selectById(accessResultId);
+        if (accessResult == null) return ApiResponse.error(404, "访问结果不存在");
+
+        String html;
+        try { html = storageService.read(accessResult.getStoragePath()); }
+        catch (Exception e) { return ApiResponse.error(500, "读取访问结果失败: " + e.getMessage()); }
+        if (html == null) return ApiResponse.error(404, "访问结果内容为空");
+
+        // 执行解析
+        Map<String, Object> result;
+        try {
+            result = adapterEngine.extract(adapterConfig, html, accessResult.getUrl());
+        } catch (Exception e) {
+            log.error("Adapter parse error", e);
+            return ApiResponse.error(500, "解析异常: " + e.getMessage());
+        }
+
+        return ApiResponse.success(result);
+    }
+
+    private Long toLong(Object val) {
+        if (val instanceof Number n) return n.longValue();
+        if (val instanceof String s) {
+            try { return Long.parseLong(s); } catch (NumberFormatException ignored) {}
+        }
+        return null;
+    }
+}
--- a/par-core/pom.xml
+++ b/par-core/pom.xml
@@ -96,5 +96,11 @@
            <artifactId>qiniu-java-sdk</artifactId>
            <version>7.15.0</version>
        </dependency>
+
+        <!-- Jsoup HTML 解析 -->
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+        </dependency>
    </dependencies>
 </project>
--- a/par-core/src/main/java/com/par/core/adapter/AdapterConfig.java
+++ b/par-core/src/main/java/com/par/core/adapter/AdapterConfig.java
@@ -0,0 +1,22 @@
+package com.par.core.adapter;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Data;
+
+import java.util.Map;
+
+/**
+ * 适配器配置根对象
+ */
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AdapterConfig {
+    /** 站点 ID */
+    private String siteId;
+    /** 站点类型 */
+    private String siteType;
+    /** 域名列表 */
+    private String[] domains;
+    /** 页面解析规则 */
+    private Map<String, PageConfig> pages;
+}
--- a/par-core/src/main/java/com/par/core/adapter/AdapterEngine.java
+++ b/par-core/src/main/java/com/par/core/adapter/AdapterEngine.java
@@ -0,0 +1,211 @@
+package com.par.core.adapter;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * 适配器解析引擎：根据配置文件的 CSS 选择器从 HTML 中提取数据
+ */
+@Slf4j
+@Service
+public class AdapterEngine {
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    /**
+     * 解析单个 HTML 页面，匹配配置中的页面规则并提取数据
+     * @return { pageName, pageUrl, extracted: [{field: value, ...}, ...] }
+     */
+    public Map<String, Object> extract(AdapterConfig config, String html, String accessUrl) {
+        Map<String, Object> result = new LinkedHashMap<>();
+        Document doc = Jsoup.parse(html);
+
+        // 匹配页面
+        Map.Entry<String, PageConfig> matched = matchPage(config.getPages(), accessUrl);
+        if (matched == null) {
+            result.put("matched", false);
+            result.put("message", "未匹配到页面规则");
+            result.put("availablePages", config.getPages().keySet());
+            return result;
+        }
+
+        String pageName = matched.getKey();
+        PageConfig page = matched.getValue();
+        result.put("matched", true);
+        result.put("pageName", pageName);
+        result.put("pageUrl", accessUrl);
+
+        // 提取数据
+        List<Map<String, Object>> extracted = extractRows(doc, page);
+        result.put("extracted", extracted);
+        result.put("rowCount", extracted.size());
+        result.put("fieldCount", page.getFields() != null ? page.getFields().size() : 0);
+
+        return result;
+    }
+
+    /**
+     * 匹配 URL 对应的页面规则
+     */
+    private Map.Entry<String, PageConfig> matchPage(Map<String, PageConfig> pages, String accessUrl) {
+        if (pages == null || pages.isEmpty() || accessUrl == null) return null;
+
+        for (Map.Entry<String, PageConfig> entry : pages.entrySet()) {
+            PageConfig page = entry.getValue();
+
+            // 精确 URL 匹配
+            if (page.getUrl() != null) {
+                String normalizedPageUrl = page.getUrl().replaceAll("\\?.+", "");
+                String normalizedAccessUrl = accessUrl.replaceAll("\\?.+", "");
+                if (normalizedAccessUrl.contains(normalizedPageUrl)
+                        || normalizedPageUrl.contains(normalizedAccessUrl)) {
+                    return entry;
+                }
+            }
+
+            // URL 正则匹配
+            if (page.getUrlPattern() != null) {
+                try {
+                    if (Pattern.compile(page.getUrlPattern()).matcher(accessUrl).find()) {
+                        return entry;
+                    }
+                } catch (Exception ignored) {}
+            }
+
+            // 模糊匹配：URL 路径中包含页面 key
+            if (accessUrl.contains("/" + entry.getKey())) {
+                return entry;
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * 从文档中提取所有行的数据
+     */
+    @SuppressWarnings("unchecked")
+    private List<Map<String, Object>> extractRows(Document doc, PageConfig page) {
+        if (page.getFields() == null || page.getFields().isEmpty()) return List.of();
+
+        Elements rows;
+        if (page.getListContainer() != null && !page.getListContainer().isBlank()) {
+            rows = doc.select(page.getListContainer());
+        } else {
+            // 无列表容器 → 整页提取一行
+            rows = new Elements();
+            rows.add(doc);
+        }
+
+        List<Map<String, Object>> results = new ArrayList<>();
+        int skip = Math.max(0, page.getSkipRows());
+        for (int i = skip; i < rows.size(); i++) {
+            Element row = rows.get(i);
+            Map<String, Object> rowData = new LinkedHashMap<>();
+            boolean hasData = false;
+
+            for (Map.Entry<String, FieldExtractor> field : page.getFields().entrySet()) {
+                String fieldName = field.getKey();
+                FieldExtractor fe = field.getValue();
+                if (fe.getSelector() == null) continue;
+
+                Element el = row.selectFirst(fe.getSelector());
+                if (el == null) {
+                    if (fe.getOptional() == null || !fe.getOptional()) {
+                        rowData.put(fieldName, null);
+                    }
+                    continue;
+                }
+
+                String raw;
+                String type = fe.getType() != null ? fe.getType() : "text";
+                switch (type) {
+                    case "attribute":
+                        raw = fe.getAttribute() != null ? el.attr(fe.getAttribute()) : el.text();
+                        break;
+                    case "html":
+                        raw = el.html();
+                        break;
+                    default:
+                        raw = el.text();
+                }
+
+                Object value = applyTransform(raw, fe.getTransform());
+                rowData.put(fieldName, value);
+                hasData = true;
+            }
+
+            if (hasData) results.add(rowData);
+        }
+
+        return results;
+    }
+
+    /**
+     * 应用内置转换器
+     */
+    private Object applyTransform(String raw, String transform) {
+        if (raw == null || transform == null) return raw;
+
+        try {
+            return switch (transform.toLowerCase()) {
+                case "int", "parseint" -> {
+                    String clean = raw.replaceAll("[^\\d-]", "");
+                    yield clean.isEmpty() ? null : Integer.parseInt(clean);
+                }
+                case "long" -> {
+                    String clean = raw.replaceAll("[^\\d-]", "");
+                    yield clean.isEmpty() ? null : Long.parseLong(clean);
+                }
+                case "double", "float" -> {
+                    String clean = raw.replaceAll("[^\\d.-]", "");
+                    yield clean.isEmpty() ? null : Double.parseDouble(clean);
+                }
+                case "filesize" -> parseFileSize(raw);
+                case "boolean", "bool" -> parseBoolean(raw);
+                default -> raw;
+            };
+        } catch (Exception e) {
+            return raw;
+        }
+    }
+
+    private Long parseFileSize(String raw) {
+        if (raw == null) return null;
+        Matcher m = Pattern.compile("([\\d.]+)\\s*(GB|MB|KB|TB|B|GiB|MiB|KiB|TiB)",
+                Pattern.CASE_INSENSITIVE).matcher(raw);
+        if (!m.find()) {
+            // try just number
+            String num = raw.replaceAll("[^\\d.]", "");
+            if (!num.isEmpty()) return (long) Double.parseDouble(num);
+            return null;
+        }
+        double val = Double.parseDouble(m.group(1));
+        String unit = m.group(2).toUpperCase();
+        return (long) (val * switch (unit) {
+            case "TB", "TIB" -> 1024L * 1024 * 1024 * 1024;
+            case "GB", "GIB" -> 1024L * 1024 * 1024;
+            case "MB", "MIB" -> 1024L * 1024;
+            case "KB", "KIB" -> 1024L;
+            default -> 1L;
+        });
+    }
+
+    private Boolean parseBoolean(String raw) {
+        if (raw == null) return null;
+        String s = raw.trim().toLowerCase();
+        if (s.equals("true") || s.equals("yes") || s.equals("1") || s.equals("是")) return true;
+        if (s.equals("false") || s.equals("no") || s.equals("0") || s.equals("否")) return false;
+        return null;
+    }
+}
--- a/par-core/src/main/java/com/par/core/adapter/FieldExtractor.java
+++ b/par-core/src/main/java/com/par/core/adapter/FieldExtractor.java
@@ -0,0 +1,22 @@
+package com.par.core.adapter;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Data;
+
+/**
+ * 字段提取器定义
+ */
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class FieldExtractor {
+    /** CSS 选择器 */
+    private String selector;
+    /** 提取类型: text | attribute | html */
+    private String type = "text";
+    /** type=attribute 时指定属性名 */
+    private String attribute;
+    /** 转换器名称 */
+    private String transform;
+    /** 是否可选 */
+    private Boolean optional;
+}
--- a/par-core/src/main/java/com/par/core/adapter/PageConfig.java
+++ b/par-core/src/main/java/com/par/core/adapter/PageConfig.java
@@ -0,0 +1,26 @@
+package com.par.core.adapter;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Data;
+
+import java.util.Map;
+
+/**
+ * 页面解析配置
+ */
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PageConfig {
+    /** 页面 URL 或 URL 模板 */
+    private String url;
+    /** URL 正则匹配模式 */
+    private String urlPattern;
+    /** 请求方法 */
+    private String method = "GET";
+    /** 列表容器 CSS 选择器 */
+    private String listContainer;
+    /** 跳过行数 */
+    private int skipRows;
+    /** 字段定义 map */
+    private Map<String, FieldExtractor> fields;
+}
--- a/pom.xml
+++ b/pom.xml
@@ -117,6 +117,13 @@
                <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
                <version>${springdoc.version}</version>
            </dependency>
+
+            <!-- Jsoup HTML 解析 -->
+            <dependency>
+                <groupId>org.jsoup</groupId>
+                <artifactId>jsoup</artifactId>
+                <version>1.18.1</version>
+            </dependency>
        </dependencies>
    </dependencyManagement>

--- a/web/src/api/index.js
+++ b/web/src/api/index.js
@@ -86,5 +86,8 @@ export default {
    bySite: (siteId) => api.get(`/admin/error-logs/by-site/${siteId}`),
    get: (id) => api.get(`/admin/error-logs/${id}`),
  },
+  adapter: {
+    parse: (configId, accessResultId) => api.post('/admin/adapter/parse', { configId, accessResultId }),
+  },
  health: () => api.get('/health', { baseURL: '' })
 }
--- a/web/src/views/AccessResultCheck.vue
+++ b/web/src/views/AccessResultCheck.vue
@@ -113,6 +113,7 @@
              <el-tag v-if="r.isValid === true" size="small" type="success">有效</el-tag>
              <el-tag v-else-if="r.isValid === false" size="small" type="danger">无效</el-tag>
              <el-button size="small" @click="viewContent(r)">查看</el-button>
+              <el-button size="small" type="primary" :disabled="!selectedConfig" @click="parseResult(r)">解析</el-button>
            </div>
          </div>
          <div v-if="!expandedUrls.has(page.url) && page.totalCount > 5" style="padding:2px 0;color:#909399;font-size:12px;text-align:center">
@@ -145,6 +146,36 @@
        <el-button v-if="viewingResult.isValid == null" type="danger" size="small" @click="singleValidate(viewingResult.id, false)">无效</el-button>
      </template>
    </el-dialog>
+
+    <!-- 适配解析结果弹窗 -->
+    <el-dialog v-model="parseVisible" title="适配解析结果" width="850px" :close-on-click-modal="false">
+      <div v-if="parseLoading" style="text-align:center;padding:40px">解析中...</div>
+      <div v-else-if="parseError" style="color:#f56c6c;padding:10px">{{ parseError }}</div>
+      <template v-else>
+        <el-descriptions :column="3" border size="small" style="margin-bottom:12px">
+          <el-descriptions-item label="匹配页面">{{ parseData.pageName || '-' }}</el-descriptions-item>
+          <el-descriptions-item label="提取行数">{{ parseData.rowCount ?? 0 }}</el-descriptions-item>
+          <el-descriptions-item label="字段数">{{ parseData.fieldCount ?? 0 }}</el-descriptions-item>
+        </el-descriptions>
+        <div v-if="!parseData.matched" style="color:#e6a23c;margin-bottom:8px">
+          未匹配到页面规则。可用页面: {{ (parseData.availablePages || []).join(', ') || '无' }}
+        </div>
+        <el-table v-if="parseData.extracted?.length" :data="parseData.extracted" border stripe size="small" max-height="400">
+          <el-table-column type="index" label="#" width="50" />
+          <el-table-column v-for="col in parseColumns" :key="col" :prop="col" :label="col" min-width="120" show-overflow-tooltip>
+            <template #default="scope">
+              <span :style="scope.row[col] == null ? 'color:#909399' : ''">{{ scope.row[col] ?? 'null' }}</span>
+            </template>
+          </el-table-column>
+        </el-table>
+        <div v-else-if="parseData.matched" style="text-align:center;padding:20px;color:#909399">
+          无提取结果（HTML 内容可能不包含匹配元素）
+        </div>
+      </template>
+      <template #footer>
+        <el-button @click="parseVisible = false">关闭</el-button>
+      </template>
+    </el-dialog>
  </div>
 </template>

@@ -175,6 +206,11 @@ const viewingResult = ref({})
 const viewingContentType = ref('html')
 const viewingContent = ref('')

+const parseVisible = ref(false)
+const parseLoading = ref(false)
+const parseError = ref('')
+const parseData = ref({})
+
 const expandedUrls = reactive(new Set())
 const MAX_VISIBLE = 5

@@ -322,6 +358,25 @@ const singleValidate = async (id, isValid) => {
  } catch { ElMessage.error('校验失败') }
 }

+const parseColumns = computed(() => {
+  if (!parseData.value.extracted?.length) return []
+  return Object.keys(parseData.value.extracted[0])
+})
+
+const parseResult = async (row) => {
+  if (!selectedConfig.value) { ElMessage.warning('请先选择适配配置'); return }
+  parseVisible.value = true
+  parseLoading.value = true
+  parseError.value = ''
+  parseData.value = {}
+  try {
+    const r = await api.adapter.parse(selectedConfig.value.id, row.id)
+    parseData.value = r.data || {}
+  } catch (e) {
+    parseError.value = e.response?.data?.message || '解析失败'
+  } finally { parseLoading.value = false }
+}
+
 const viewContent = async (row) => {
  viewingResult.value = { ...row }
  viewingContentType.value = row.contentType || 'html'