yescent/par
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 删除旧AdapterEngine及JSON配置相关代码，全面迁移到Groovy脚本

Browse Source
This commit is contained in:
mediabot-pt
2026-07-02 11:33:11 +08:00
parent 23111cc4f3
commit 4afc3b632b
12 changed files with 15 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
par-core/src/main/java/com/par/core/adapter/AdapterConfig.java
View File

@@ -1,22 +0,0 @@
package com.par.core.adapter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import java.util.Map;
/**
* 适配器配置根对象
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class AdapterConfig {
/** 站点 ID */
private String siteId;
/** 站点类型 */
private String siteType;
/** 域名列表 */
private String[] domains;
/** 页面解析规则 */
private Map<String, PageConfig> pages;
}

287
par-core/src/main/java/com/par/core/adapter/AdapterEngine.java
View File

@@ -1,287 +0,0 @@
package com.par.core.adapter;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 适配器解析引擎：根据配置文件的 CSS 选择器（HTML）或点分路径（JSON）提取数据
*/
@Slf4j
@Service
public class AdapterEngine {
private static final ObjectMapper objectMapper = new ObjectMapper();
public Map<String, Object> extract(AdapterConfig config, String content, String contentType, String accessUrl) {
if ("json".equalsIgnoreCase(contentType)) {
return extractJson(config, content, accessUrl);
}
return extractHtml(config, content, accessUrl);
}
public Map<String, Object> extract(AdapterConfig config, String content, String accessUrl) {
return extract(config, content, "html", accessUrl);
}
// ==================== JSON ====================
private Map<String, Object> extractJson(AdapterConfig config, String jsonStr, String accessUrl) {
Map<String, Object> result = new LinkedHashMap<>();
try {
Object parsed = objectMapper.readValue(jsonStr, Object.class);
result.put("contentType", "json");
Map.Entry<String, PageConfig> matched = matchPage(config.getPages(), accessUrl);
if (matched != null && matched.getValue().getFields() != null && !matched.getValue().getFields().isEmpty()) {
// 有页面规则 → 按 selector 提取
PageConfig page = matched.getValue();
result.put("matched", true);
result.put("pageName", matched.getKey());
List<Object> items = new ArrayList<>();
if (page.getListContainer() != null && !page.getListContainer().isBlank()) {
Object container = resolveJsonPath(parsed, page.getListContainer());
if (container instanceof List) items = (List<Object>) container;
else items.add(parsed);
} else {
items.add(parsed);
}
List<Map<String, Object>> rows = new ArrayList<>();
int max = Math.min(items.size(), 50);
for (int i = 0; i < max; i++) {
Map<String, Object> row = new LinkedHashMap<>();
boolean hasData = false;
for (Map.Entry<String, FieldExtractor> field : page.getFields().entrySet()) {
String fieldName = field.getKey();
FieldExtractor fe = field.getValue();
if (fe.getSelector() == null) continue;
Object val = resolveJsonPath(items.get(i), fe.getSelector());
if (val != null) hasData = true;
if (fe.getOptional() != null && fe.getOptional() && val == null) continue;
if (val instanceof String s) {
row.put(fieldName, applyTransform(s, fe.getTransform()));
} else if (val != null) {
row.put(fieldName, toDisplayString(val));
} else {
row.put(fieldName, null);
}
}
if (hasData) rows.add(row);
}
result.put("extracted", rows);
result.put("rowCount", rows.size());
result.put("fieldCount", page.getFields().size());
result.put("message", rows.isEmpty() ? "selector 未匹配到数据" : "共 " + rows.size() + " 行");
} else {
// 无页面规则 → 兜底展示原始结构
String reason = matched == null ? "未匹配到页面规则" : "页面规则未定义字段";
result.put("matched", true);
result.put("pageName", "JSON原始结构");
result.put("message", reason + "，展示原始数据");
List<Map<String, Object>> rows = new ArrayList<>();
if (parsed instanceof Map) {
Map<String, Object> row = new LinkedHashMap<>();
for (String key : ((Map<String, Object>) parsed).keySet()) {
row.put(key, toDisplayString(((Map<String, Object>) parsed).get(key)));
}
rows.add(row);
} else if (parsed instanceof List) {
int m = Math.min(((List<?>) parsed).size(), 10);
for (int i = 0; i < m; i++) {
Map<String, Object> row = new LinkedHashMap<>();
row.put("value", ((List<?>) parsed).get(i));
rows.add(row);
}
} else {
Map<String, Object> row = new LinkedHashMap<>();
row.put("value", parsed);
rows.add(row);
}
result.put("extracted", rows);
result.put("rowCount", rows.size());
result.put("fieldCount", rows.isEmpty() ? 0 : rows.get(0).size());
}
} catch (Exception e) {
result.put("contentType", "json");
result.put("matched", false);
result.put("message", "JSON 解析失败: " + e.getMessage());
}
return result;
}
@SuppressWarnings("unchecked")
private Object resolveJsonPath(Object root, String path) {
if (root == null || path == null || path.isBlank()) return null;
Object current = root;
for (String part : path.split("\\.")) {
if (current == null) return null;
String key = part;
int bracket = key.indexOf('[');
int arrIdx = -1;
if (bracket > 0) {
try {
arrIdx = Integer.parseInt(key.substring(bracket + 1, key.indexOf(']')));
key = key.substring(0, bracket);
} catch (Exception ignored) {}
}
if (current instanceof Map) {
current = ((Map<String, Object>) current).get(key);
} else if (current instanceof List && !key.isEmpty()) {
try {
int li = Integer.parseInt(key);
current = ((List<Object>) current).size() > li ? ((List<Object>) current).get(li) : null;
} catch (NumberFormatException e) { return null; }
} else {
return null;
}
if (arrIdx >= 0 && current instanceof List) {
current = ((List<Object>) current).size() > arrIdx ? ((List<Object>) current).get(arrIdx) : null;
}
}
return current;
}
// ==================== HTML ====================
private Map<String, Object> extractHtml(AdapterConfig config, String html, String accessUrl) {
Map<String, Object> result = new LinkedHashMap<>();
Document doc = Jsoup.parse(html);
Map.Entry<String, PageConfig> matched = matchPage(config.getPages(), accessUrl);
if (matched == null) {
result.put("matched", false);
result.put("contentType", "html");
result.put("message", "未匹配到页面规则");
result.put("availablePages", new ArrayList<>(config.getPages() != null ? config.getPages().keySet() : Set.of()));
return result;
}
PageConfig page = matched.getValue();
result.put("matched", true);
result.put("contentType", "html");
result.put("pageName", matched.getKey());
result.put("fieldCount", page.getFields() != null ? page.getFields().size() : 0);
List<Map<String, Object>> extracted = extractRows(doc, page);
result.put("extracted", extracted);
result.put("rowCount", extracted.size());
return result;
}
// ==================== 页面匹配 ====================
private Map.Entry<String, PageConfig> matchPage(Map<String, PageConfig> pages, String accessUrl) {
if (pages == null || pages.isEmpty() || accessUrl == null) return null;
for (Map.Entry<String, PageConfig> entry : pages.entrySet()) {
PageConfig page = entry.getValue();
if (page.getUrl() != null) {
String a = page.getUrl().replaceAll("\\?.+", "");
String b = accessUrl.replaceAll("\\?.+", "");
if (b.contains(a) || a.contains(b)) return entry;
}
if (page.getUrlPattern() != null) {
try { if (Pattern.compile(page.getUrlPattern()).matcher(accessUrl).find()) return entry; }
catch (Exception ignored) {}
}
if (accessUrl.contains("/" + entry.getKey())) return entry;
}
return null;
}
// ==================== HTML 提取 ====================
private List<Map<String, Object>> extractRows(Document doc, PageConfig page) {
if (page.getFields() == null || page.getFields().isEmpty()) return List.of();
Elements rows;
if (page.getListContainer() != null && !page.getListContainer().isBlank()) {
rows = doc.select(page.getListContainer());
} else {
rows = new Elements(); rows.add(doc);
}
List<Map<String, Object>> results = new ArrayList<>();
for (int i = Math.max(0, page.getSkipRows()); i < rows.size(); i++) {
Element row = rows.get(i);
Map<String, Object> rowData = new LinkedHashMap<>();
boolean hasData = false;
for (Map.Entry<String, FieldExtractor> f : page.getFields().entrySet()) {
FieldExtractor fe = f.getValue();
if (fe.getSelector() == null) continue;
Element el = row.selectFirst(fe.getSelector());
if (el == null) {
if (fe.getOptional() == null || !fe.getOptional()) rowData.put(f.getKey(), null);
continue;
}
String raw = switch (fe.getType() != null ? fe.getType() : "text") {
case "attribute" -> fe.getAttribute() != null ? el.attr(fe.getAttribute()) : el.text();
case "html" -> el.html();
default -> el.text();
};
rowData.put(f.getKey(), applyTransform(raw, fe.getTransform()));
hasData = true;
}
if (hasData) results.add(rowData);
}
return results;
}
// ==================== 转换器 ====================
private Object applyTransform(String raw, String transform) {
if (raw == null || transform == null) return raw;
try {
return switch (transform.toLowerCase()) {
case "int", "parseint" -> { String c = raw.replaceAll("[^\\d-]", ""); yield c.isEmpty() ? null : Integer.parseInt(c); }
case "long" -> { String c = raw.replaceAll("[^\\d-]", ""); yield c.isEmpty() ? null : Long.parseLong(c); }
case "double", "float" -> { String c = raw.replaceAll("[^\\d.-]", ""); yield c.isEmpty() ? null : Double.parseDouble(c); }
case "filesize" -> parseFileSize(raw);
case "boolean", "bool" -> parseBoolean(raw);
default -> raw;
};
} catch (Exception e) { return raw; }
}
private Long parseFileSize(String raw) {
if (raw == null) return null;
Matcher m = Pattern.compile("([\\d.]+)\\s*(GB|MB|KB|TB|B|GiB|MiB|KiB|TiB)", Pattern.CASE_INSENSITIVE).matcher(raw);
if (!m.find()) {
String n = raw.replaceAll("[^\\d.]", "");
return n.isEmpty() ? null : (long) Double.parseDouble(n);
}
double v = Double.parseDouble(m.group(1));
return (long) (v * switch (m.group(2).toUpperCase()) {
case "TB","TIB" -> 1024L*1024*1024*1024;
case "GB","GIB" -> 1024L*1024*1024;
case "MB","MIB" -> 1024L*1024;
case "KB","KIB" -> 1024L;
default -> 1L;
});
}
private Boolean parseBoolean(String raw) {
if (raw == null) return null;
String s = raw.trim().toLowerCase();
if (s.equals("true") || s.equals("yes") || s.equals("1") || s.equals("是")) return true;
if (s.equals("false") || s.equals("no") || s.equals("0") || s.equals("否")) return false;
return null;
}
private String toDisplayString(Object val) {
if (val == null) return "null";
if (val instanceof String s) return s;
if (val instanceof Number || val instanceof Boolean) return val.toString();
try { return objectMapper.writeValueAsString(val); } catch (Exception e) { return val.toString(); }
}
}

22
par-core/src/main/java/com/par/core/adapter/FieldExtractor.java
View File

@@ -1,22 +0,0 @@
package com.par.core.adapter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
/**
* 字段提取器定义
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class FieldExtractor {
/** CSS 选择器 */
private String selector;
/** 提取类型: text | attribute | html */
private String type = "text";
/** type=attribute 时指定属性名 */
private String attribute;
/** 转换器名称 */
private String transform;
/** 是否可选 */
private Boolean optional;
}

26
par-core/src/main/java/com/par/core/adapter/PageConfig.java
View File

@@ -1,26 +0,0 @@
package com.par.core.adapter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import java.util.Map;
/**
* 页面解析配置
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class PageConfig {
/** 页面 URL 或 URL 模板 */
private String url;
/** URL 正则匹配模式 */
private String urlPattern;
/** 请求方法 */
private String method = "GET";
/** 列表容器 CSS 选择器 */
private String listContainer;
/** 跳过行数 */
private int skipRows;
/** 字段定义 map */
private Map<String, FieldExtractor> fields;
}

3
par-core/src/main/java/com/par/core/dto/SiteConfigDTO.java
View File

@@ -14,10 +14,7 @@ public class SiteConfigDTO {
private Long id;
private String siteId;
private String version;
private String configHash;
private String scriptStoragePath;
private Long fileSize;
private String schemaVersion;
private ConfigStatus status;
private Boolean isLatest;
private Long submitterId;

12
par-core/src/main/java/com/par/core/entity/SiteConfig.java
View File

@@ -22,21 +22,9 @@ public class SiteConfig {
@TableField("version")
private String version;
@TableField("config_hash")
private String configHash;
@TableField("storage_path")
private String storagePath;
@TableField("script_storage_path")
private String scriptStoragePath;
@TableField("file_size")
private Long fileSize;
@TableField("schema_version")
private String schemaVersion;
@TableField("status")
private ConfigStatus status;

5
par-core/src/main/java/com/par/core/service/ConfigService.java
View File

@@ -31,10 +31,9 @@ public interface ConfigService {
SiteConfig getConfig(String siteId, String version);
/**
* 提交新配置（需审核），可选 Groovy 脚本
* 提交新适配脚本（需审核）
*/
SiteConfig submitConfig(String siteId, String version, String configJson,
String schemaVersion, String groovyScript, Long submitterId);
SiteConfig submitConfig(String siteId, String version, String groovyScript, Long submitterId);
/**
* 转换为 DTO

50
par-core/src/main/java/com/par/core/service/impl/ConfigServiceImpl.java
View File

@@ -10,7 +10,6 @@ import com.par.core.storage.CachedStorageService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.stream.Collectors;
@@ -54,57 +53,25 @@ public class ConfigServiceImpl implements ConfigService {
}
@Override
public SiteConfig submitConfig(String siteId, String version, String configJson,
String schemaVersion, String groovyScript, Long submitterId) {
boolean hasConfig = configJson != null && !configJson.isBlank();
public SiteConfig submitConfig(String siteId, String version, String groovyScript, Long submitterId) {
boolean hasScript = groovyScript != null && !groovyScript.isBlank();
String configHash = hasConfig ? HmacUtil.sha256(configJson) : HmacUtil.sha256(groovyScript);
String idPrefix = hasConfig ? configHash.substring(0, 8) : Long.toHexString(System.currentTimeMillis());
if (!hasScript) throw new IllegalArgumentException("groovyScript is required");
// 相同 hash 的提交 → 真正重复，跳过
SiteConfig sameHash = siteConfigMapper.selectOne(
new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper<SiteConfig>()
.eq(SiteConfig::getSiteId, siteId)
.eq(SiteConfig::getConfigHash, configHash)
.eq(SiteConfig::getDeleted, 0));
if (sameHash != null) {
log.info("配置未变化，跳过: siteId={}, hash={}", siteId, configHash);
return sameHash;
}
// 存储 JSON 配置（可选）
String storagePath = null;
if (hasConfig) {
storagePath = String.format("configs/%s/pending/%s.json", siteId, idPrefix);
try { storageService.store(storagePath, configJson); }
catch (Exception e) {
log.error("存储配置文件失败 siteId={} version={}: {}", siteId, version, e.getMessage());
throw new RuntimeException("存储配置文件失败: " + e.getMessage(), e);
}
}
// 存储 Groovy 脚本（用脚本自己的 hash，不和 config 共用）
String scriptPath = null;
if (hasScript) {
String scriptHash = HmacUtil.sha256(groovyScript);
scriptPath = String.format("configs/%s/pending/%s.groovy", siteId, scriptHash.substring(0, 8));
storageService.store(scriptPath, groovyScript);
}
// 存储 Groovy 脚本
String scriptHash = HmacUtil.sha256(groovyScript);
String scriptPath = String.format("configs/%s/pending/%s.groovy", siteId, scriptHash.substring(0, 8));
storageService.store(scriptPath, groovyScript);
SiteConfig config = new SiteConfig();
config.setSiteId(siteId);
config.setVersion(version);
config.setConfigHash(configHash);
config.setStoragePath(storagePath);
config.setScriptStoragePath(scriptPath);
config.setFileSize((long) configJson.getBytes(StandardCharsets.UTF_8).length);
config.setSchemaVersion(schemaVersion != null ? schemaVersion : "1.0.0");
config.setStatus(ConfigStatus.PENDING);
config.setSubmitterId(submitterId);
config.setIsLatest(false);
siteConfigMapper.insert(config);
log.info("Config submitted: siteId={}, version={}, id={}, hash={}, hasScript={}", siteId, version, config.getId(), configHash, scriptPath != null);
log.info("Config submitted: siteId={}, version={}, id={}, scriptPath={}", siteId, version, config.getId(), scriptPath);
return config;
}
@@ -115,10 +82,7 @@ public class ConfigServiceImpl implements ConfigService {
dto.setId(config.getId());
dto.setSiteId(config.getSiteId());
dto.setVersion(config.getVersion());
dto.setConfigHash(config.getConfigHash());
dto.setScriptStoragePath(config.getScriptStoragePath());
dto.setFileSize(config.getFileSize());
dto.setSchemaVersion(config.getSchemaVersion());
dto.setStatus(config.getStatus());
dto.setIsLatest(config.getIsLatest());
dto.setSubmitterId(config.getSubmitterId());