feat: access-result提交5分钟内容去重,防止mediabot重复提交

This commit is contained in:
mediabot-pt
2026-07-01 17:09:38 +08:00
parent ebb7aa7a2d
commit 2972ebab7d
3 changed files with 37 additions and 5 deletions

View File

@@ -54,11 +54,14 @@ public class AccessResultController {
String clientId = (String) request.getAttribute("clientId"); String clientId = (String) request.getAttribute("clientId");
Long submitterId = (Long) request.getAttribute("accountId"); Long submitterId = (Long) request.getAttribute("accountId");
AccessResult result = accessResultService.submit( try {
siteId, configVersion, clientId, contentType, content, AccessResult result = accessResultService.submit(
url, statusCode, responseTimeMs, submitterId); siteId, configVersion, clientId, contentType, content,
url, statusCode, responseTimeMs, submitterId);
return ApiResponse.success(accessResultService.toDTO(result)); return ApiResponse.success(accessResultService.toDTO(result));
} catch (com.par.core.exception.DuplicateSubmissionException e) {
return ApiResponse.success(Map.of("skipped", true, "reason", "duplicate"));
}
} }
// ==================== 管理员查看接口(JWT 鉴权) ==================== // ==================== 管理员查看接口(JWT 鉴权) ====================

View File

@@ -0,0 +1,7 @@
package com.par.core.exception;
/**
* 重复提交异常:相同内容的访问结果在去重窗口内再次提交
*/
public class DuplicateSubmissionException extends RuntimeException {
}

View File

@@ -11,6 +11,8 @@ import com.par.core.mapper.AccessResultMapper;
import com.par.core.mapper.SiteConfigMapper; import com.par.core.mapper.SiteConfigMapper;
import com.par.core.mapper.SiteMapper; import com.par.core.mapper.SiteMapper;
import com.par.core.service.AccessResultService; import com.par.core.service.AccessResultService;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.par.core.storage.CachedStorageService; import com.par.core.storage.CachedStorageService;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -19,6 +21,7 @@ import org.springframework.stereotype.Service;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.*; import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Slf4j @Slf4j
@@ -31,12 +34,31 @@ public class AccessResultServiceImpl implements AccessResultService {
private final SiteMapper siteMapper; private final SiteMapper siteMapper;
private final CachedStorageService storageService; private final CachedStorageService storageService;
/** 去重缓存:siteId|url → contentHash,5分钟内相同内容跳过 */
private final Cache<String, String> dedupCache = Caffeine.newBuilder()
.maximumSize(10000)
.expireAfterWrite(5, TimeUnit.MINUTES)
.build();
@Override @Override
public AccessResult submit(String siteId, String configVersion, String clientId, public AccessResult submit(String siteId, String configVersion, String clientId,
String contentType, String content, String url, String contentType, String content, String url,
Integer statusCode, Integer responseTimeMs, Long submitterId) { Integer statusCode, Integer responseTimeMs, Long submitterId) {
String type = (contentType != null && !contentType.isBlank()) ? contentType : "html"; String type = (contentType != null && !contentType.isBlank()) ? contentType : "html";
String ext = "json".equalsIgnoreCase(type) ? "json" : "html"; String ext = "json".equalsIgnoreCase(type) ? "json" : "html";
// 去重:5分钟内相同site+url+contentHash跳过
String dedupKey = siteId + "|" + (url != null ? url : "");
String contentHash = content.length() > 1000
? Integer.toHexString(content.substring(0, 1000).hashCode())
: Integer.toHexString(content.hashCode());
String existing = dedupCache.getIfPresent(dedupKey);
if (contentHash.equals(existing)) {
log.debug("跳过重复提交: siteId={}, url={}", siteId, url);
throw new DuplicateSubmissionException();
}
dedupCache.put(dedupKey, contentHash);
String storagePath = String.format("access-results/%s/%d_%s.%s", String storagePath = String.format("access-results/%s/%d_%s.%s",
siteId, System.currentTimeMillis(), UUID.randomUUID().toString().substring(0, 8), ext); siteId, System.currentTimeMillis(), UUID.randomUUID().toString().substring(0, 8), ext);