yescent/par
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: access-result提交5分钟内容去重，防止mediabot重复提交

Browse Source
This commit is contained in:
mediabot-pt
2026-07-01 17:09:38 +08:00
parent ebb7aa7a2d
commit 2972ebab7d
3 changed files with 37 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
par-core/src/main/java/com/par/core/exception/DuplicateSubmissionException.java Normal file
View File

@@ -0,0 +1,7 @@
package com.par.core.exception;
/**
* 重复提交异常：相同内容的访问结果在去重窗口内再次提交
*/
public class DuplicateSubmissionException extends RuntimeException {
}

22
par-core/src/main/java/com/par/core/service/impl/AccessResultServiceImpl.java
View File

@@ -11,6 +11,8 @@ import com.par.core.mapper.AccessResultMapper;
import com.par.core.mapper.SiteConfigMapper;
import com.par.core.mapper.SiteMapper;
import com.par.core.service.AccessResultService;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.par.core.storage.CachedStorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -19,6 +21,7 @@ import org.springframework.stereotype.Service;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Slf4j
@@ -31,12 +34,31 @@ public class AccessResultServiceImpl implements AccessResultService {
private final SiteMapper siteMapper;
private final CachedStorageService storageService;
/** 去重缓存：siteId|url → contentHash，5分钟内相同内容跳过 */
private final Cache<String, String> dedupCache = Caffeine.newBuilder()
.maximumSize(10000)
.expireAfterWrite(5, TimeUnit.MINUTES)
.build();
@Override
public AccessResult submit(String siteId, String configVersion, String clientId,
String contentType, String content, String url,
Integer statusCode, Integer responseTimeMs, Long submitterId) {
String type = (contentType != null && !contentType.isBlank()) ? contentType : "html";
String ext = "json".equalsIgnoreCase(type) ? "json" : "html";
// 去重：5分钟内相同site+url+contentHash跳过
String dedupKey = siteId + "|" + (url != null ? url : "");
String contentHash = content.length() > 1000
? Integer.toHexString(content.substring(0, 1000).hashCode())
: Integer.toHexString(content.hashCode());
String existing = dedupCache.getIfPresent(dedupKey);
if (contentHash.equals(existing)) {
log.debug("跳过重复提交: siteId={}, url={}", siteId, url);
throw new DuplicateSubmissionException();
}
dedupCache.put(dedupKey, contentHash);
String storagePath = String.format("access-results/%s/%d_%s.%s",
siteId, System.currentTimeMillis(), UUID.randomUUID().toString().substring(0, 8), ext);