731 lines
31 KiB
Java
731 lines
31 KiB
Java
import java.io.File;
|
||
import java.io.IOException;
|
||
import java.lang.annotation.ElementType;
|
||
import java.lang.annotation.Retention;
|
||
import java.lang.annotation.RetentionPolicy;
|
||
import java.lang.annotation.Target;
|
||
import java.lang.reflect.Field;
|
||
import java.net.URI;
|
||
import java.net.http.HttpClient;
|
||
import java.net.http.HttpRequest;
|
||
import java.net.http.HttpResponse;
|
||
import java.nio.charset.StandardCharsets;
|
||
import java.nio.file.Files;
|
||
import java.nio.file.Path;
|
||
import java.nio.file.StandardOpenOption;
|
||
import java.time.Duration;
|
||
import java.time.LocalDate;
|
||
import java.time.LocalDateTime;
|
||
import java.time.format.DateTimeFormatter;
|
||
import java.util.*;
|
||
import java.util.concurrent.*;
|
||
import java.util.function.BiConsumer;
|
||
import java.util.function.Function;
|
||
import java.util.regex.Matcher;
|
||
import java.util.regex.Pattern;
|
||
import java.util.stream.Collectors;
|
||
|
||
/**
|
||
* <p>
|
||
* JDK版本必须大于或等于21, 直接运行将生成一份bat脚本或shell脚本, 下载JDK可以在浏览器打开链接按需下载:
|
||
* https://www.azul.com/downloads/?version=java-21-lts&package=jdk#zulu
|
||
* </p>
|
||
*/
|
||
public class LLMBenchmarkTester {
|
||
|
||
public static final String SEP = "=============================================================================================";
|
||
|
||
public static final Field[] PARAM_FIELD = ScriptParameter.class.getDeclaredFields();
|
||
|
||
public static final Pattern CONTENT_PATTERN = Pattern.compile("\"content\"\\s*:\\s*\"([^\"]*)\"");
|
||
|
||
public static void main(String[] args) throws Exception {
|
||
if (args == null || args.length == 0) {
|
||
createRunScript();
|
||
} else {
|
||
ScriptParameter param = readScriptParameter(args);
|
||
printScriptParam(param);
|
||
List<ExecuteContext> executeContexts = new ArrayList<>();
|
||
if (param.isTestChatModel()) {
|
||
TextQuestion textQuestion = new TextQuestion(param);
|
||
List<ExecuteContext> tasks = submit(
|
||
param.modelName.split(","),
|
||
param.threadSize.split(","),
|
||
param,
|
||
textQuestion,
|
||
null
|
||
);
|
||
if (!tasks.isEmpty()) {
|
||
executeContexts.addAll(tasks);
|
||
}
|
||
}
|
||
if (param.isTestVlModel()) {
|
||
ImageQuestion imageQuestion = new ImageQuestion(param);
|
||
List<ExecuteContext> tasks = submit(
|
||
param.vlModelName.split(","),
|
||
param.threadSize.split(","),
|
||
param,
|
||
null,
|
||
imageQuestion
|
||
);
|
||
if (!tasks.isEmpty()) {
|
||
executeContexts.addAll(tasks);
|
||
}
|
||
}
|
||
if (!executeContexts.isEmpty()) {
|
||
String today = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
||
String logName = String.format("llm_bench_%s.log", today);
|
||
Path logPath = Path.of(System.getProperty("user.dir"), logName);
|
||
for (ExecuteContext executeContext : executeContexts) {
|
||
executeContext.latch.await();
|
||
writeLog(logPath, executeContext);
|
||
executeContext.executor.shutdownNow();
|
||
executeContext.sessionMap.clear();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void writeLog(Path logPath, ExecuteContext executeContext) throws IOException {
|
||
Collection<HttpContext> values = executeContext.sessionMap.values();
|
||
double avgResponse = values.stream().mapToLong(HttpContext::toEndMillis).filter(d -> d > 0L).average().orElse(0D);
|
||
long totalTime = values.stream().mapToLong(HttpContext::toFinishMillis).sum();
|
||
long successNum = values.stream().filter(d -> d.success).count();
|
||
long maxResponse = values.stream().mapToLong(HttpContext::toEndMillis).max().orElse(0L);
|
||
int outTextLength = values.stream().mapToInt(d -> d.outTexts.stream().mapToInt(s -> s != null ? s.length() : 0).sum()).sum();
|
||
int outTextCount = values.stream().mapToInt(d -> d.outTexts != null ? d.outTexts.size() : 0).sum();
|
||
String format = """
|
||
------------------------------------------
|
||
模型: %s
|
||
并发量: %d
|
||
问题数量: %d
|
||
成功: %d
|
||
首次响应最长耗时: %d毫秒
|
||
首次响应平均耗时: %f毫秒
|
||
一共输出: %d字, 共输出%d次, 共计耗时:%d毫秒
|
||
------------------------------------------
|
||
%s
|
||
""";
|
||
String msg = String.format(
|
||
format,
|
||
executeContext.model,
|
||
executeContext.threadSize,
|
||
executeContext.sessionMap.size(),
|
||
successNum,
|
||
maxResponse,
|
||
avgResponse,
|
||
outTextLength,
|
||
outTextCount,
|
||
totalTime,
|
||
System.lineSeparator()
|
||
);
|
||
Files.writeString(logPath, msg, StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||
}
|
||
|
||
private static List<ExecuteContext> submit(String[] models, String[] threadSizeStr, ScriptParameter param, TextQuestion textQuestion, ImageQuestion imageQuestion) {
|
||
List<Integer> threadSizeList = Arrays.stream(threadSizeStr).map(s -> Integer.parseInt(s.strip())).toList();
|
||
List<ExecuteContext> executeContexts = new ArrayList<>();
|
||
for (Integer threadSize : threadSizeList) {
|
||
for (String model : models) {
|
||
if (textQuestion != null) {
|
||
executeContexts.add(execute(threadSize, model, param, textQuestion.getRequestParams(model)));
|
||
} else if (imageQuestion != null) {
|
||
executeContexts.add(execute(threadSize, model, param, imageQuestion.getRequestParams(model)));
|
||
}
|
||
}
|
||
}
|
||
return executeContexts;
|
||
}
|
||
|
||
private static ExecuteContext execute(int threadSize, String model, ScriptParameter param, List<String> requestParams) {
|
||
CountDownLatch latch = new CountDownLatch(requestParams.size());
|
||
ConcurrentHashMap<Long, HttpContext> sessionMap = new ConcurrentHashMap<>(requestParams.size());
|
||
ExecutorService executorService = Executors.newFixedThreadPool(threadSize);
|
||
URI uri = URI.create(param.openAiApiHost);
|
||
executorService.execute(() -> {
|
||
for (String requestBody : requestParams) {
|
||
startHttp(uri, param.apiKey, requestBody, latch, sessionMap);
|
||
}
|
||
});
|
||
return new ExecuteContext(model, threadSize, sessionMap, latch, executorService);
|
||
}
|
||
|
||
record ExecuteContext(String model,
|
||
int threadSize,
|
||
ConcurrentHashMap<Long, HttpContext> sessionMap,
|
||
CountDownLatch latch,
|
||
ExecutorService executor) {
|
||
|
||
}
|
||
|
||
private static void startHttp(URI uri, String apiKey, String requestBody, CountDownLatch latch, Map<Long, HttpContext> sessionMap) {
|
||
HttpRequest httpRequest = HttpRequest.newBuilder()
|
||
.uri(uri)
|
||
.header("Content-Type", "application/json")
|
||
.header("Authorization", "Bearer " + apiKey)
|
||
.timeout(Duration.ofSeconds(15))
|
||
.POST(HttpRequest.BodyPublishers.ofString(requestBody))
|
||
.build();
|
||
try (HttpClient client = HttpClient.newHttpClient()) {
|
||
HttpContext context = new HttpContext();
|
||
context.start = LocalDateTime.now();
|
||
context.outTexts = new ArrayList<>();
|
||
Flow.Subscriber<String> subscriber = createResponseFluxHandler(context);
|
||
CompletableFuture<HttpResponse<Void>> future =
|
||
client.sendAsync(httpRequest, HttpResponse.BodyHandlers.fromLineSubscriber(subscriber));
|
||
handleHttpResponseFuture(future, latch, sessionMap, context);
|
||
}
|
||
}
|
||
|
||
private static class HttpContext {
|
||
long sessionId = System.nanoTime();
|
||
LocalDateTime start;
|
||
LocalDateTime end;
|
||
LocalDateTime completed;
|
||
boolean success;
|
||
List<String> outTexts;
|
||
|
||
public long toEndMillis() {
|
||
return this.end != null ? Duration.between(this.start, this.end).toMillis() : 0L;
|
||
}
|
||
|
||
public long toFinishMillis() {
|
||
return this.completed != null ? Duration.between(this.start, this.completed).toMillis() : 0L;
|
||
}
|
||
}
|
||
|
||
private static void handleHttpResponseFuture(CompletableFuture<HttpResponse<Void>> future,
|
||
CountDownLatch latch,
|
||
Map<Long, HttpContext> sessionMap,
|
||
HttpContext context) {
|
||
future.whenComplete((response, exception) -> context.success = false)
|
||
.thenAccept(response -> {
|
||
context.success = response.statusCode() == 200;
|
||
sessionMap.putIfAbsent(context.sessionId, context);
|
||
latch.countDown();
|
||
}).exceptionally(err -> {
|
||
context.success = false;
|
||
sessionMap.putIfAbsent(context.sessionId, context);
|
||
latch.countDown();
|
||
return null;
|
||
});
|
||
}
|
||
|
||
private static Flow.Subscriber<String> createResponseFluxHandler(HttpContext context) {
|
||
return new Flow.Subscriber<>() {
|
||
@Override
|
||
public void onSubscribe(Flow.Subscription subscription) {
|
||
context.end = LocalDateTime.now();
|
||
subscription.request(Long.MAX_VALUE);
|
||
}
|
||
|
||
@Override
|
||
public void onNext(String item) {
|
||
if (item != null && !item.isEmpty()) {
|
||
Matcher matcher = CONTENT_PATTERN.matcher(item);
|
||
String group;
|
||
if (matcher.find() && null != (group = matcher.group(1)) && !group.isEmpty()) {
|
||
context.outTexts.add(group);
|
||
}
|
||
}
|
||
}
|
||
|
||
@Override
|
||
public void onError(Throwable throwable) {
|
||
context.success = false;
|
||
}
|
||
|
||
@Override
|
||
public void onComplete() {
|
||
context.completed = LocalDateTime.now();
|
||
// System.out.println(context.outTexts);
|
||
}
|
||
};
|
||
}
|
||
|
||
private static class ImageQuestion {
|
||
private static final Map<String, List<String>> cache = new ConcurrentHashMap<>();
|
||
private static final String template = """
|
||
{
|
||
"model": "${model}",
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{
|
||
"type": "text",
|
||
"text": "这张图片里有什么？"
|
||
},
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {
|
||
"url": "${imageBase64}"
|
||
}
|
||
}
|
||
]
|
||
}
|
||
],
|
||
"stream": true
|
||
}
|
||
""".strip();
|
||
|
||
List<String> list;
|
||
|
||
public String getImgHead(File file) {
|
||
if (file.getName().endsWith("png")) {
|
||
return "image/png";
|
||
}
|
||
if (file.getName().endsWith("jpg") || file.getName().endsWith("jpeg")) {
|
||
return "image/jpeg";
|
||
}
|
||
return null;
|
||
}
|
||
|
||
public String tryEncodeBase64(File file, Path path) {
|
||
String imgHead = getImgHead(file);
|
||
if (imgHead == null || imgHead.isBlank()) {
|
||
return null;
|
||
}
|
||
try {
|
||
return "data:" + imgHead + ";base64," + Base64.getEncoder().encodeToString(Files.readAllBytes(path));
|
||
} catch (IOException e) {
|
||
throw new RuntimeException(e);
|
||
}
|
||
}
|
||
|
||
public List<String> base64Image(ScriptParameter parameter) throws IOException {
|
||
try (var files = Files.list(Path.of(parameter.vlImgFolder))) {
|
||
return files.map(path -> {
|
||
File file = path.toFile();
|
||
return tryEncodeBase64(file, path);
|
||
}).filter(Objects::nonNull).toList();
|
||
}
|
||
}
|
||
|
||
public ImageQuestion(ScriptParameter parameter) throws IOException {
|
||
List<String> base64List = base64Image(parameter);
|
||
int imgNum = Integer.parseInt(parameter.imgSize);
|
||
this.list = new ArrayList<>(imgNum);
|
||
do {
|
||
for (String item : base64List) {
|
||
this.list.add(item);
|
||
if (this.list.size() == imgNum) {
|
||
break;
|
||
}
|
||
}
|
||
} while (this.list.size() < imgNum);
|
||
}
|
||
|
||
public List<String> getRequestParams(String model) {
|
||
List<String> cacheParams = cache.get(model);
|
||
if (cacheParams != null && !cacheParams.isEmpty()) {
|
||
return cacheParams;
|
||
}
|
||
List<String> params = this.list.stream().map(s -> this.toJsonParam(model, s)).toList();
|
||
cache.put(model, params);
|
||
return params;
|
||
}
|
||
|
||
public String toJsonParam(String model, String imageBase64) {
|
||
return template.replace("${model}", model).replace("${imageBase64}", imageBase64);
|
||
}
|
||
}
|
||
|
||
private static class TextQuestion {
|
||
private static final Map<String, List<String>> cache = new ConcurrentHashMap<>();
|
||
private static final String template = """
|
||
{
|
||
"model": "${model}",
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": "${prompt}"
|
||
}
|
||
],
|
||
"stream": true
|
||
}
|
||
""".strip();
|
||
// 解析文件得到的问题列表
|
||
List<String> list;
|
||
|
||
public TextQuestion(ScriptParameter parameter) throws IOException {
|
||
this.list = Files.readAllLines(Path.of(parameter.chatDatasetsPath));
|
||
}
|
||
|
||
public List<String> getRequestParams(String model) {
|
||
List<String> cacheParams = cache.get(model);
|
||
if (cacheParams != null && cacheParams.isEmpty()) {
|
||
return cacheParams;
|
||
}
|
||
List<String> params = this.list.stream().map(s -> this.toJsonParam(model, s)).toList();
|
||
cache.put(model, params);
|
||
return params;
|
||
}
|
||
|
||
public String toJsonParam(String model, String prompt) {
|
||
return template.replace("${model}", model).replace("${prompt}", prompt);
|
||
}
|
||
}
|
||
|
||
private static void printScriptParam(ScriptParameter param) throws IllegalAccessException {
|
||
System.out.println("本次执行脚本的参数如下:");
|
||
for (Field field : PARAM_FIELD) {
|
||
if (field.isAnnotationPresent(EnvName.class)) {
|
||
System.out.println(SEP);
|
||
EnvName annotation = field.getAnnotation(EnvName.class);
|
||
field.setAccessible(true);
|
||
Object value = field.get(param);
|
||
System.out.printf("参数: %s 数值: %s%n", annotation.value(), value);
|
||
}
|
||
}
|
||
System.out.println(SEP);
|
||
}
|
||
|
||
private static void createRunScript() throws IOException {
|
||
String osName = System.getProperty("os.name").toLowerCase();
|
||
if (osName.contains("win")) {
|
||
generateWindowsBat();
|
||
} else {
|
||
generateShellScript();
|
||
}
|
||
}
|
||
|
||
private static File createScripeFile(String extName) {
|
||
String date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd"));
|
||
File file = new File(String.format("llm_benchmark_tester_%s.%s", date, extName));
|
||
if (file.exists()) {
|
||
throw new RuntimeException(String.format("您可以通过 %s 脚本直接运行", file.getAbsolutePath()));
|
||
}
|
||
boolean createBat;
|
||
try {
|
||
createBat = file.createNewFile();
|
||
} catch (Exception e) {
|
||
throw new RuntimeException(String.format("创建 %s 脚本文件异常: %s", file.getAbsolutePath(), e.getMessage()), e);
|
||
}
|
||
if (!createBat) {
|
||
throw new RuntimeException(String.format("创建 %s 脚本文件失败", file.getAbsolutePath()));
|
||
}
|
||
System.out.println("已为你生成一份脚本, 请修改脚本中的环境变量, 使用脚本运行");
|
||
System.out.printf("脚本的存储路径 %s%n", file.getAbsolutePath());
|
||
System.out.println("运行脚本之前, 请确保脚本文件的换行符与系统相匹配, 否则会无法运行");
|
||
return file;
|
||
}
|
||
|
||
private static void writeScriptFile(File file, String template, BiConsumer<EnvName, List<String>> eachFunc) throws IOException {
|
||
List<String> envLines = new ArrayList<>();
|
||
for (Field field : PARAM_FIELD) {
|
||
if (field.isAnnotationPresent(EnvName.class)) {
|
||
EnvName annotation = field.getAnnotation(EnvName.class);
|
||
eachFunc.accept(annotation, envLines);
|
||
}
|
||
}
|
||
if (!envLines.isEmpty()) {
|
||
String envList = envLines.stream().collect(Collectors.joining(System.lineSeparator()));
|
||
String script = template.replace("${ENV_LINES}", envList);
|
||
Files.writeString(file.toPath(), script.strip(), StandardCharsets.UTF_8);
|
||
}
|
||
}
|
||
|
||
private static void generateWindowsBat() throws IOException {
|
||
File file = createScripeFile("bat");
|
||
String batTemplate = """
|
||
@echo off
|
||
|
||
:: java可执行文件路径, 不是JAVA_HOME, 是完整的java可执行文件路径, 例如: D:\\jdk-2108\\bin\\java
|
||
set JAVA_BIN=
|
||
|
||
:: 脚本存放路径, 例如: E:\\JExample\\src\\LLMBenchmarkTester.java
|
||
set SCRIPT_PATH=
|
||
|
||
${ENV_LINES}
|
||
|
||
:: 基于环境变量的方式执行, 交互式命令行执行用这个命令: %JAVA_BIN% %SCRIPT_PATH% -p input
|
||
%JAVA_BIN% %SCRIPT_PATH% -p env
|
||
|
||
pause
|
||
""";
|
||
writeScriptFile(file, batTemplate, (envName, envLines) -> {
|
||
envLines.add(":: " + envName.desc());
|
||
envLines.add("set " + envName.value() + "=");
|
||
});
|
||
}
|
||
|
||
private static void generateShellScript() throws IOException {
|
||
File file = createScripeFile("sh");
|
||
String bashTemplate = """
|
||
#!/bin/bash
|
||
# java可执行文件路径, 不是JAVA_HOME, 是完整的java可执行文件路径, 例如: /opt/jdk-2108/bin/java
|
||
JAVA_BIN=""
|
||
|
||
# 脚本存放路径, 例如: /home/user/JExample/src/LLMBenchmarkTester.java
|
||
SCRIPT_PATH=""
|
||
|
||
${ENV_LINES}
|
||
|
||
# 基于环境变量的方式执行, 交互式命令行执行用这个命令: $JAVA_BIN $SCRIPT_PATH -p input
|
||
"$JAVA_BIN" "$SCRIPT_PATH" -p env
|
||
""";
|
||
writeScriptFile(file, bashTemplate, (envName, envLines) -> {
|
||
envLines.add("# " + envName.desc());
|
||
envLines.add(envName.value() + "=");
|
||
});
|
||
}
|
||
|
||
private static ScriptParameter readScriptParameter(String[] args) throws IllegalAccessException {
|
||
if (args != null && args.length > 0) {
|
||
boolean p = Arrays.stream(args).anyMatch(s -> s.equalsIgnoreCase("-p"));
|
||
if (p && Arrays.stream(args).anyMatch(s -> s.equalsIgnoreCase("env"))) {
|
||
return initScriptParamFromEnv();
|
||
}
|
||
if (p && Arrays.stream(args).anyMatch(s -> s.equalsIgnoreCase("input"))) {
|
||
return initScriptParamFromAsk();
|
||
}
|
||
}
|
||
throw new RuntimeException("命令错误, 请检查参数是否正确");
|
||
}
|
||
|
||
private static ScriptParameter initScriptParamFromEnv() throws IllegalAccessException {
|
||
ScriptParameter param = new ScriptParameter();
|
||
param.channel = 1;
|
||
for (Field field : PARAM_FIELD) {
|
||
if (field.isAnnotationPresent(EnvName.class)) {
|
||
EnvName envName = field.getAnnotation(EnvName.class);
|
||
String fieldValue = System.getenv(envName.value());
|
||
String formatValue = formatValue(fieldValue, field);
|
||
if (field.isAnnotationPresent(NotBlank.class) && (formatValue == null || formatValue.isBlank())) {
|
||
throw new RuntimeException(String.format("环境变量[%s]不能为空或空白字符", envName.value()));
|
||
}
|
||
if (!isValidValue(formatValue, field)) {
|
||
throw new RuntimeException(String.format("环境变量[%s]数值不合法, 当前值:[%s]", envName.value(), formatValue));
|
||
}
|
||
field.setAccessible(true);
|
||
field.set(param, fieldValue);
|
||
}
|
||
}
|
||
return param;
|
||
}
|
||
|
||
private static ScriptParameter initScriptParamFromAsk() throws IllegalAccessException {
|
||
ScriptParameter param = new ScriptParameter();
|
||
param.channel = 2;
|
||
Scanner scanner = new Scanner(System.in);
|
||
for (Field field : PARAM_FIELD) {
|
||
if (field.isAnnotationPresent(AskUser.class)) {
|
||
AskUser askUser = field.getAnnotation(AskUser.class);
|
||
System.out.println(askUser.value() + ":");
|
||
boolean isNotBlank = field.isAnnotationPresent(NotBlank.class);
|
||
for (; ; ) {
|
||
String userInput = scanner.nextLine().trim();
|
||
String formatValue = formatValue(userInput, field);
|
||
// 允许为空并且输入值为空
|
||
if (!isNotBlank && (formatValue == null || formatValue.isBlank())) {
|
||
break;
|
||
}
|
||
// 非空并且输入值合法
|
||
if (isNotBlank && formatValue != null && !formatValue.isBlank() && isValidValue(formatValue, field)) {
|
||
field.setAccessible(true);
|
||
field.set(param, formatValue);
|
||
break;
|
||
}
|
||
System.out.print("请重新输入:");
|
||
}
|
||
}
|
||
}
|
||
return param;
|
||
}
|
||
|
||
// 顺序校验
|
||
private static Boolean isValidValue(String formatValue, Field field) {
|
||
if (field.isAnnotationPresent(Validator.class)) {
|
||
Validator anno = field.getAnnotation(Validator.class);
|
||
return Arrays.stream(anno.value())
|
||
.map(validator -> Constants.TEXT_VALIDATOR.get(validator.name()).apply(formatValue))
|
||
.allMatch(Boolean.TRUE::equals);
|
||
}
|
||
return Boolean.TRUE;
|
||
}
|
||
|
||
// 顺序格式化
|
||
private static String formatValue(String fieldValue, Field field) {
|
||
if (field.isAnnotationPresent(Formatter.class)) {
|
||
Formatter anno = field.getAnnotation(Formatter.class);
|
||
for (TextFormater fmt : anno.value()) {
|
||
fieldValue = Constants.TEXT_FORMATER.get(fmt.name()).apply(fieldValue);
|
||
}
|
||
}
|
||
return fieldValue;
|
||
}
|
||
|
||
@Retention(RetentionPolicy.RUNTIME)
|
||
@Target({ElementType.FIELD})
|
||
public @interface AskUser {
|
||
String value();
|
||
}
|
||
|
||
@Retention(RetentionPolicy.RUNTIME)
|
||
@Target({ElementType.FIELD})
|
||
public @interface EnvName {
|
||
String value() default "";
|
||
|
||
String desc();
|
||
}
|
||
|
||
@Retention(RetentionPolicy.RUNTIME)
|
||
@Target({ElementType.FIELD})
|
||
public @interface NotBlank {
|
||
|
||
}
|
||
|
||
@Retention(RetentionPolicy.RUNTIME)
|
||
@Target({ElementType.FIELD})
|
||
public @interface Validator {
|
||
TextValidator[] value();
|
||
}
|
||
|
||
@Retention(RetentionPolicy.RUNTIME)
|
||
@Target({ElementType.FIELD})
|
||
public @interface Formatter {
|
||
TextFormater[] value();
|
||
}
|
||
|
||
public enum TextValidator {
|
||
MUST_URL, MUST_FOLDER, MUST_TXT, MUST_NUM;
|
||
}
|
||
|
||
public enum TextFormater {
|
||
STRIP, COMMA_CN_2_EN;
|
||
}
|
||
|
||
private static class Constants {
|
||
// 去除字符串两端空白字符和制表符
|
||
public static final Function<String, String> STRIP_FORMATTER =
|
||
str -> Optional.ofNullable(str).map(java.lang.String::strip).orElse("");
|
||
|
||
// 中文逗号替换成英文逗号
|
||
public static final Function<String, String> COMMA_CN_2_EN =
|
||
str -> Optional.ofNullable(str).map(d -> d.replaceAll("，", ",")).orElse("");
|
||
|
||
// 字符串必须是一个http链接
|
||
public static final Function<String, Boolean> URL_VALIDATOR =
|
||
str -> str != null && (str.startsWith("http://") || str.startsWith("https://"));
|
||
|
||
// 字符串必须是一个合法的文件路径且已存在的文件夹
|
||
public static final Function<String, Boolean> FOLDER_VALIDATOR = str -> {
|
||
if (str != null && !str.isBlank()) {
|
||
try {
|
||
File file = new File(str);
|
||
return file.exists() && file.isDirectory();
|
||
} catch (Exception e) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
};
|
||
|
||
// 字符串必须是一个合法的文件路径且已存在的txt文件
|
||
public static final Function<String, Boolean> TXT_FILE_VALIDATOR = str -> {
|
||
if (str != null && !str.isBlank()) {
|
||
try {
|
||
File file = new File(str);
|
||
return file.exists() && file.isFile() && file.getName().endsWith(".txt");
|
||
} catch (Exception e) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
};
|
||
|
||
// 字符串必须是一个整数
|
||
public static final Function<String, Boolean> NUMBER_VALIDATOR = str -> {
|
||
if (str != null && !str.isBlank()) {
|
||
try {
|
||
Integer.parseInt(str);
|
||
} catch (Exception e) {
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
};
|
||
|
||
// 文本格式化工具注册表
|
||
public static final Map<String, Function<String, String>> TEXT_FORMATER =
|
||
Map.of(
|
||
TextFormater.STRIP.name(), Constants.STRIP_FORMATTER,
|
||
TextFormater.COMMA_CN_2_EN.name(), Constants.COMMA_CN_2_EN
|
||
);
|
||
|
||
// 文本验证工具注册表
|
||
public static final Map<String, Function<String, Boolean>> TEXT_VALIDATOR =
|
||
Map.of(
|
||
TextValidator.MUST_URL.name(), URL_VALIDATOR,
|
||
TextValidator.MUST_FOLDER.name(), FOLDER_VALIDATOR,
|
||
TextValidator.MUST_TXT.name(), TXT_FILE_VALIDATOR,
|
||
TextValidator.MUST_NUM.name(), NUMBER_VALIDATOR
|
||
);
|
||
}
|
||
|
||
public static class ScriptParameter {
|
||
|
||
// 1=环境变量, 2=交互式命令行
|
||
int channel;
|
||
|
||
@NotBlank
|
||
@EnvName(value = "BENCH_LLM_API_HOST", desc = "OpenAI API 的访问地址, 例如: http://localhost:8080/v1/chat/completions")
|
||
@Validator(value = TextValidator.MUST_URL)
|
||
@Formatter(value = TextFormater.STRIP)
|
||
@AskUser(value = "请输入 OpenAI API 的访问地址 (例如: http://localhost:8080/v1/chat/completions)")
|
||
String openAiApiHost;
|
||
|
||
@NotBlank
|
||
@EnvName(value = "BENCH_LLM_API_KEY", desc = "ApiKey或者叫API令牌")
|
||
@Formatter(value = TextFormater.STRIP)
|
||
@AskUser(value = "请输入ApiKey或者叫API令牌")
|
||
String apiKey;
|
||
|
||
@NotBlank
|
||
@EnvName(value = "BENCH_THREAD_SIZE_ARRAY", desc = "请输入线程池配置, 示例值: 10,50,100")
|
||
@Formatter(value = {TextFormater.STRIP, TextFormater.COMMA_CN_2_EN})
|
||
@AskUser(value = "请输入线程池配置 (示例值: 10,50,100)")
|
||
String threadSize;
|
||
|
||
@EnvName(value = "BENCH_LLM_MODEL_NAME", desc = "文本模型名称, 多个使用英文逗号隔开, 如果不测试文生文模型可以不设置, 示例值: qwen2.5,qwen3")
|
||
@Formatter(value = {TextFormater.STRIP, TextFormater.COMMA_CN_2_EN})
|
||
@AskUser(value = "请输入文本模型名称, 多个使用英文逗号隔开, 如果不测试文生文模型可以直接回车 (示例值: qwen2.5,qwen3)")
|
||
String modelName;
|
||
|
||
@EnvName(value = "BENCH_LLM_VL_MODEL_NAME", desc = "VL模型名称, 多个用英文逗号隔开, 如果不测试VL模型可以不设置")
|
||
@Formatter(value = {TextFormater.STRIP, TextFormater.COMMA_CN_2_EN})
|
||
@AskUser(value = "请输入VL模型名称, 多个用英文逗号隔开, 如果不测试VL模型可以直接回车")
|
||
String vlModelName;
|
||
|
||
@EnvName(value = "BENCH_LLM_VL_IMG_FOLDER", desc = "调用VL模型的图片存储目录, 如果不测试VL模型可以不设置, 示例值: /home/image")
|
||
@Validator(value = TextValidator.MUST_FOLDER)
|
||
@Formatter(value = {TextFormater.STRIP})
|
||
@AskUser(value = "请输入调用VL模型的图片存储目录, 如果不测试VL模型可以直接回车 (示例值: /home/image)")
|
||
String vlImgFolder;
|
||
|
||
@EnvName(value = "BENCH_LLM_CHAT_MODEL_DATASETS", desc = "文生文测试数据集的文件路径, 如果不测试文生文模型可以不设置, 必须是一个.txt文件 (示例值: /home/datasets.txt)")
|
||
@Validator(value = TextValidator.MUST_TXT)
|
||
@AskUser(value = "请输入文生文测试数据集的文件路径, 必须是一个.txt文件, 如果不测试文生文模型可以直接回车 (示例值: /home/datasets.txt)")
|
||
String chatDatasetsPath;
|
||
|
||
@EnvName(value = "BENCH_LLM_VL_IMG_SIZE", desc = "调用VL模型的测试图片数量, 如果文件夹下的图片数量不够, 会复制直到到足够数量, 如果不测试VL模型可以不设置 (示例值: 300)")
|
||
@Validator(value = TextValidator.MUST_NUM)
|
||
@Formatter(value = TextFormater.STRIP)
|
||
@AskUser("请输入调用VL模型的测试图片数量, 如果文件夹下的图片数量不够, 会复制直到到足够数量, 如果不测试VL模型可以直接回车 (示例值: 300)")
|
||
String imgSize;
|
||
|
||
public boolean isTestChatModel() {
|
||
return this.modelName != null && !this.modelName.isBlank()
|
||
&& this.chatDatasetsPath != null && !this.chatDatasetsPath.isBlank();
|
||
}
|
||
|
||
public boolean isTestVlModel() {
|
||
return this.vlModelName != null && !this.vlModelName.isBlank()
|
||
&& this.vlImgFolder != null && !this.vlImgFolder.isBlank()
|
||
&& this.imgSize != null && !this.imgSize.isBlank();
|
||
}
|
||
}
|
||
|
||
}
|