Commit 4ac6af4f authored by yutao's avatar yutao

init

parent ba9495c1
......@@ -96,6 +96,10 @@
<artifactId>javax.servlet.jsp</artifactId>
<groupId>org.glassfish.web</groupId>
</exclusion>
<exclusion>
<artifactId>junit</artifactId>
<groupId>junit</groupId>
</exclusion>
</exclusions>
......@@ -141,5 +145,12 @@
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.jsqlparser</groupId>
<artifactId>jsqlparser</artifactId>
<version>5.3</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
package com.ediagnosis.cdr.util;
package com.ediagnosis.cdr.context;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.util.Optional;
......
package com.ediagnosis.cdr.util;
package com.ediagnosis.cdr.context;
import org.slf4j.Logger;
......
......@@ -3,7 +3,7 @@ package com.ediagnosis.cdr.dao;
import com.ediagnosis.cdr.dao.entity.DimDataSummaryEntity;
import com.ediagnosis.cdr.dao.mapper.DimDataSummaryMapper;
import com.ediagnosis.cdr.domain.DimDataSummary;
import com.ediagnosis.cdr.util.TimeUtil;
import com.ediagnosis.cdr.context.TimeUtil;
import org.springframework.stereotype.Repository;
import java.time.LocalDateTime;
......
package com.ediagnosis.cdr.dao;
import com.ediagnosis.cdr.value.Page;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Repository;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Repository
public class DwsRepository {
private final HiveQueryExecutor hiveQueryExecutor;
public DwsRepository(HiveQueryExecutor hiveQueryExecutor) {
this.hiveQueryExecutor = hiveQueryExecutor;
}
public Page<Map<String, String>> queryPage(String sql, int pageNo,int pageSize) {
com.mybatisflex.core.paginate.Page<Row> rowPage = hiveQueryExecutor.selectByFullSql(sql, pageNo, pageSize);
List<Map<String, String>> result = rowPage.getRecords().stream()
.map(row -> {
Map<String, Object> camelKeysMap = row.toCamelKeysMap();
Map<String, String> flatMap = new HashMap<>();
camelKeysMap.forEach((table, value) -> {
if (value instanceof Map<?, ?> m) {
m.forEach((key, val) -> {
if (key instanceof String field && val instanceof String fieldValue) {
flatMap.put(table + "." + field, fieldValue);
}
});
}
});
return flatMap;
})
.filter(map -> !map.isEmpty()) // 可选:过滤掉空 map
.toList();
return new Page<>(pageNo, pageSize, rowPage.getTotalRow(), result);
}
}
package com.ediagnosis.cdr.dao;
import com.ediagnosis.cdr.dao.sqlutil.SqlParser;
import com.mybatisflex.core.datasource.DataSourceKey;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Optional;
@Component
public class HiveQueryExecutor {
public Page<Row> selectByFullSql(String fullSql, int pageNo, int pageSize) {
Page<Row> page = new Page<>(pageNo, pageSize);
Optional<String> countSql = SqlParser.getCountSql(fullSql);
String limitSql = SqlParser.addLimitClause(fullSql);
try {
DataSourceKey.use("ds-hive");
if(countSql.isPresent()){
long count = Db.selectCount(countSql.get());
page.setTotalRow(count);
}
List<Row> rows = Db.selectListBySql(limitSql);
page.setRecords(rows);
} finally {
DataSourceKey.clear();
}
return page;
}
}
package com.ediagnosis.cdr.dao.sqlutil;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Optional;
public class SqlParser {
private static final Logger log = LoggerFactory.getLogger(SqlParser.class);
public static Optional<String> getCountSql(String sql) {
Optional<PlainSelect> plainSelectOptional = detectSelectSql(sql);
if (plainSelectOptional.isEmpty()) {
return Optional.empty();
}
PlainSelect plainSelect = plainSelectOptional.get();
List<SelectItem<?>> selectItems = plainSelect.getSelectItems();
selectItems.clear();
selectItems.add(new SelectItem<>(
new Function("count", new AllColumns())
)
);
String newSql = plainSelect.toString();
log.info("change to count sql: {}", newSql);
return Optional.of(newSql);
}
public static String addLimitClause(String sql) {
String newSql = sql;
Optional<PlainSelect> plainSelectOptional = detectSelectSql(sql);
if (plainSelectOptional.isEmpty()) {
return newSql;
}
PlainSelect plainSelect = plainSelectOptional.get();
Limit limit = plainSelect.getLimit();
if (limit == null) {
Select select = plainSelect.withLimit(
new Limit()
.withRowCount(new LongValue(10))
.withOffset(new LongValue(0))
);
newSql = select.toString();
log.info("原始sql没有limit从句,添加limit从句,新sql语句:{}", newSql);
} else {
log.info("原始sql存在limit从句,不做处理");
}
return newSql;
}
private static Optional<PlainSelect> detectSelectSql(String sql) {
Statement statement = null;
try {
statement = CCJSqlParserUtil.parse(sql);
log.debug("sql语句语法正确");
} catch (JSQLParserException e) {
log.error("解析sql失败", e);
return Optional.empty();
}
if ((statement instanceof PlainSelect plainSelect)) {
log.debug("原始sql是select语句");
return Optional.of(plainSelect);
} else {
log.warn("原始sql不是select语句");
return Optional.empty();
}
}
}
package com.ediagnosis.cdr.service;
import com.ediagnosis.cdr.vo.Page;
public class HiveService {
public Page<?> selectPage(String sql, int pageNo, int pageSize) {
return new Page<>();
}
}
package com.ediagnosis.cdr.vo;
package com.ediagnosis.cdr.value;
import java.util.List;
public class Page<T> {
private int pageNo;
private int pageSize;
private int totalCount;
private long pageNo;
private long pageSize;
private long totalCount;
private List<T> content;
private int totalPage;
private long totalPage;
public Page() {
}
public Page(int pageNo, int pageSize,
int totalCount, List<T> content) {
public Page(long pageNo, long pageSize,
long totalCount, List<T> content) {
this.pageNo = pageNo;
this.pageSize = pageSize;
this.totalCount = totalCount;
this.content = content;
int tmp=totalCount / pageSize;
int tmp1=totalCount % pageSize;
long tmp=totalCount / pageSize;
long tmp1=totalCount % pageSize;
if (tmp1>0) {
this.totalPage = tmp+1;
}else {
......@@ -29,22 +29,22 @@ public class Page<T> {
}
public int getPageNo() {
public long getPageNo() {
return pageNo;
}
public int getPageSize() {
public long getPageSize() {
return pageSize;
}
public int getTotalCount() {
public long getTotalCount() {
return totalCount;
}
public int getTotalPage() {
public long getTotalPage() {
return totalPage;
}
......@@ -53,15 +53,15 @@ public class Page<T> {
return content;
}
public void setPageNo(int pageNo) {
public void setPageNo(long pageNo) {
this.pageNo = pageNo;
}
public void setPageSize(int pageSize) {
public void setPageSize(long pageSize) {
this.pageSize = pageSize;
}
public void setTotalCount(int totalCount) {
public void setTotalCount(long totalCount) {
this.totalCount = totalCount;
}
......@@ -69,7 +69,7 @@ public class Page<T> {
this.content = content;
}
public void setTotalPage(int totalPage) {
public void setTotalPage(long totalPage) {
this.totalPage = totalPage;
}
......
package com.ediagnosis.cdr.value;
import java.io.Serializable;
public class Response<T> implements Serializable {
private boolean status;
private String msg;
private T content;
public Response(boolean status, String msg) {
this.status = status;
this.msg = msg;
}
public Response(boolean status, String msg, T content) {
this.status = status;
this.msg = msg;
this.content = content;
}
public static <T> Response<T> success(T content) {
return new Response<>(true, "", content);
}
public static <T> Response<T> success(String msg, T content) {
return new Response<>(true, msg, content);
}
public static <T> Response<T> success(String msg) {
return new Response<>(true, msg, null);
}
public static <T> Response<T> error(String msg, T defaultContent) {
return new Response<>(false, msg, defaultContent);
}
public static <T> Response<T> error(String msg) {
return new Response<>(false, msg, null);
}
public boolean isStatus() {
return status;
}
public void setStatus(boolean status) {
this.status = status;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public T getContent() {
return content;
}
public void setContent(T content) {
this.content = content;
}
}
mybatis-flex:
datasource:
ds-mysql:
......@@ -6,4 +7,6 @@ mybatis-flex:
password: Admin123456#
ds-hive:
url: jdbc:hive2://10.11.4.21:10000
username: root
\ No newline at end of file
username: root
server:
port: 7001
\ No newline at end of file
package com.ediagnosis.cdr.context;
public class GraphTest {
}
package com.ediagnosis.cdr.util;
package com.ediagnosis.cdr.context;
import org.junit.jupiter.api.Test;
......
package com.ediagnosis.cdr.context;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.List;
import static org.mockito.Mockito.*;
@SpringBootTest
public class MockTest {
@Test
public void testBasicVerification() {
List<String> mockList = mock(List.class);
// 模拟调用
mockList.add("apple");
mockList.add("banana");
mockList.add("apple");
// 1. 验证方法被调用【恰好一次】(默认行为)
verify(mockList).add("banana");
// 2. 验证方法被调用【指定次数】
verify(mockList, times(2)).add("apple"); // 精确2次
// 3. 验证方法【从未调用】
verify(mockList, never()).clear();
// 4. 验证【调用顺序】
InOrder inOrder = inOrder(mockList);
inOrder.verify(mockList).add("apple");
inOrder.verify(mockList).add("banana");
inOrder.verify(mockList).add("apple");
verifyNoMoreInteractions(mockList);
}
}
package com.ediagnosis.cdr.util;
package com.ediagnosis.cdr.context;
import java.util.Objects;
......
package com.ediagnosis.cdr.util;
package com.ediagnosis.cdr.context;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
......
package com.ediagnosis.cdr.dao;
import com.mybatisflex.core.paginate.Page;
import com.mybatisflex.core.row.Row;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.util.Assert;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ExtendWith(SpringExtension.class)
@SpringBootTest
public class DwsRepositoryTest {
@Autowired
HiveQueryExecutor hiveQueryExecutor;
@Test
public void queryPageWithSql() {
String sql = """
select p.patient_id,p.patient_name,r.patient_id,r.visit_no from ods.xjd_patient_info p, ods.xjd_emergency_record r
where p.patient_id=r.patient_id and p.id='3';
""";
Page<Row> rowPage = hiveQueryExecutor.selectByFullSql(sql, 1, 10);
System.out.println(rowPage);
Assert.isTrue(rowPage.getRecords().size() <= 10, "数据量不符合");
List<Map<String, String>> result = rowPage.getRecords().stream()
.map(row -> {
Map<String, Object> camelKeysMap = row.toCamelKeysMap();
Map<String, String> flatMap = new HashMap<>();
camelKeysMap.forEach((table, value) -> {
if (value instanceof Map<?, ?> m) {
m.forEach((key, val) -> {
if (key instanceof String field && val instanceof String fieldValue) {
flatMap.put(table + "." + field, fieldValue);
}
});
}
});
return flatMap;
})
.filter(map -> !map.isEmpty()) // 可选:过滤掉空 map
.toList();
System.out.println(result);
}
}
package com.ediagnosis.cdr.dao;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
public class StringTest {
@Test
public void test() {
String sql = "字符串";
changString(sql);
System.out.println(sql);
}
private void changString(String sql) {
sql+= ";changString();";
}
private String returnString(String sql) {
return sql+= ";returnString();";
}
private String returnNewString(String sql) {
return sql + ";returnNewString();";
}
private String returnNewString2(String sql) {
String newString = new String(sql);
return newString + ";returnNewString2();";
}
}
package com.ediagnosis.cdr.dashBoard;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
@SpringBootTest
public class AsyncTest {
@Test
public void test() throws ExecutionException, InterruptedException {
List<Future<Integer>> futures = new ArrayList<>(5);
try (ExecutorService executorService = Executors.newFixedThreadPool(10)) {
for (int j=1;j<6;j++) {
int finalJ = j;
Future<Integer> future = executorService.submit(() -> {
System.out.println(LocalDateTime.now()+" task " + finalJ + " submit from " + Thread.currentThread().getName());
try {
Thread.sleep(1000L * finalJ);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
return finalJ;
});
futures.add(future);
}
for (int j=0;j<5;j++) {
Future<Integer> future = futures.get(j);
Integer integer = future.get();
System.out.println(LocalDateTime.now()+" task " + integer+ " complete from " + Thread.currentThread().getName());
}
}
}
}
package com.ediagnosis.cdr.dashBoard;
import java.io.BufferedReader;
import java.io.InputStreamReader;
public class DashBoardTest {
public static void main(String[] args) throws Exception {
System.out.println("top -bn1");
getCpuUsage();
System.out.println("free -h");
getMemoryUsage();
System.out.println("df -h /");
getDiskUsage();
System.out.println("cat /proc/loadavg");
getLoadAverage();
}
public static void getCpuUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("top", "-bn1");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.contains("Cpu(s)")) {
System.out.println(line);
}
}
}
public static void getMemoryUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("free", "-h");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.startsWith("Mem:")) {
System.out.println(line);
}
}
}
public static void getDiskUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("df", "-h", "/");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
public static void getLoadAverage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("cat", "/proc/loadavg");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
System.out.println("cpu cores:"+ Runtime.getRuntime().availableProcessors());
}
}
package com.ediagnosis.cdr.dashBoard;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DashBoardTest2 {
public static void main(String[] args) throws Exception {
System.out.println("cpu info:-----------------");
System.out.println(getCpuUsage());
System.out.println("memory info:-----------------");
System.out.println(getMemoryUsage());
System.out.println("disk info:-----------------");
System.out.println(getDiskUsage());
}
public static Map<String, String> getCpuUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("top", "-bn1");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
String cpuLine = "";
String loadLine = "";
while ((line = reader.readLine()) != null) {
if (line.contains("load average:")) {
loadLine = line;
} else if (line.contains("Cpu(s)")) {
cpuLine = line;
}
if (!cpuLine.isBlank() && !loadLine.isBlank()) {
break;
}
}
// 去掉前缀 "%Cpu(s):",并提取数字
String[] parts = cpuLine
.replace("%Cpu(s):", "")
.trim()
.split(",");
// 存储各个字段的值
double id = 0;
for (String part : parts) {
part = part.trim();
if (part.endsWith("id")) {
id = Double.parseDouble(part.replace("id", "").trim());
}
}
String cpuUsage = Math.round(100 - id)+ "%";
String cpuCores = String.valueOf(
Runtime.getRuntime().availableProcessors()
);
String loadAverage = "";
// 正则匹配 "load average: 数值, 数值, 数值"
Pattern pattern = Pattern.compile("load average: ([\\d.]+), ([\\d.]+), ([\\d.]+)");
Matcher matcher = pattern.matcher(loadLine);
if (matcher.find()) {
double fiveMin = Double.parseDouble(matcher.group(2));
loadAverage = String.valueOf(fiveMin);
}
// 方法一:100 - idle
return Map.of(
"cpuUsage", cpuUsage,
"cpuCores", cpuCores,
"loadAverage", loadAverage
);
}
public static Map<String, String> getMemoryUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("free", "-h");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
String memLine="";
while ((line = reader.readLine()) != null) {
if (line.startsWith("Mem:")) {
memLine = line;
}
}
// 将多个空格替换为单个空格,便于分割
String[] parts = memLine.trim().replaceAll("\\s+", " ").split(" ");
// 解析各字段(单位:KB、M、G)
String totalStr = parts[1];
String availableStr = parts[6];
// 转换为统一单位(GB)
double total = parseSizeToGB(totalStr);
double available = parseSizeToGB(availableStr);
// 计算内存使用率:(1 - available / total) * 100%
double usagePercent = (1 - (available / total)) * 100;
String memoryUsage =Math.round(usagePercent) +"%";
String used=(total-available)+"G";
String size=total+"G";
String freee=available+"G";
return Map.of(
"used",used,
"size",size,
"freee",freee,
"memoryUsage",memoryUsage
);
}
private static double parseSizeToGB(String size) {
double value = 0;
if (size.endsWith("G")) {
value = Double.parseDouble(size.replace("G", ""));
} else if (size.endsWith("M")) {
value = Double.parseDouble(size.replace("M", "")) / 1024;
} else if (size.endsWith("K")) {
value = Double.parseDouble(size.replace("K", "")) / (1024 * 1024);
} else {
value = Double.parseDouble(size) / (1024 * 1024 * 1024);
}
return value;
}
public static Map<String, String> getDiskUsage() throws Exception {
ProcessBuilder pb = new ProcessBuilder("df", "-h", "/");
Process process = pb.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String header=reader.readLine();
String body = reader.readLine();
String[] parts = body.trim().replaceAll("\\s+", " ").split(" ");
// 确保是有效行(至少有6列)
if (parts.length < 6 || parts[4].indexOf('%') == -1) {
throw new IllegalArgumentException("无效的磁盘使用行");
}
// 提取 Use% 字段(如 "69%")
String usePercent = parts[4];
String size=parts[1];
String free=parts[3];
String used=parts[2];
// 转换为整数
return Map.of("usePercent", usePercent,
"size", size,
"free", free,
"used", used);
}
}
package com.ediagnosis.cdr.service;
import com.mybatisflex.core.datasource.DataSourceKey;
import com.mybatisflex.core.row.Db;
import com.mybatisflex.core.row.Row;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import java.util.List;
@SpringBootTest
class HiveServiceTest {
@Test
public void selectPage() {
String sql = "SELECT * FROM ods.xjd_patient_info ORDER BY id LIMIT 10";
try {
DataSourceKey.use("ds-hive");
List<Row> rows = Db.selectListBySql(sql);
System.out.println(rows);
} finally {
DataSourceKey.clear();
}
}
@Test
public void insertTest() {
String sql = "INSERT INTO TABLE ods.test1(id) VALUES (?)";
try {
DataSourceKey.use("ds-hive");
int rows = Db.insertBySql(sql,2);
System.out.println(rows);
} finally {
DataSourceKey.clear();
}
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment