diff --git a/pom.xml b/pom.xml index 3770289e7c7f2d0f13fcbf4b7e4e90c4c393a77c..136eb2b234d870a8b5e6a095ff1efea92b5ad7bb 100644 --- a/pom.xml +++ b/pom.xml @@ -39,102 +39,113 @@ runtime + + com.facebook.presto + presto-jdbc + 0.293 + - - - org.apache.hive - hive-jdbc - 3.1.2 - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - log4j-1.2-api - org.apache.logging.log4j - - - log4j-slf4j-impl - org.apache.logging.log4j - - - log4j-web - org.apache.logging.log4j - - - commons-logging - commons-logging - - - jdk.tools - jdk.tools - - - org.eclipse.jetty.aggregate - * - + com.github.ben-manes.caffeine + caffeine + 3.2.2 + - - org.eclipse.jetty - jetty-runner - - - servlet-api - javax.servlet - - - javax.servlet.jsp-api - javax.servlet.jsp - - - javax.servlet.jsp - org.glassfish.web - - - junit - junit - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - org.apache.hadoop - hadoop-common - 3.3.4 - - - commons-logging - commons-logging - - - - - org.apache.hadoop - hadoop-hdfs - 3.3.4 - - - commons-logging - commons-logging - - - + - - org.apache.hadoop - hadoop-client - 3.3.4 - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + com.zaxxer HikariCP diff --git a/src/main/java/com/ediagnosis/cdr/cache/CacheExecutor.java b/src/main/java/com/ediagnosis/cdr/cache/CacheExecutor.java new file mode 100644 index 0000000000000000000000000000000000000000..99ac0590c129113bf7f086cc6bdd316c0a474b9e --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/cache/CacheExecutor.java @@ -0,0 +1,10 @@ +package com.ediagnosis.cdr.cache; + +import java.util.Optional; + +public interface CacheExecutor { + + void put(String key, Object value); + + Optional get(String key, Class clazz); +} diff --git a/src/main/java/com/ediagnosis/cdr/cache/CacheFacade.java b/src/main/java/com/ediagnosis/cdr/cache/CacheFacade.java new file mode 100644 index 0000000000000000000000000000000000000000..6a164ee1c36c1a44ba4860367a93bc81532b25ce --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/cache/CacheFacade.java @@ -0,0 +1,49 @@ +package com.ediagnosis.cdr.cache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Optional; +// todo: 待扩展，按需缓存策略进行缓存 +@Component +public class CacheFacade { + + private static final Logger log = LoggerFactory.getLogger(CacheFacade.class); + private final List cacheExecutors; + + + public CacheFacade(List cacheExecutors) { + this.cacheExecutors = cacheExecutors; + CacheStrategyHolder.initCacheStrategy(); + } + + + public Optional getCache(String key, Class clazz) { + if (cacheExecutors.isEmpty()) { + log.warn("没有可用的缓存执行器"); + return Optional.empty(); + } + + + for (CacheExecutor executor : cacheExecutors) { + Optional optional = executor.get(key, clazz); + if (optional.isPresent()) { + return optional; + } + } + return Optional.empty(); + + } + + public void putCache(String key, String value) { + if (cacheExecutors.isEmpty()) { + log.warn("没有可用的缓存执行器"); + } + for (CacheExecutor executor : cacheExecutors) { + executor.put(key, value); + } + } + +} diff --git a/src/main/java/com/ediagnosis/cdr/cache/CacheStrategyHolder.java b/src/main/java/com/ediagnosis/cdr/cache/CacheStrategyHolder.java new file mode 100644 index 0000000000000000000000000000000000000000..37bbd68eb1ad9b3b597cfb8f62949ec42bedc842 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/cache/CacheStrategyHolder.java @@ -0,0 +1,24 @@ +package com.ediagnosis.cdr.cache; + +import com.ediagnosis.cdr.context.JsonProcessor; + +import java.util.Map; +import java.util.Optional; + + +public class CacheStrategyHolder { + private static Map cacheStrategy; + + + + + public static void initCacheStrategy() { + String CACHE_STRATEGY_PATH = "classpath:cache-strategy.json"; + Optional optionalMap = JsonProcessor.readJsonFileToObject(CACHE_STRATEGY_PATH, Map.class); + optionalMap.ifPresent(map -> cacheStrategy = map); + } + + public static Object getStrategy(String key) { + return cacheStrategy.get(key); + } +} diff --git a/src/main/java/com/ediagnosis/cdr/cache/CaffeineCacheExecutor.java b/src/main/java/com/ediagnosis/cdr/cache/CaffeineCacheExecutor.java new file mode 100644 index 0000000000000000000000000000000000000000..0f85f2c8ea9d3a9b83400237421ab1c44164e4cb --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/cache/CaffeineCacheExecutor.java @@ -0,0 +1,53 @@ +package com.ediagnosis.cdr.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; + +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +@Component +@Order(1) +public class CaffeineCacheExecutor implements CacheExecutor { + + + private static final Logger log = LoggerFactory.getLogger(CaffeineCacheExecutor.class); + private final Cache cache; + + public CaffeineCacheExecutor() { + this.cache = Caffeine.newBuilder() + .expireAfterWrite(3, TimeUnit.MINUTES) + .maximumSize(10_000) + .build(); + } + + + @Override + public void put(String key, Object value) { + log.info("put cache key:{}", key); + cache.put(key, value); + + } + + @Override + public Optional get(String key, Class clazz) { + Object object = cache.getIfPresent(key); + if (object != null) { + if (clazz.isInstance(object)) { + return Optional.of(clazz.cast(object)); + } else { + // 类型不匹配，可以记录警告或直接返回 empty + log.warn("Cached object type mismatch for key '{}': expected {}, got {}", + key, clazz.getName(), object.getClass().getName()); + return Optional.empty(); + } + } + log.warn("Cache miss for key '{}'", key); + return Optional.empty(); + } +} diff --git a/src/main/java/com/ediagnosis/cdr/context/JsonProcessor.java b/src/main/java/com/ediagnosis/cdr/context/JsonProcessor.java index 692a078c24376d911e61fdfa2f96e1898b1ce7c7..631d3ab10cdf919f62565b8ba6a8f48f7f98f083 100644 --- a/src/main/java/com/ediagnosis/cdr/context/JsonProcessor.java +++ b/src/main/java/com/ediagnosis/cdr/context/JsonProcessor.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; +import java.io.File; +import java.io.IOException; import java.util.Optional; @@ -15,24 +18,66 @@ public class JsonProcessor { public static Optional toObject(String json, Class clazz) { - T value = null; + T value; try { value = objectMapper.readValue(json, clazz); } catch (Exception e) { log.error("json to object error", e); + return Optional.empty(); } return Optional.ofNullable(value); } public static Optional toJson(Object object) { - String value = ""; + String value; try { value = objectMapper.writeValueAsString(object); } catch (JsonProcessingException e) { log.error("object to json error", e); + return Optional.empty(); + } + return Optional.ofNullable(value); + } + + public static void print(Object object) { + toJson(object).ifPresent(System.out::println); + } + public static Optional readJsonFileToObject(String path, Class clazz) { + T value; + if (path.startsWith("classpath:")) { + path = path.substring("classpath:".length()); + ClassPathResource resource = new ClassPathResource(path); + if (!resource.exists()) { + log.error("classpath resource not exists: {}", path); + return Optional.empty(); + } + try { + value = objectMapper.readValue(resource.getInputStream(), clazz); + } catch (IOException e) { + log.error("read json file error", e); + return Optional.empty(); + } + return Optional.ofNullable(value); + }else { + File file = new File(path); + if (!file.exists()) { + log.error("file not exists: {}", path); + return Optional.empty(); + } + + try { + value = objectMapper.readValue(file, clazz); + } catch (IOException e) { + log.error("read json file error", e); + return Optional.empty(); + } } + + + return Optional.ofNullable(value); } + } diff --git a/src/main/java/com/ediagnosis/cdr/value/Page.java b/src/main/java/com/ediagnosis/cdr/context/value/Page.java similarity index 97% rename from src/main/java/com/ediagnosis/cdr/value/Page.java rename to src/main/java/com/ediagnosis/cdr/context/value/Page.java index 86e353e2fc5e9deaa677f4f9b45155292fc4726b..6d67eecc07b7cd33406ba9ebad4f66ad7002f21a 100644 --- a/src/main/java/com/ediagnosis/cdr/value/Page.java +++ b/src/main/java/com/ediagnosis/cdr/context/value/Page.java @@ -1,4 +1,4 @@ -package com.ediagnosis.cdr.value; +package com.ediagnosis.cdr.context.value; import java.util.List; diff --git a/src/main/java/com/ediagnosis/cdr/value/Response.java b/src/main/java/com/ediagnosis/cdr/context/value/Response.java similarity index 97% rename from src/main/java/com/ediagnosis/cdr/value/Response.java rename to src/main/java/com/ediagnosis/cdr/context/value/Response.java index 26603f7ff33adfc8642fd955998684b97b401144..3a76ec49d0bb0e5b83d47bff96ce588fabf5fb33 100644 --- a/src/main/java/com/ediagnosis/cdr/value/Response.java +++ b/src/main/java/com/ediagnosis/cdr/context/value/Response.java @@ -1,4 +1,4 @@ -package com.ediagnosis.cdr.value; +package com.ediagnosis.cdr.context.value; import java.io.Serializable; diff --git a/src/main/java/com/ediagnosis/cdr/controller/DashBoardController.java b/src/main/java/com/ediagnosis/cdr/controller/DashBoardController.java deleted file mode 100644 index 6441acf552eda0cbd31e25d3ba8b1b1534c25650..0000000000000000000000000000000000000000 --- a/src/main/java/com/ediagnosis/cdr/controller/DashBoardController.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.ediagnosis.cdr.controller; - -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequestMapping("/dashBoard") -public class DashBoardController { - - - -} diff --git a/src/main/java/com/ediagnosis/cdr/dao/DimDataSummaryRepository.java b/src/main/java/com/ediagnosis/cdr/dao/DimDataSummaryRepository.java deleted file mode 100644 index 64a0c4e134b8696a99b245ba1dda663fafe21618..0000000000000000000000000000000000000000 --- a/src/main/java/com/ediagnosis/cdr/dao/DimDataSummaryRepository.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.ediagnosis.cdr.dao; - -import com.ediagnosis.cdr.dao.entity.DimDataSummaryEntity; -import com.ediagnosis.cdr.dao.mapper.DimDataSummaryMapper; -import com.ediagnosis.cdr.domain.DimDataSummary; -import com.ediagnosis.cdr.context.TimeUtil; -import org.springframework.stereotype.Repository; - -import java.time.LocalDateTime; -import java.util.List; -import java.util.Optional; - -@Repository -public class DimDataSummaryRepository { - - private final DimDataSummaryMapper dimDataSummaryMapper; - - public DimDataSummaryRepository(DimDataSummaryMapper dimDataSummaryMapper) { - this.dimDataSummaryMapper = dimDataSummaryMapper; - } - - public List getAllDimDataSummary() { - List entityList = dimDataSummaryMapper.selectAll(); - return entityList.stream().map(this::convertToDimDataSummary).toList(); - } - - - private DimDataSummary convertToDimDataSummary(DimDataSummaryEntity entity) { - Optional timeOptional = TimeUtil.toLocalDateTime(entity.getUpdateTime()); - - return new DimDataSummary( - entity.getName(), - entity.getCode(), - entity.getValue(), - timeOptional.orElse(null) - ); - } - - -} diff --git a/src/main/java/com/ediagnosis/cdr/dao/DwsRepository.java b/src/main/java/com/ediagnosis/cdr/dao/DwsRepository.java index d904fb3a07a210a713e79272e52f90d146a4239c..3cf5f6ee1610628d233ddafc78dbfde97f79c4d5 100644 --- a/src/main/java/com/ediagnosis/cdr/dao/DwsRepository.java +++ b/src/main/java/com/ediagnosis/cdr/dao/DwsRepository.java @@ -1,7 +1,7 @@ package com.ediagnosis.cdr.dao; -import com.ediagnosis.cdr.value.Page; +import com.ediagnosis.cdr.context.value.Page; import com.mybatisflex.core.row.Row; import org.springframework.stereotype.Repository; diff --git a/src/main/java/com/ediagnosis/cdr/dao/IndicatorRepository.java b/src/main/java/com/ediagnosis/cdr/dao/IndicatorRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..0864c7c84bc936b7087b23fb61ccbaa6ec4ca555 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dao/IndicatorRepository.java @@ -0,0 +1,37 @@ +package com.ediagnosis.cdr.dao; + +import com.ediagnosis.cdr.dao.entity.IndicatorEntity; +import com.ediagnosis.cdr.dao.mapper.IndicatorMapper; +import com.ediagnosis.cdr.context.TimeUtil; +import com.ediagnosis.cdr.dataGovernance.value.Indicator; +import org.springframework.stereotype.Repository; + +import java.time.LocalDateTime; + +import java.util.Optional; + +@Repository +public class IndicatorRepository { + + private final IndicatorMapper indicatorMapper; + + public IndicatorRepository(IndicatorMapper indicatorMapper) { + this.indicatorMapper = indicatorMapper; + } + + + + + private Optional convertToIndicator(IndicatorEntity entity) { + Optional timeOptional = TimeUtil.toLocalDateTime(entity.getUpdateTime()); + return timeOptional.map(localDateTime -> new Indicator( + entity.getName(), + entity.getCode(), + entity.getValue(), + localDateTime + )); + + } + + +} diff --git a/src/main/java/com/ediagnosis/cdr/dao/entity/DimDataSummaryEntity.java b/src/main/java/com/ediagnosis/cdr/dao/entity/IndicatorEntity.java similarity index 85% rename from src/main/java/com/ediagnosis/cdr/dao/entity/DimDataSummaryEntity.java rename to src/main/java/com/ediagnosis/cdr/dao/entity/IndicatorEntity.java index 735f2222c763b3534876761526d9f74ade53b002..c19a457e493a2daade0ba9af8177cb9e8139b6c5 100644 --- a/src/main/java/com/ediagnosis/cdr/dao/entity/DimDataSummaryEntity.java +++ b/src/main/java/com/ediagnosis/cdr/dao/entity/IndicatorEntity.java @@ -9,8 +9,8 @@ import com.mybatisflex.annotation.Table; * 记录数据总量概览 * @TableName dim_data_summary */ -@Table(value ="dim_data_summary") -public class DimDataSummaryEntity { +@Table(value ="indicator") +public class IndicatorEntity { /** * */ @@ -32,6 +32,8 @@ public class DimDataSummaryEntity { */ private String value; + private String description; + /** * 更新时间 */ @@ -93,6 +95,15 @@ public class DimDataSummaryEntity { this.value = value; } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + /** * 更新时间 */ diff --git a/src/main/java/com/ediagnosis/cdr/dao/mapper/DimDataSummaryMapper.java b/src/main/java/com/ediagnosis/cdr/dao/mapper/IndicatorMapper.java similarity index 51% rename from src/main/java/com/ediagnosis/cdr/dao/mapper/DimDataSummaryMapper.java rename to src/main/java/com/ediagnosis/cdr/dao/mapper/IndicatorMapper.java index 4d8fb3d8c4f62026c725e0d2cebdf53c6e149240..e6586a0a80fa4a777801625004cc809322d32aaa 100644 --- a/src/main/java/com/ediagnosis/cdr/dao/mapper/DimDataSummaryMapper.java +++ b/src/main/java/com/ediagnosis/cdr/dao/mapper/IndicatorMapper.java @@ -1,12 +1,12 @@ package com.ediagnosis.cdr.dao.mapper; -import com.ediagnosis.cdr.dao.entity.DimDataSummaryEntity; +import com.ediagnosis.cdr.dao.entity.IndicatorEntity; import com.mybatisflex.core.BaseMapper; import org.mybatis.spring.annotation.MapperScan; @MapperScan -public interface DimDataSummaryMapper extends BaseMapper { +public interface IndicatorMapper extends BaseMapper { } diff --git a/src/main/java/com/ediagnosis/cdr/dashBoard/DashBoardController.java b/src/main/java/com/ediagnosis/cdr/dashBoard/DashBoardController.java new file mode 100644 index 0000000000000000000000000000000000000000..86c3e1c0079a5e76864dc9310d78a2f2112461cd --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dashBoard/DashBoardController.java @@ -0,0 +1,35 @@ +package com.ediagnosis.cdr.dashBoard; + +import com.ediagnosis.cdr.context.value.Response; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/host") +public class DashBoardController { + + private final HostMonitorValueRepository hostMonitorValueRepository; + + public DashBoardController(HostMonitorValueRepository hostMonitorValueRepository) { + this.hostMonitorValueRepository = hostMonitorValueRepository; + } + + + @GetMapping("/list") + public Response> list() { + HostMonitorValue.HostValue hostValue = hostMonitorValueRepository.getHostValue(); + return Response.success(List.of(hostValue)); + + } + + @PostMapping("/monitor") + public Response> monitor(String hostName) { + + HostMonitorValue hostMonitorValue = hostMonitorValueRepository.getHostMonitorValue(hostName); + return Response.success(List.of(hostMonitorValue)); + } +} diff --git a/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitor.java b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitor.java new file mode 100644 index 0000000000000000000000000000000000000000..f55c8974404e50a03ee4c8fbe77f6afe87a3135e --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitor.java @@ -0,0 +1,254 @@ +package com.ediagnosis.cdr.dashBoard; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class HostMonitor { + + + private static final Logger log = LoggerFactory.getLogger(HostMonitor.class); + + public static Optional> getHostIpAndName() { + String hostname = ""; + String ip = ""; + Optional optionalHostname = executeCommand("hostname"); + Optional optionalIp = executeCommand("hostname", "-I"); + + if (optionalHostname.isPresent()) { + hostname = optionalHostname.get(); + + } + if (optionalIp.isPresent()) { + ip = optionalIp.get(); + } + + Map host = Map.of("hostName", hostname, + "ip", ip); + log.info("主机信息: " + host); + return Optional.of(host); + + } + + /** + * 执行系统命令并返回第一行输出 + */ + private static Optional executeCommand(String... command) { + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); // 合并标准输出和错误输出 + Optional optionalString = Optional.empty(); + log.info("执行命令: " + String.join(" ", command)); + Process process; + try { + process = pb.start(); + } catch (IOException e) { + // 可以记录日志 + log.error("执行命令失败:" + String.join(" ", command), e); + return optionalString; + } + // 使用 try-with-resources 自动关闭流 + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + + String line = reader.readLine(); // 通常只关心第一行 + + optionalString = Optional.of(line); + + } catch (IOException e) { + // 可以记录日志 + log.error("读取命令结果失败:" + String.join(" ", command), e); + return optionalString; + } + return optionalString; + } + + public static Optional> getCpuUsage() { + Optional> optionalMap = Optional.empty(); + ProcessBuilder pb = new ProcessBuilder("top", "-bn1"); + log.info("执行命令:top -bn1"); + Process process = null; + try { + process = pb.start(); + } catch (IOException e) { + log.error("执行命令时，启动进程失败: top -bn1", e); + return optionalMap; + } + String cpuLine = ""; + String loadLine = ""; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.contains("load average:")) { + loadLine = line; + } else if (line.contains("Cpu(s)")) { + cpuLine = line; + } + if (!cpuLine.isBlank() && !loadLine.isBlank()) { + break; + } + } + } catch (IOException e) { + log.error("读取cpu信息时失败:top -bn1 ", e); + return optionalMap; + } + + // 去掉前缀 "%Cpu(s):"，并提取数字 + String[] parts = cpuLine + .replace("%Cpu(s):", "") + .trim() + .split(","); + + // 存储各个字段的值 + double id = 0; + + for (String part : parts) { + part = part.trim(); + + if (part.endsWith("id")) { + id = Double.parseDouble(part.replace("id", "").trim()); + } + + } + String cpuUsage = Math.round(100 - id) + "%"; + String cpuCores = String.valueOf( + Runtime.getRuntime().availableProcessors() + ); + + String loadAverage = ""; + // 正则匹配 "load average: 数值, 数值, 数值" + Pattern pattern = Pattern.compile("load average: ([\\d.]+), ([\\d.]+), ([\\d.]+)"); + Matcher matcher = pattern.matcher(loadLine); + + if (matcher.find()) { + double fiveMin = Double.parseDouble(matcher.group(2)); + loadAverage = String.valueOf(fiveMin); + } + + + // 方法一：100 - idle + Map cpu = Map.of( + "cpuUsage", cpuUsage, + "cpuCores", cpuCores, + "loadAverage", loadAverage + ); + + log.info("cpu信息: " + cpu); + return Optional.of(cpu); + } + + + public static Optional> getMemoryUsage() { + + Optional> optionalMap = Optional.empty(); + ProcessBuilder pb = new ProcessBuilder("free", "-h"); + log.info("执行命令:free -h"); + Process process = null; + try { + process = pb.start(); + } catch (IOException e) { + log.error("执行命令时，启动进程失败: free -h"); + return optionalMap; + } + String memLine = ""; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("Mem:")) { + memLine = line; + } + } + } catch (IOException e) { + log.error("读取内存信息时失败:free -h "); + } + + // 将多个空格替换为单个空格，便于分割 + String[] parts = memLine.trim().replaceAll("\\s+", " ").split(" "); + + // 解析各字段（单位：KB、M、G） + String totalStr = parts[1]; + String availableStr = parts[6]; + + // 转换为统一单位（GB） + double total = parseSizeToGB(totalStr); + double available = parseSizeToGB(availableStr); + + // 计算内存使用率：(1 - available / total) * 100% + double usagePercent = (1 - (available / total)) * 100; + String memoryUsage = Math.round(usagePercent) + "%"; + String used = (total - available) + "G"; + String size = total + "G"; + String free = available + "G"; + Map memory = Map.of( + "used", used, + "size", size, + "free", free, + "memoryUsage", memoryUsage + ); + log.info("内存信息: " + memory); + return Optional.of(memory); + } + + private static double parseSizeToGB(String size) { + double value; + if (size.endsWith("G")) { + value = Double.parseDouble(size.replace("G", "")); + } else if (size.endsWith("M")) { + value = Double.parseDouble(size.replace("M", "")) / 1024; + } else if (size.endsWith("K")) { + value = Double.parseDouble(size.replace("K", "")) / (1024 * 1024); + } else { + value = Double.parseDouble(size) / (1024 * 1024 * 1024); + } + return value; + } + + public static Optional> getDiskUsage() { + ProcessBuilder pb = new ProcessBuilder("df", "-h", "/"); + Optional> optionalMap = Optional.empty(); + log.info("执行命令:df -h /"); + Process process = null; + try { + process = pb.start(); + } catch (IOException e) { + log.error("执行命令时，启动进程失败: df -h /", e); + return optionalMap; + } + String body; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + body = reader.readLine(); + } catch (IOException e) { + log.error("读取磁盘信息时失败:df -h /", e); + return optionalMap; + } + String[] parts = body.trim().replaceAll("\\s+", " ").split(" "); + + // 确保是有效行（至少有6列） + if (parts.length < 6 || parts[4].indexOf('%') == -1) { + throw new IllegalArgumentException("无效的磁盘使用行"); + } + + // 提取 Use% 字段（如 "69%"） + String usePercent = parts[4]; + String size = parts[1]; + String free = parts[3]; + String used = parts[2]; + + // 转换为整数 + Map disk = Map.of( + "usePercent", usePercent, + "size", size, + "free", free, + "used", used); + log.info("磁盘信息: " + disk); + return Optional.of(disk); + + } +} diff --git a/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValue.java b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValue.java new file mode 100644 index 0000000000000000000000000000000000000000..843c9a947904fe42a0c50d98a7088ceed85b1a29 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValue.java @@ -0,0 +1,46 @@ +package com.ediagnosis.cdr.dashBoard; + +public record HostMonitorValue( + HostValue host, + CpuValue cpu, + MemoryValue memory, + DiskValue disk + +) { + + public record HostValue( + String hostName, + String hostIp + ){ + + } + + + public record CpuValue( + int coreNum, + double fiveMinLoad, + String usage + ){ + + } + + public record MemoryValue( + String usage, + String size, + String used, + String free + ){ + + } + + public record DiskValue( + String usage, + String size, + String used, + String free + ){ + + } + + +} diff --git a/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValueRepository.java b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValueRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..a54d03417cb17bda23afe3cec4cb16510bd8ea74 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dashBoard/HostMonitorValueRepository.java @@ -0,0 +1,86 @@ +package com.ediagnosis.cdr.dashBoard; + +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.Optional; + +@Component +public class HostMonitorValueRepository { + + + // todo: 待扩展，获取集群主机的列表 + public HostMonitorValue.HostValue getHostValue() { + Optional> hostIpAndNameOptional = HostMonitor.getHostIpAndName(); + String hostname = ""; + String ip = ""; + if (hostIpAndNameOptional.isPresent()) { + Map hostMap = hostIpAndNameOptional.get(); + hostname = hostMap.get("hostname"); + ip = hostMap.get("ip"); + } + + + return new HostMonitorValue.HostValue(hostname, ip); + } + + // todo: 带扩展，获取指定主机的监控信息 + public HostMonitorValue getHostMonitorValue(String hostName) { + + Optional> hostIpAndNameOptional = HostMonitor.getHostIpAndName(); + HostMonitorValue.HostValue hostValue; + + if (hostIpAndNameOptional.isPresent()) { + Map hostMap = hostIpAndNameOptional.get(); + String hostNameInner = hostMap.get("hostname"); + String ip = hostMap.get("ip"); + hostValue = new HostMonitorValue.HostValue(hostNameInner, ip); + }else { + hostValue = new HostMonitorValue.HostValue("", ""); + } + + + Optional> cpuUsageOptional = HostMonitor.getCpuUsage(); + HostMonitorValue.CpuValue cpuValue; + if (cpuUsageOptional.isPresent()) { + Map cpuMap = cpuUsageOptional.get(); + String cpuUsage = cpuMap.get("cpuUsage"); + String cpuCores = cpuMap.get("cpuCores"); + int cpuCoresNum = Integer.parseInt(cpuCores); + String loadAverage = cpuMap.get("loadAverage"); + double loadAverageNum = Double.parseDouble(loadAverage); + cpuValue = new HostMonitorValue.CpuValue(cpuCoresNum, loadAverageNum, cpuUsage); + } else { + cpuValue = new HostMonitorValue.CpuValue(0, 0, ""); + } + Optional> memoryUsageOptional = HostMonitor.getMemoryUsage(); + HostMonitorValue.MemoryValue memoryValue; + if (memoryUsageOptional.isPresent()) { + Map memoryMap = memoryUsageOptional.get(); + String used = memoryMap.get("used"); + String size = memoryMap.get("size"); + String free = memoryMap.get("free"); + String memoryUsage = memoryMap.get("memoryUsage"); + memoryValue = new HostMonitorValue.MemoryValue(memoryUsage, size, used, free); + } else { + memoryValue = new HostMonitorValue.MemoryValue("", "", "", ""); + } + + Optional> diskUsageOptional = HostMonitor.getDiskUsage(); + HostMonitorValue.DiskValue diskValue; + if (diskUsageOptional.isPresent()) { + + Map diskMap = diskUsageOptional.get(); + String usePercent = diskMap.get("usePercent"); + String used = diskMap.get("used"); + String size = diskMap.get("size"); + String free = diskMap.get("free"); + diskValue = new HostMonitorValue.DiskValue(usePercent, size, used, free); + } else { + diskValue = new HostMonitorValue.DiskValue("", "", "", ""); + } + + HostMonitorValue hostMonitorValue = new HostMonitorValue(hostValue, cpuValue, memoryValue, diskValue); + return hostMonitorValue; + } +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/CacheableTableDefinitionRepository.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/CacheableTableDefinitionRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..7e72875778fc3a4a89bcc1fa143c9f2497d5e03e --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/CacheableTableDefinitionRepository.java @@ -0,0 +1,35 @@ +package com.ediagnosis.cdr.dataGovernance; + +import com.ediagnosis.cdr.cache.CacheFacade; +import com.ediagnosis.cdr.dataGovernance.value.TableDefinitionItem; +import com.ediagnosis.cdr.dataGovernance.value.TableTree; +import org.springframework.stereotype.Component; + +import java.util.List; +@Component +public class CacheableTableDefinitionRepository implements TableDefinitionRepository{ + private final DefaultTableDefinitionRepository defaultTableDefinitionRepository; + + private final CacheFacade cacheFacade; + + public CacheableTableDefinitionRepository(DefaultTableDefinitionRepository defaultTableDefinitionRepository, + CacheFacade cacheFacade) { + this.defaultTableDefinitionRepository = defaultTableDefinitionRepository; + this.cacheFacade = cacheFacade; + } + + @Override + public TableTree getTableTree(String keyword) { + + cacheFacade.getCache(keyword, TableTree.class); + + defaultTableDefinitionRepository.getTableTree(keyword); + + return null; + } + + @Override + public List getTableDefinitionItem(String tableId) { + return List.of(); + } +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/DataGovernanceController.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/DataGovernanceController.java new file mode 100644 index 0000000000000000000000000000000000000000..e4d97a459687490e3d9f4d33cc7ff641b303ef87 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/DataGovernanceController.java @@ -0,0 +1,11 @@ +package com.ediagnosis.cdr.dataGovernance; + +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class DataGovernanceController { + + + + +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/DefaultTableDefinitionRepository.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/DefaultTableDefinitionRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..625a10d410fd7c02293553f96d68696f466e7434 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/DefaultTableDefinitionRepository.java @@ -0,0 +1,24 @@ +package com.ediagnosis.cdr.dataGovernance; + +import com.ediagnosis.cdr.dataGovernance.value.TableDefinitionItem; +import com.ediagnosis.cdr.dataGovernance.value.TableTree; +import org.springframework.stereotype.Component; + +import java.util.List; +@Component +public class DefaultTableDefinitionRepository implements TableDefinitionRepository { + + + @Override + public TableTree getTableTree(String keyword) { + + + return null; + } + + @Override + public List getTableDefinitionItem(String tableId) { + + return List.of(); + } +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionRepository.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..88b57bca2788860564f6fe43b2b42ccb12139ead --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionRepository.java @@ -0,0 +1,19 @@ +package com.ediagnosis.cdr.dataGovernance; + +import com.ediagnosis.cdr.dataGovernance.value.TableDefinitionItem; +import com.ediagnosis.cdr.dataGovernance.value.TableTree; + +import java.util.List; + + +public interface TableDefinitionRepository { + + /** + * 关键字搜索表 + * @param keyword 关键字 + */ + TableTree getTableTree(String keyword); + + List getTableDefinitionItem(String tableId); + +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/request/SourceTableRequest.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/request/SourceTableRequest.java new file mode 100644 index 0000000000000000000000000000000000000000..880b55efa39d9c386be7afeabe348b2a810e0188 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/request/SourceTableRequest.java @@ -0,0 +1,6 @@ +package com.ediagnosis.cdr.dataGovernance.request; + +public record SourceTableRequest( + +) { +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/request/TableFieldRequest.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/request/TableFieldRequest.java new file mode 100644 index 0000000000000000000000000000000000000000..02fac284a708891b84dc1cde971e2089e6ce7d70 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/request/TableFieldRequest.java @@ -0,0 +1,8 @@ +package com.ediagnosis.cdr.dataGovernance.request; + +public record TableFieldRequest( + Integer pageNo, + Integer pageSize, + Integer tableId +) { +} diff --git a/src/main/java/com/ediagnosis/cdr/domain/DimDataSummary.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/Indicator.java similarity index 65% rename from src/main/java/com/ediagnosis/cdr/domain/DimDataSummary.java rename to src/main/java/com/ediagnosis/cdr/dataGovernance/value/Indicator.java index 36252f03b22328780e37d6f1b7b39a1101988e10..65e2ac00f6e87098c8665c7edd3119e80adab0ab 100644 --- a/src/main/java/com/ediagnosis/cdr/domain/DimDataSummary.java +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/Indicator.java @@ -1,9 +1,9 @@ -package com.ediagnosis.cdr.domain; +package com.ediagnosis.cdr.dataGovernance.value; import java.time.LocalDateTime; -public record DimDataSummary( +public record Indicator( String name, String code, String value, diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableDefinitionItem.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableDefinitionItem.java new file mode 100644 index 0000000000000000000000000000000000000000..b6e0646f10fdd8a0b8e9af18843d52cb3b0f6525 --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableDefinitionItem.java @@ -0,0 +1,9 @@ +package com.ediagnosis.cdr.dataGovernance.value; + +public record TableDefinitionItem( + String column, + String type, + String comment +) { + +} diff --git a/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableTree.java b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableTree.java new file mode 100644 index 0000000000000000000000000000000000000000..82891fe72784b44fdcbbde3aedee0187016d0afb --- /dev/null +++ b/src/main/java/com/ediagnosis/cdr/dataGovernance/value/TableTree.java @@ -0,0 +1,77 @@ +package com.ediagnosis.cdr.dataGovernance.value; + +import java.util.LinkedList; +import java.util.List; + +public class TableTree { + + + + public static class TreeNode { + private final String id; + private final String code; + private final List children = new LinkedList<>(); + private final boolean tableNodeFlag; + + public TreeNode(String id, String code, boolean tableNodeFlag) { + this.id = id; + this.code = code; + this.tableNodeFlag = tableNodeFlag; + } + + public TreeNode appendChild(String id, String code,boolean tableNodeFlag) { + TreeNode node = new TreeNode(id,code,tableNodeFlag); + this.children.add(node); + return node; + } + + + public String getId() { + return id; + } + + public String getCode() { + return code; + } + + public boolean isTableNodeFlag() { + return tableNodeFlag; + } + + public List getChildren() { + return children; + } + } + + + private List rootNodes; + + public static TableTree create() { + TableTree tableTree = new TableTree(); + tableTree.rootNodes = new LinkedList<>(); + return tableTree; + } + + public TableTree ofRoot(TreeNode... node) { + rootNodes.addAll(List.of(node)); + return this; + } + + public TableTree ofChild(TreeNode parent, TreeNode... childNode) { + parent.children.addAll(List.of(childNode)); + return this; + } + + public TreeNode addRoot(TreeNode node) { + rootNodes.add(node); + return node; + } + + public List getRootNodes() { + return rootNodes; + } + + public void setRootNodes(List rootNodes) { + this.rootNodes = rootNodes; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 36969e003121e8e3ea1e2946e536bf65a9000211..7dc9c09614480d11da6f2073cc3cb846ded2ad4f 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -8,5 +8,8 @@ mybatis-flex: ds-hive: url: jdbc:hive2://10.11.4.21:10000 username: root + ds-presto: + url: jdbc:presto://10.11.4.21:8999 + username: root server: port: 7001 \ No newline at end of file diff --git a/src/main/resources/generator/mapper/DimDataSummaryMapper.xml b/src/main/resources/generator/mapper/DimDataSummaryMapper.xml deleted file mode 100644 index d16ef85ba3f6355c9ee83e0d4abd176497febf5d..0000000000000000000000000000000000000000 --- a/src/main/resources/generator/mapper/DimDataSummaryMapper.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - id,name,code,value,update_time - - diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000000000000000000000000000000000000..ab5e89de3ef111dc5ed01238a7d55d2fae2fd7ee --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,47 @@ + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + ${LOG_PATH}/${LOG_FILE}.log + + + + + + ${LOG_PATH}/archive/${LOG_FILE}-%d{yyyy-MM-dd}.%i.log + + + 100MB + + + + + + + 30 + + + %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/java/com/ediagnosis/cdr/context/JsonProcessorTest.java b/src/test/java/com/ediagnosis/cdr/context/JsonProcessorTest.java index a9fd73173901232f71a03be439e15baf3132d436..48f1d4f26fafa62741c6c23782845a46d225c6ee 100644 --- a/src/test/java/com/ediagnosis/cdr/context/JsonProcessorTest.java +++ b/src/test/java/com/ediagnosis/cdr/context/JsonProcessorTest.java @@ -5,6 +5,8 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.util.Assert; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; @@ -30,4 +32,23 @@ class JsonProcessorTest { Assert.isTrue(json.get().equals("{\"name\":\"张三\",\"age\":18}"), "转换结果与预期不一致"); } + + @Test + void readJsonFileToObject() { + String path="classpath:cache-strategy.json"; + Optional optionalMap = JsonProcessor.readJsonFileToObject(path, Map.class); + Assert.isTrue(optionalMap.isPresent(), "转换结果不应为空"); + System.out.println(optionalMap); + Map map = optionalMap.get(); + Object tableTree = map.get("fullTableTree"); + if(tableTree instanceof Map tableTreeMap ){ + Object expiration = tableTreeMap.get("expiration"); + if(expiration instanceof String s){ + System.out.println(s); + } + + } + + + } } \ No newline at end of file diff --git a/src/test/java/com/ediagnosis/cdr/dashBoard/AsyncTest.java b/src/test/java/com/ediagnosis/cdr/dashBoard/AsyncTest.java index dc12741304a927145fc340fab0bf3b08e06cad91..4404ebe4a6d19d9bab98ee4c1d8310576d3b7b93 100644 --- a/src/test/java/com/ediagnosis/cdr/dashBoard/AsyncTest.java +++ b/src/test/java/com/ediagnosis/cdr/dashBoard/AsyncTest.java @@ -10,7 +10,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; + @SpringBootTest public class AsyncTest { diff --git a/src/test/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionTest.java b/src/test/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a786c9561f0bca384d157c1040e8004f3ebb61a7 --- /dev/null +++ b/src/test/java/com/ediagnosis/cdr/dataGovernance/TableDefinitionTest.java @@ -0,0 +1,53 @@ +package com.ediagnosis.cdr.dataGovernance; + +import com.mybatisflex.core.datasource.DataSourceKey; +import com.mybatisflex.core.row.Db; +import com.mybatisflex.core.row.Row; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.ArrayList; +import java.util.List; + +@SpringBootTest +public class TableDefinitionTest { + + @Test + public void testPresto() { + try{ + DataSourceKey.use("ds-presto"); + String sql = "select count(*) from hive.ods.xjd_patient_info"; + List rows = Db.selectListBySql(sql); + System.out.println(rows.size()); + System.out.println(rows); + }finally{ + DataSourceKey.clear(); + } + } + + + @Test + public void testTableDefinition() { + List rows=null; + try{ + DataSourceKey.use("ds-presto"); + String sql = "DESCRIBE hive.ods.xjd_patient_info"; + rows = Db.selectListBySql(sql); + }finally{ + DataSourceKey.clear(); + } + List items = new ArrayList<>(rows.size()); + for (Row row : rows) { + TestTableDefinitionItem item = new TestTableDefinitionItem(); + item.setColumn(row.getString("Column")); + item.setType(row.getString("Type")); + item.setComment(row.getString("Comment")); + System.out.println(item); + items.add(item); + } + System.out.println(rows.size()); + System.out.println(items.size()); + + + } +} diff --git a/src/test/java/com/ediagnosis/cdr/dataGovernance/TableTreeTest.java b/src/test/java/com/ediagnosis/cdr/dataGovernance/TableTreeTest.java new file mode 100644 index 0000000000000000000000000000000000000000..6557619d8d975083a9b1de8f3b0e0de54c2239a5 --- /dev/null +++ b/src/test/java/com/ediagnosis/cdr/dataGovernance/TableTreeTest.java @@ -0,0 +1,26 @@ +package com.ediagnosis.cdr.dataGovernance; + +import com.ediagnosis.cdr.context.JsonProcessor; +import com.ediagnosis.cdr.dataGovernance.value.TableTree; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + + +@SpringBootTest +public class TableTreeTest { + + @Test + public void testTableTree() { + TableTree tableTree = TableTree.create(); + tableTree.addRoot(new TableTree.TreeNode("1","ip-xxxx", false)) + .appendChild("2","dataBase-xxxx",false) + .appendChild("3","table-xxx",true); + tableTree.addRoot(new TableTree.TreeNode("4","ip-yyyy", false)) + .appendChild("5","catalog-yyyy",false) + .appendChild("6","schema-yyyy", true) + .appendChild("7","table-yyyy", true); + tableTree.addRoot(new TableTree.TreeNode("8","ip-zzzz",false )) + .appendChild("9","table-zzzz",true); + JsonProcessor.print(tableTree); + } +} diff --git a/src/test/java/com/ediagnosis/cdr/dataGovernance/TestTableDefinitionItem.java b/src/test/java/com/ediagnosis/cdr/dataGovernance/TestTableDefinitionItem.java new file mode 100644 index 0000000000000000000000000000000000000000..9cf408e4e3b853506459ac3fb57c0def35f57018 --- /dev/null +++ b/src/test/java/com/ediagnosis/cdr/dataGovernance/TestTableDefinitionItem.java @@ -0,0 +1,41 @@ +package com.ediagnosis.cdr.dataGovernance; + +public class TestTableDefinitionItem { + + private String column; + private String type; + private String comment; + + public String getColumn() { + return column; + } + + public void setColumn(String column) { + this.column = column; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + @Override + public String toString() { + return "TestTableDefinitionItem{" + + "column='" + column + '\'' + + ", type='" + type + '\'' + + ", comment='" + comment + '\'' + + '}'; + } +} diff --git a/src/test/resources/cache-strategy.json b/src/test/resources/cache-strategy.json new file mode 100644 index 0000000000000000000000000000000000000000..ae0e0124ecadcc9957e9d0df73518a2fc736c577 --- /dev/null +++ b/src/test/resources/cache-strategy.json @@ -0,0 +1,6 @@ +{ + "fullTableTree": { + "expiration": "3000", + "otherProperties": "xxxx" + } +} \ No newline at end of file