diff --git a/dubhe-server/.gitignore b/dubhe-server/.gitignore index a103ef3..35a9b43 100644 --- a/dubhe-server/.gitignore +++ b/dubhe-server/.gitignore @@ -36,7 +36,7 @@ Thumbs.db target/ output/ *.jar -!/dubhe-dcm/lib/dcm4che-core-5.19.1.jar +!/dubhe-data-dcm/lib/dcm4che-core-5.19.1.jar !.mvn/wrapper/* *.war *.zip @@ -48,6 +48,4 @@ output/ .project .classpath logs/ -/dubhe-k8s/src/main/resources/kubeconfig -*.log -/dubhe-admin/kubeconfig \ No newline at end of file +*.log \ No newline at end of file diff --git a/dubhe-server/.gitmessage b/dubhe-server/.gitmessage index f893509..e07bb61 100644 --- a/dubhe-server/.gitmessage +++ b/dubhe-server/.gitmessage @@ -1,5 +1,4 @@ -# type: subject # type:feat增加新功能；fix修复错误；docs修改文档；style修改样式；refactor代码重构；test增加测试模块，不涉及生产环境的代码；chore更新核心模块，包配置文件，不涉及生产环境的代码 # subject: 一两句话简述提交原因，与下方详细描述间隔一行 diff --git a/dubhe-server/Dockerfile b/dubhe-server/Dockerfile index 09b4f13..4c58cf6 100644 --- a/dubhe-server/Dockerfile +++ b/dubhe-server/Dockerfile @@ -1,62 +1,5 @@ -FROM docker +FROM harbor.dubhe.ai/dubhe/docker-in-docker-java8-ssh:v3 +RUN apk add ffmpeg ENV LANG=C.UTF-8 - -#更换为aliyun源 -RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/community" > /etc/apk/repositories -RUN echo "http://mirrors.aliyun.com/alpine/latest-stable/main" >> /etc/apk/repositories -#update apk bash vim wget curl net-tools sshpass -RUN apk update \ - && apk upgrade \ - && apk --no-cache add ca-certificates \ - && apk add bash \ - bash-doc \ - bash-completion \ - vim \ - wget \ - curl \ - net-tools \ - sshpass \ - && rm -rf /var/cache/apk/* \ - && /bin/bash - -#setup glibc -RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-2.32-r0.apk -RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-bin-2.32-r0.apk -RUN wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.32-r0/glibc-i18n-2.32-r0.apk -RUN apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \ - echo \ - "-----BEGIN PUBLIC KEY-----\ - MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApZ2u1KJKUu/fW4A25y9m\ - y70AGEa/J3Wi5ibNVGNn1gT1r0VfgeWd0pUybS4UmcHdiNzxJPgoWQhV2SSW1JYu\ - tOqKZF5QSN6X937PTUpNBjUvLtTQ1ve1fp39uf/lEXPpFpOPL88LKnDBgbh7wkCp\ - m2KzLVGChf83MS0ShL6G9EQIAUxLm99VpgRjwqTQ/KfzGtpke1wqws4au0Ab4qPY\ - KXvMLSPLUp7cfulWvhmZSegr5AdhNw5KNizPqCJT8ZrGvgHypXyiFvvAH5YRtSsc\ - Zvo9GI2e2MaZyo9/lvb+LbLEJZKEQckqRj4P26gmASrZEPStwc+yqy1ShHLA0j6m\ - 1QIDAQAB\ - -----END PUBLIC KEY-----" | sed 's/ */\n/g' > "/etc/apk/keys/sgerrand.rsa.pub" -RUN apk add glibc-2.32-r0.apk -RUN apk add glibc-bin-2.32-r0.apk -RUN apk add glibc-i18n-2.32-r0.apk -RUN rm -rf *.apk - -#setup date -RUN apk add tzdata -RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime - -#安装jdk -RUN wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz" -RUN tar xzf jdk-8u141-linux-x64.tar.gz -C /usr/local/ -ENV JAVA_HOME /usr/local/jdk1.8.0_141 -ENV PATH ${PATH}:${JAVA_HOME}/bin -ENV CLASSPATH $JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar - -#设置到nfs-host-ip的免密登录 -RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa >/dev/null 2>&1 && \ - #让连接新主机时，不进行公钥确认 - echo "StrictHostKeyChecking no">>/etc/ssh/ssh_config && \ - #发送公钥到目标服务器 - sshpass -p nfs-host-pwd ssh-copy-id -p 22 -i ~/.ssh/id_rsa.pub nfs-host-user@nfs-host-ip - -COPY dubhe-admin/target/dubhe-admin-1.0-exec.jar /dubhe/ -COPY dubhe-task/target/dubhe-task-1.0.jar /dubhe/ -COPY dubhe-serving-gateway/target/dubhe-serving-gateway.jar /dubhe/ \ No newline at end of file +COPY */target/*-exec.jar /dubhe/ +CMD ["bash", "-c","exec java $JVM_PARAM -Dspring.security.strategy=MODE_INHERITABLETHREADLOCAL -jar -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005 /dubhe/$JAR_BALL"] diff --git a/dubhe-server/LICENSE b/dubhe-server/LICENSE deleted file mode 100644 index 87562c3..0000000 --- a/dubhe-server/LICENSE +++ /dev/null @@ -1,211 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Other dependencies and licenses: ----------------------------------------------------------------------------------------- - -Open Source Software Licensed Under the Apache License, Version 2.0: -The below software in this distribution may have been modified. ----------------------------------------------------------------------------------------- -1. EL-ADMIN -Copyright 2019-2020 Zheng Jie \ No newline at end of file diff --git a/dubhe-server/README.md b/dubhe-server/README.md index a2b4755..1f6a8ed 100644 --- a/dubhe-server/README.md +++ b/dubhe-server/README.md @@ -1,108 +1,64 @@ -# 之江天枢-服务端 +# Spring Cloud +## 微服务框架核心组件 +Nacos + Fegin + Gateway + （Spring Security + JWT + OAuth2） -**之江天枢一站式人工智能开源平台**（简称：**之江天枢**），包括海量数据处理、交互式模型构建（包含Notebook和模型可视化）、AI模型高效训练。多维度产品形态满足从开发者到大型企业的不同需求，将提升人工智能技术的研发效率、扩大算法模型的应用范围，进一步构建人工智能生态“朋友圈”。 +数据库连接池 Druid -## 源码部署 +需要额外部署 Mysql Nacos -### 准备环境 -安装如下软件环境。 -- OpenJDK：1.8+ -- Redis: 5.0+ -- Maven: 3.0+ -- MYSQL: 5.7.0+ +## 初始化配置 +### Mysql -### 下载源码 -``` bash -git clone https://codeup.teambition.com/zhejianglab/dubhe-server.git -# 进入项目根目录 -cd dubhe-server -``` +初始化sql位置 /sql -### 创建DB -在MySQL中依次执行如下sql文件 -``` -sql/v1/00-Dubhe-DB.sql -sql/v1/01-Dubhe-DDL.sql -sql/v1/02-Dubhe-DML.sql -``` +**地址：** 127.0.0.1:3306 +**用户名：** test **密码：** test -### 配置 -根据实际情况修改如下配置文件。 -``` -dubhe-admin/src/main/resources/config/application-prod.yml -``` +### Nacos -### 构建 -``` bash -# 构建，生成的 jar 包位于 ./dubhe-admin/target/dubhe-admin-1.0.jar -mvn clean compile package -``` +**如何部署：** https://nacos.io/zh-cn/docs/quick-start.html -### 启动 -``` bash -# 指定启动环境为 prod -## admin模块 -java -jar ./dubhe-admin/target/dubhe-admin-1.0-exec.jar --spring.profiles.active=prod +配置中心配置参考 /yaml -## task模块 -java -jar ./dubhe-task/target/dubhe-task-1.0.jar --spring.profiles.active=prod +**配置规则：** ${prefix}-${spring.profiles.active}.${file-extension} -## serving gateway模块 -java -jar ./dubhe-serving-gateway/target/dubhe-serving-gateway.jar --spring.profiles.active=prod -``` +**详见：** https://nacos.io/zh-cn/docs/quick-start-spring-cloud.html + +**地址：** http://127.0.0.1:8848/nacos/#/login + +**用户名:** nacos **密码：** nacos -## 本地开发 +开发人员进行开发自测时，请使用以自己名字命名的namespace进行测试，不要使用dev或者test -### 必要条件： - 导入maven项目，下载所需的依赖包 - mysql下创建数据库dubhe，初始化数据脚本 - 安装redis -### 启动： - mvn spring-boot:run +### swagger +默认开启,若需要关闭，手动配置swagger.enabled: false -## 代码结构： +**各模块swagger访问地址：** http://{IP}:{port}/doc.html + +**可通过swagger统一的网关访问地址：** http://{gateway IP}:{gateway port}/doc.html查看基于gateway路由配置的后台rest服务 + +### OAuth2 +授权token获取样例： +*POST* http://localhost:8866/oauth/token?grant_type=password&username=admin&client_id=dubhe-client&client_secret=dubhe-secret&password=123456&scope=all + +请求资源时在header添加： +Authorization: 'Bearer '+${access_token} + +测试服务提供者配置中心动态获取配置样例： +header 添加-> Authorization：'Bearer '+${access_token} +*GET* http://localhost:8860/config/get + +刷新token样例： +*POST* http://localhost:8866/oauth/token?grant_type=refresh_token&client_id=dubhe-client&client_secret=dubhe-secret&scope=all&refresh_token=${refresh_token} + +### Admin +登录接口 +```$xslt +url: +*POST* http://localhost:8870/auth/login +param: +{"username":"admin","password":"RBb2Czac2HBI9XNj4ZLF1QcTytOe5pN1vHZHYuAVgSAPRcYbndn/4zGDxKdXS1j0sLsDsKZLUojEXFnYHpsKxA==","code":"jggg","uuid":"validate_codeea991a3cb8ea47cca05744a47ad17a37"} ``` -├── common 公共模块 -├── dubhe-admin 开发与训练模块 -│   ├── src -│   │   └── main -│   │   ├── java -│   │   │   └── org -│   │   │   └── dubhe -│   │   │   ├── AppRun.java -│   │   │   ├── domain 实体对象 -│   │   │   ├── repository 数据库层 -│   │   │   ├── rest 控制层 -│   │   │   └── service 服务层 -│   │   │   ├── dto 数据传输对象 -│   │   │   ├── impl 服务实现 -│   │   │   └── mapper 对象转化 -│   │   └── resources 配置文件 -├── dubhe-data 数据处理模块 -├── dubhe-model 模型管理模块 -├── dubhe-system 系统管理 -├── dubhe-task 定时任务模块 -├── dubhe-serving 云端serving模块 -├── dubhe-serving-gateway 云端serving gateway网关模块 -``` - -## docker服务器 - 上传镜像功能依赖docker服务，harbor与dokcer的信任配置如下： -### 1、对外开放端口 - vi /lib/systemd/system/docker.service - ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock -H tcp://0.0.0.0:2375 -H unix://var/run/docker.sock -### 2、信任harbor地址 - vi /etc/docker/daemon.json - { - "exec-opts": ["native.cgroupdriver=systemd"], - "log-driver": "json-file", - "insecure-registries":[harbor地址], - "log-opts": { - "max-size": "100m" - } - } -### 3、重新启动 - systemctl daemon-reload - service docker restart - systemctl status docker + + diff --git a/dubhe-server/common/pom.xml b/dubhe-server/common/pom.xml deleted file mode 100644 index 008f43c..0000000 --- a/dubhe-server/common/pom.xml +++ /dev/null @@ -1,120 +0,0 @@ - - - - dubheplatform - zhejianglab - 1.0 - - 4.0.0 - - 5.0.6 - 7.0.2 - 1.8.9 - - - common - 公共模块 - - - - - cn.hutool - hutool-all - ${hutool.version} - - - com.google.guava - guava - 21.0 - - - - org.apache.shiro - shiro-spring - ${shiro.version} - - - com.auth0 - java-jwt - ${jwt.version} - - - * - * - - - - - io.minio - minio - ${minio.version} - - - - - org.aspectj - aspectjweaver - ${aspectjweaver.version} - - - - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 - - - commons-io - commons-io - 1.3.2 - compile - - - net.lingala.zip4j - zip4j - 1.3.2 - - - org.apache.commons - commons-compress - 1.20 - - - com.github.whvcse - easy-captcha - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - - true - src/main/resources - - - - diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java deleted file mode 100644 index f4193f9..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/ApiVersion.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.annotation; - -import java.lang.annotation.*; - -/** - * @description API版本控制注解 - * @date 2020-04-06 - */ -@Target({ElementType.METHOD, ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface ApiVersion { - //标识版本号 - int value() default 1; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java deleted file mode 100644 index 0ed9b97..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermission.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.annotation; - -/** - * @description 数据权限注解 - * @date 2020-09-24 - */ -import java.lang.annotation.*; - -@Target({ElementType.METHOD, ElementType.TYPE}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface DataPermission { - - /** - * 只在类的注解上使用，代表方法的数据权限类型 - * @return - */ - String permission() default ""; - - /** - * 不需要数据权限的方法名 - * @return - */ - String[] ignoresMethod() default {}; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java deleted file mode 100644 index a7ba413..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/DataPermissionMethod.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.annotation; - -import org.dubhe.enums.DatasetTypeEnum; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * @description 数据权限方法注解 - * @date 2020-09-24 - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface DataPermissionMethod { - - /** - * 是否需要拦截标识 true: 不拦截 false: 拦截 - * - * @return 拦截标识 - */ - boolean interceptFlag() default false; - - /** - * 数据类型 - * - * @return 数据集类型 - */ - DatasetTypeEnum dataType() default DatasetTypeEnum.PRIVATE; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java deleted file mode 100644 index ddff6b1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/EnumValue.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.annotation; - -import javax.validation.Constraint; -import javax.validation.ConstraintValidator; -import javax.validation.ConstraintValidatorContext; -import javax.validation.Payload; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -/** - * @description 接口枚举类检测标注类 - * @date 2020-05-21 - */ -@Target({ ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE }) -@Retention(RetentionPolicy.RUNTIME) -@Constraint(validatedBy = EnumValue.Validator.class) -public @interface EnumValue { - - String message() default "custom.value.invalid"; - - Class[] groups() default {}; - - Class[] payload() default {}; - - Class> enumClass(); - - String enumMethod(); - - class Validator implements ConstraintValidator { - - private Class> enumClass; - private String enumMethod; - - @Override - public void initialize(EnumValue enumValue) { - enumMethod = enumValue.enumMethod(); - enumClass = enumValue.enumClass(); - } - - @Override - public boolean isValid(Object value, ConstraintValidatorContext constraintValidatorContext) { - if (value == null) { - return Boolean.TRUE; - } - - if (enumClass == null || enumMethod == null) { - return Boolean.TRUE; - } - Class valueClass = value.getClass(); - - try { - Method method = enumClass.getMethod(enumMethod, valueClass); - if (!Boolean.TYPE.equals(method.getReturnType()) && !Boolean.class.equals(method.getReturnType())) { - throw new RuntimeException(String.format("%s method return is not boolean type in the %s class", enumMethod, enumClass)); - } - - Boolean result = (Boolean)method.invoke(null, value); - return result == null ? false : result; - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - throw new RuntimeException(e); - } catch (NoSuchMethodException | SecurityException e) { - throw new RuntimeException(String.format("This %s(%s) method does not exist in the %s", enumMethod, valueClass, enumClass), e); - } - } - - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java deleted file mode 100644 index 8821c91..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/FlagValidator.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.annotation; - -import javax.validation.Constraint; -import javax.validation.ConstraintValidator; -import javax.validation.ConstraintValidatorContext; -import javax.validation.Payload; -import java.lang.annotation.*; -import java.util.Arrays; - -/** - * @description 自定义状态校验注解(传入值是否在指定状态范围内) - * @date 2020-09-18 - */ -@Target({ElementType.FIELD, ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -@Constraint(validatedBy = FlagValidator.Validator.class) -@Documented -public @interface FlagValidator { - - String[] value() default {}; - - String message() default "flag value is invalid"; - - Class[] groups() default {}; - - Class[] payload() default {}; - - /** - * @description 校验传入值是否在默认值范围校验逻辑 - * @date 2020-09-18 - */ - class Validator implements ConstraintValidator { - - private String[] values; - - @Override - public void initialize(FlagValidator flagValidator) { - this.values = flagValidator.value(); - } - - @Override - public boolean isValid(Integer value, ConstraintValidatorContext constraintValidatorContext) { - if (value == null) { - //当状态为空时，使用默认值 - return false; - } - return Arrays.stream(values).anyMatch(value::equals); - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java deleted file mode 100644 index 15714ae..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/Log.java +++ /dev/null @@ -1,23 +0,0 @@ - /** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.dubhe.annotation; - -/** - * @description 日志 - * @date 2020-03-15 - */ -public @interface Log { -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java deleted file mode 100644 index 40e1b77..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/Query.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * @description 构建Wrapper的注解 - * @date 2020-03-26 - */ -@Target(ElementType.FIELD) -@Retention(RetentionPolicy.RUNTIME) -public @interface Query { - - - String propName() default ""; - - Type type() default Type.EQ; - - - String blurry() default ""; - - enum Type { - // 相等 - EQ - // 不等于 - , NE - // 大于 - , GT - // 大于等于 - , GE - // 小于 - , LT - // 小于等于 - , LE, - BETWEEN, - NOT_BETWEEN, - LIKE, - NOT_LIKE, - LIkE_LEFT, - LIKE_RIGHT, - IS_NULL, - IS_NOT_NULL, - IN, - NOT_IN, - INSQL, - NOT_INSQL, - ORDER_BY - } - -} - diff --git a/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java b/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java deleted file mode 100644 index 87001dc..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/annotation/RateLimit.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.annotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * @description 限流 - * @date 2020-03-15 - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface RateLimit { - double limitNum() default 20; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java b/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java deleted file mode 100755 index 172eae2..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/aspect/LogAspect.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.aspect; - -import lombok.extern.slf4j.Slf4j; -import org.aspectj.lang.JoinPoint; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.LogUtil; -import org.slf4j.MDC; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import java.util.UUID; - -/** - * @description 日志切面 - * @date 2020-04-10 - */ -@Component -@Aspect -@Slf4j -public class LogAspect { - - public static final String TRACE_ID = "traceId"; - - @Pointcut("execution(* org.dubhe..service..*.*(..))) " - + "and execution(* org.dubhe..schedule..*.*(..))) " - + "and execution(* org.dubhe..util*..*.*(..)))") - public void serviceAspect() { - } - - @Pointcut("execution(* org.dubhe..rest..*.*(..))) ") - public void restAspect() { - } - - @Pointcut(" execution(* org.dubhe..task..*.*(..))) ") - public void taskAspect() { - } - - @Pointcut(" serviceAspect() ") - public void aroundAspect() { - } - - @Around("aroundAspect()") - public Object around(JoinPoint joinPoint) throws Throwable { - if (StringUtils.isEmpty(MDC.get(TRACE_ID))) { - MDC.put(TRACE_ID, UUID.randomUUID().toString()); - } - return combineLogInfo(joinPoint); - } - - @Around("restAspect()") - public Object aroundRest(JoinPoint joinPoint) throws Throwable { - MDC.clear(); - MDC.put(TRACE_ID, UUID.randomUUID().toString()); - return combineLogInfo(joinPoint); - } - - private Object combineLogInfo(JoinPoint joinPoint) throws Throwable { - Object[] param = joinPoint.getArgs(); - LogUtil.info(LogEnum.LOG_ASPECT, "uri:{},input:{},==>begin", joinPoint.getSignature(), param); - long start = System.currentTimeMillis(); - Object result = ((ProceedingJoinPoint) joinPoint).proceed(); - long end = System.currentTimeMillis(); - LogUtil.info(LogEnum.LOG_ASPECT, "uri:{},output:{},proc_time:{},<==end", joinPoint.getSignature().toString(), - result, end - start); - return result; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java b/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java deleted file mode 100644 index c6c8ef6..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/aspect/PermissionAspect.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.aspect; - -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.aspectj.lang.reflect.MethodSignature; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.base.BaseService; -import org.dubhe.base.DataContext; -import org.dubhe.domain.dto.CommonPermissionDataDTO; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.LogUtil; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Method; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -/** - * @description 数据权限切面 - * @date 2020-09-24 - */ -@Aspect -@Component -public class PermissionAspect { - - /** - * 公共数据的有用户ID - */ - public static final Long PUBLIC_DATA_USER_ID = 0L; - - /** - * 基于注解的切面方法 - */ - @Pointcut("@annotation(org.dubhe.annotation.DataPermissionMethod)") - private void cutMethod() { - - } - - /** - *环绕通知 - * @param joinPoint 切入参数对象 - * @return 返回方法结果集 - * @throws Throwable - */ - @Around("cutMethod()") - public Object around(ProceedingJoinPoint joinPoint) throws Throwable { - // 获取方法传入参数 - Object[] params = joinPoint.getArgs(); - DataPermissionMethod dataPermissionMethod = getDeclaredAnnotation(joinPoint); - - if (!Objects.isNull(JwtUtils.getCurrentUserDto()) && !Objects.isNull(dataPermissionMethod)) { - Set ids = new HashSet<>(); - ids.add(JwtUtils.getCurrentUserDto().getId()); - CommonPermissionDataDTO commonPermissionDataDTO = CommonPermissionDataDTO.builder().type(dataPermissionMethod.interceptFlag()).resourceUserIds(ids).build(); - if (DatasetTypeEnum.PUBLIC.equals(dataPermissionMethod.dataType())) { - ids.add(PUBLIC_DATA_USER_ID); - commonPermissionDataDTO.setResourceUserIds(ids); - } - DataContext.set(commonPermissionDataDTO); - } - // 执行源方法 - try { - return joinPoint.proceed(params); - } finally { - // 模拟进行验证 - BaseService.removeContext(); - } - } - - /** - * 获取方法中声明的注解 - * - * @param joinPoint 切入参数对象 - * @return DataPermissionMethod 方法注解类型 - */ - public DataPermissionMethod getDeclaredAnnotation(ProceedingJoinPoint joinPoint){ - // 获取方法名 - String methodName = joinPoint.getSignature().getName(); - // 反射获取目标类 - Class targetClass = joinPoint.getTarget().getClass(); - // 拿到方法对应的参数类型 - Class[] parameterTypes = ((MethodSignature) joinPoint.getSignature()).getParameterTypes(); - // 根据类、方法、参数类型（重载）获取到方法的具体信息 - Method objMethod = null; - try { - objMethod = targetClass.getMethod(methodName, parameterTypes); - } catch (NoSuchMethodException e) { - LogUtil.error(LogEnum.BIZ_DATASET,"获取注解方法参数异常 error:{}",e); - } - // 拿到方法定义的注解信息 - DataPermissionMethod annotation = objMethod.getDeclaredAnnotation(DataPermissionMethod.class); - // 返回 - return annotation; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java deleted file mode 100644 index 4b75c40..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseConvert.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.base; - -import java.util.List; - -/** - * @description DTO Entity 转换 - * @date 2020-03-15 - */ -public interface BaseConvert { - - /** - * DTO转Entity - * - * @param dto / - * @return / - */ - E toEntity(D dto); - - /** - * Entity转DTO - * - * @param entity / - * @return / - */ - D toDto(E entity); - - /** - * DTO集合转Entity集合 - * - * @param dtoList / - * @return / - */ - List toEntity(List dtoList); - - /** - * Entity集合转DTO集合 - * - * @param entityList / - * @return / - */ - List toDto(List entityList); - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java deleted file mode 100644 index e91a107..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -import lombok.Getter; -import lombok.Setter; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description DTO基础类 - * @date 2020-03-15 - */ -@Getter -@Setter -public class BaseDTO implements Serializable { - - private Boolean deleted; - - private Timestamp createTime; - - private Timestamp updateTime; - - @Override - public String toString() { - return "BaseDTO{" + - "deleted=" + deleted + - ", createTime=" + createTime + - ", updateTime=" + updateTime + - '}'; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java deleted file mode 100644 index 6c949b4..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseEntity.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -import com.baomidou.mybatisplus.annotation.FieldFill; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableLogic; -import lombok.Data; -import org.apache.commons.lang3.builder.ToStringBuilder; - -import java.io.Serializable; -import java.lang.reflect.Field; -import java.sql.Timestamp; - -/** - * @description Entity基础类 - * @date 2020-03-15 - */ -@Data -public class BaseEntity implements Serializable { - - private static final long serialVersionUID = 4936056317364745513L; - - /** - * 删除标识 - **/ - @TableField(value = "deleted",fill = FieldFill.INSERT) - @TableLogic - private Boolean deleted = false; - - @TableField(value = "create_user_id",fill = FieldFill.INSERT) - private Long createUserId; - - @TableField(value = "update_user_id",fill = FieldFill.INSERT_UPDATE) - private Long updateUserId; - - @TableField(value = "create_time",fill = FieldFill.INSERT) - private Timestamp createTime; - - - @TableField(value = "update_time",fill = FieldFill.INSERT_UPDATE) - private Timestamp updateTime; - - @Override - public String toString() { - ToStringBuilder builder = new ToStringBuilder(this); - Field[] fields = this.getClass().getDeclaredFields(); - try { - for (Field f : fields) { - f.setAccessible(true); - builder.append(f.getName(), f.get(this)).append("\n"); - } - } catch (Exception e) { - builder.append("toString builder encounter an error"); - } - return builder.toString(); - } - - public @interface Update { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java deleted file mode 100644 index 57da55c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseImageDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.base; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotBlank; -import java.io.Serializable; - -/** - * @description 镜像基础类DTO - * @date 2020-07-14 - */ -@Data -@Accessors(chain = true) -public class BaseImageDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "镜像版本", required = true) - @NotBlank(message = "镜像版本不能为空") - private String imageTag; - - @ApiModelProperty(value = "镜像名称", required = true) - @NotBlank(message = "镜像名称不能为空") - private String imageName; - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java deleted file mode 100644 index 310b673..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseService.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.base; - -import org.dubhe.constant.PermissionConstant; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.Role; -import org.dubhe.exception.BaseErrorCode; -import org.dubhe.exception.BusinessException; -import org.dubhe.utils.JwtUtils; -import org.springframework.util.CollectionUtils; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @description 服务层基础数据公共方法类 - * @date 2020-03-27 - */ -public class BaseService { - - private BaseService (){} - - /** - * 校验是否具有管理员权限 - */ - public static void checkAdminPermission() { - if(!isAdmin()){ - throw new BusinessException(BaseErrorCode.DATASET_ADMIN_PERMISSION_ERROR); - } - } - - /** - * 校验是否是管理管理员 - * - * @return 校验标识 - */ - public static Boolean isAdmin() { - UserDTO currentUserDto = JwtUtils.getCurrentUserDto(); - if (currentUserDto != null && !CollectionUtils.isEmpty(currentUserDto.getRoles())) { - List roles = currentUserDto.getRoles(); - List roleList = roles.stream(). - filter(a -> a.getId().compareTo(PermissionConstant.ADMIN_USER_ID) == 0) - .collect(Collectors.toList()); - if (!CollectionUtils.isEmpty(roleList)) { - return true; - } - } - return false; - } - - - /** - * 清除本地线程数据权限数据 - */ - public static void removeContext(){ - if( !Objects.isNull(DataContext.get())){ - DataContext.remove(); - } - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java b/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java deleted file mode 100644 index c347515..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/BaseVO.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description VO基础类 - * @date 2020-05-22 - */ -@Data -public class BaseVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("创建人") - private Long createUserId; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("修改人") - private Long updateUserId; - - @ApiModelProperty("修改时间") - private Timestamp updateTime; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java b/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java deleted file mode 100644 index 3223c90..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/DataContext.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - - -import org.dubhe.domain.dto.CommonPermissionDataDTO; - -/** - * @description 共享上下文数据集信息 - * @date 2020-04-10 - */ -public class DataContext { - - /** - * 私有化构造参数 - */ - private DataContext() { - } - - private static final ThreadLocal CONTEXT = new ThreadLocal<>(); - - /** - * 存放数据集信息 - * - * @param datasetVO - */ - public static void set(CommonPermissionDataDTO datasetVO) { - CONTEXT.set(datasetVO); - } - - /** - * 获取用户信息 - * - * @return - */ - public static CommonPermissionDataDTO get() { - return CONTEXT.get(); - } - - /** - * 清除当前线程内引用，防止内存泄漏 - */ - public static void remove() { - CONTEXT.remove(); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java b/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java deleted file mode 100644 index 6699252..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/DataResponseBody.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.slf4j.MDC; - -import java.io.Serializable; - -/** - * @description 统一的公共响应体 - * @date 2020-03-16 - */ -@Data -@ApiModel(description = "统一的公共响应体") -public class DataResponseBody implements Serializable { - - /** - * 返回状态码 - */ - @ApiModelProperty("返回状态码") - private Integer code; - /** - * 返回信息 - */ - @ApiModelProperty("返回信息") - private String msg; - /** - * 泛型数据 - */ - @ApiModelProperty("泛型数据") - private T data; - /** - * 链路追踪ID - */ - @ApiModelProperty("链路追踪ID") - private String traceId; - - public DataResponseBody() { - this(ResponseCode.SUCCESS, null); - } - - public DataResponseBody(T data) { - this(ResponseCode.SUCCESS, null, data); - } - - public DataResponseBody(Integer code, String msg) { - this(code, msg, null); - } - - public DataResponseBody(Integer code, String msg, T data) { - this.code = code; - this.msg = msg; - this.data = data; - this.traceId = MDC.get("traceId"); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java b/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java deleted file mode 100644 index 84e7c4a..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/MagicNumConstant.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -/** - * @description 常用常量类 - * @date 2020-05-14 - */ -public final class MagicNumConstant { - - public static final int NEGATIVE_ONE = -1; - public static final int ZERO = 0; - public static final int ONE = 1; - public static final int TWO = 2; - public static final int THREE = 3; - public static final int FOUR = 4; - public static final int FIVE = 5; - public static final int SIX = 6; - public static final int SEVEN = 7; - public static final int EIGHT = 8; - public static final int NINE = 9; - public static final int TEN = 10; - - public static final int ELEVEN = 11; - public static final int SIXTEEN = 16; - public static final int TWENTY = 20; - public static final int FIFTY = 50; - public static final int SIXTY = 60; - public static final int SIXTY_TWO = 62; - public static final int SIXTY_FOUR = 64; - public static final int INTEGER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; - public static final int ONE_HUNDRED = 100; - public static final int ONE_HUNDRED_TWENTY_EIGHT = 128; - public static final int TWO_HUNDRED = 200; - public static final int FIVE_HUNDRED = 500; - public static final int FIVE_HUNDRED_AND_SIXTEEN = 516; - public static final int ONE_THOUSAND = 1000; - public static final int BINARY_TEN_EXP = 1024; - public static final int ONE_THOUSAND_ONE_HUNDRED = 1100; - public static final int ONE_THOUSAND_ONE_HUNDRED_ONE = 1101; - public static final int ONE_THOUSAND_TWO_HUNDRED = 1200; - public static final int ONE_THOUSAND_TWO_HUNDRED_ONE = 1201; - public static final int ONE_THOUSAND_TWENTY_FOUR = 1024; - public static final int ONE_THOUSAND_THREE_HUNDRED = 1300; - public static final int ONE_THOUSAND_THREE_HUNDRED_ONE = 1301; - public static final int ONE_THOUSAND_THREE_HUNDRED_NINE = 1309; - public static final int ONE_THOUSAND_FIVE_HUNDRED = 1500; - public static final int TWO_THOUSAND = 2000; - public static final int TWO_THOUSAND_TWENTY_EIGHT = 2048; - public static final int THREE_THOUSAND = 3000; - public static final int FOUR_THOUSAND = 4000; - public static final int NINE_THOUSAND = 9000; - public static final int NINE_THOUSAND_NINE_HUNDRED_NINTY_NINE = 9999; - public static final int TEN_THOUSAND = 10000; - public static final int FIFTEEN_THOUSAND = 15000; - public static final int HUNDRED_THOUSAND = 100000; - public static final int MILLION = 1000000; - public static final int ONE_MINUTE = 60000; - - public static final long NEGATIVE_ONE__LONG = -1L; - public static final long ZERO_LONG = 0L; - public static final long ONE_LONG = 1L; - public static final long TWO_LONG = 2L; - public static final long THREE_LONG = 3L; - public static final long FOUR_LONG = 4L; - public static final long FIVE_LONG = 5L; - public static final long SIX_LONG = 6L; - public static final long SEVEN_LONG = 7L; - public static final long EIGHT_LONG = 8L; - public static final long NINE_LONG = 9L; - public static final long TEN_LONG = 10L; - - public static final long TWELVE_LONG = 12L; - public static final long SIXTY_LONG = 60L; - public static final long THOUSAND_LONG = 1000L; - public static final long TEN_THOUSAND_LONG = 10000L; - public static final long ONE_ZERO_ONE_ZERO_ONE_ZERO_LONG = 101010L; - public static final long NINE_ZERO_NINE_ZERO_NINE_ZERO_LONG = 909090L; - public static final long ONE_YEAR_BEFORE_LONG = 1552579200000L; - - public static final int SIXITY_0XFF = 0xFF; - - - private MagicNumConstant() { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java b/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java deleted file mode 100644 index ce56dd3..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/PageQueryBase.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.constant.NumberConstant; - -import javax.validation.constraints.Min; - -/** - * @description 分页基类 - * @date 2020-05-08 - */ -@Data -@Accessors(chain = true) -public class PageQueryBase { - - @ApiModelProperty(value = "分页-当前页数") - @Min(value = 1, message = "current不能小于1") - private Integer current; - - @ApiModelProperty(value = "分页-每页展示数") - @Min(value = 1, message = "size不能小于1") - private Integer size; - - - @ApiModelProperty(value = "排序字段") - private String sort; - - @ApiModelProperty(value = "排序方式,asc | desc") - private String order; - - public Page toPage() { - Page page = new Page(); - if (this.current != null) { - page.setCurrent(this.current); - } - if (this.size != null && this.size < NumberConstant.MAX_PAGE_SIZE) { - page.setSize(this.size); - } - return page; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java b/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java deleted file mode 100644 index 247a50a..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/ResponseCode.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.base; - -/** - * @description 返回状态码 - * @date 2020-02-23 - */ -public class ResponseCode { - public static Integer SUCCESS = 200; - public static Integer UNAUTHORIZED = 401; - public static Integer ERROR = 10000; - public static Integer ENTITY_NOT_EXIST = 10001; - public static Integer BADREQUEST = 10002; - public static Integer SERVICE_ERROR = 10003; - public static Integer DOCKER_ERROR = 10004; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java b/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java deleted file mode 100644 index a4ade6c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/base/ScheduleTaskHandler.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.base; - -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.LogUtil; - -/** - * @description 定时任务处理器, 主要做日志标识 - * @date 2020-08-13 - */ -public class ScheduleTaskHandler { - - - public static void process(Handler handler) { - LogUtil.startScheduleTrace(); - try { - handler.run(); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_SYS, "There is something wrong in schedule task handler ：{}", e); - } finally { - LogUtil.cleanTrace(); - } - } - - - public interface Handler { - void run(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java deleted file mode 100644 index 45a57c8..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/GlobalCorsConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.cors.CorsConfiguration; -import org.springframework.web.cors.UrlBasedCorsConfigurationSource; -import org.springframework.web.filter.CorsFilter; - -/** - * @description 允许跨域 - * @date 2020-02-23 - */ -@Configuration -public class GlobalCorsConfig { - - @Bean - public CorsFilter corsFilter() { - //1.添加CORS配置信息 - CorsConfiguration config = new CorsConfiguration(); - //放行哪些原始域 - config.addAllowedOrigin("*"); - //是否发送Cookie信息 - config.setAllowCredentials(true); - //放行哪些原始域(请求方式) - config.addAllowedMethod("OPTIONS"); - config.addAllowedMethod("HEAD"); - config.addAllowedMethod("GET"); - config.addAllowedMethod("PUT"); - config.addAllowedMethod("POST"); - config.addAllowedMethod("DELETE"); - config.addAllowedMethod("PATCH"); - config.addAllowedHeader("*"); - - //2.添加映射路径 - UrlBasedCorsConfigurationSource configSource = new UrlBasedCorsConfigurationSource(); - configSource.registerCorsConfiguration("/**", config); - - //3.返回新的CorsFilter. - return new CorsFilter(configSource); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java deleted file mode 100644 index de702d5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/K8sNameConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -/** - * @description k8s命名相关配置 - * - * @date 2020-05-13 - */ -@Getter -@Configuration -public class K8sNameConfig { - - @Value("${k8s.nfs-root-path}") - private String nfsRootPath; - - @Value("${k8s.namespace}") - private String namespace; - - @Value("${minio.bucketName}") - private String fileBucket; - - @Value("${train-job.docker-dataset-path}") - private String datasetPath; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java deleted file mode 100644 index 8fba81e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/LocalDateTimeSerializerConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -@Configuration -public class LocalDateTimeSerializerConfig { - @Value("${spring.jackson.date-format}") - private String pattern; - - @Bean - public LocalDateTimeSerializer localDateTimeDeserializer() { - return new LocalDateTimeSerializer(DateTimeFormatter.ofPattern(pattern)); - } - - @Bean - public Jackson2ObjectMapperBuilderCustomizer jackson2ObjectMapperBuilderCustomizer() { - return builder -> builder.serializerByType(LocalDateTime.class, localDateTimeDeserializer()); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java deleted file mode 100644 index 7202208..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/MetaHandlerConfig.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; -import org.apache.ibatis.reflection.MetaObject; -import org.dubhe.constant.StringConstant; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.enums.SwitchEnum; -import org.dubhe.utils.DateUtil; -import org.dubhe.utils.JwtUtils; -import org.springframework.stereotype.Component; - -import java.util.Objects; - -/** - * @description 处理新增和更新的基础数据填充，配合BaseEntity和MyBatisPlusConfig使用 - * @date 2020-06-10 - */ -@Component -public class MetaHandlerConfig implements MetaObjectHandler { - - - /** - * 新增数据执行 - * - * @param metaObject 基础数据 - */ - @Override - public void insertFill(MetaObject metaObject) { - if (Objects.isNull(getFieldValByName(StringConstant.CREATE_TIME, metaObject))) { - this.setFieldValByName(StringConstant.CREATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) { - this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) { - this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.CREATE_USER_ID, metaObject))) { - this.setFieldValByName(StringConstant.CREATE_USER_ID, getUserId(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.ORIGIN_USER_ID, metaObject))) { - this.setFieldValByName(StringConstant.ORIGIN_USER_ID, getUserId(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.DELETED, metaObject))) { - this.setFieldValByName(StringConstant.DELETED, SwitchEnum.getBooleanValue(SwitchEnum.OFF.getValue()), metaObject); - } - } - - /** - * 更新数据执行 - * - * @param metaObject 基础数据 - */ - @Override - public void updateFill(MetaObject metaObject) { - if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_TIME, metaObject))) { - this.setFieldValByName(StringConstant.UPDATE_TIME, DateUtil.getCurrentTimestamp(), metaObject); - } - if (Objects.isNull(getFieldValByName(StringConstant.UPDATE_USER_ID, metaObject))) { - this.setFieldValByName(StringConstant.UPDATE_USER_ID, getUserId(), metaObject); - } - - } - - - /** - * 获取用户ID - * - * @return - */ - private Long getUserId() { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - return Objects.isNull(userDTO) ? null : userDTO.getId(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java deleted file mode 100644 index 0740c65..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/MybatisPlusConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.dubhe.interceptor.PaginationInterceptor; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.transaction.annotation.EnableTransactionManagement; -/** - * @description mybatis plus拦截器 - * @date 2020-06-10 - */ -@EnableTransactionManagement -@Configuration -public class MybatisPlusConfig { - - /** - * 注入 MybatisPlus 分页拦截器 - * - * @return 自定义MybatisPlus分页拦截器 - */ - @Bean - public PaginationInterceptor paginationInterceptor() { - PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); - return paginationInterceptor; - } -} - diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java deleted file mode 100644 index 2881aa9..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/NfsConfig.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @description NFS config - * @create 2020/5/13 - */ -@Data -@Component -public class NfsConfig { - - @Value("${k8s.nfs}") - private String nfsIp; - - @Value("${k8s.nfs-root-path}") - private String rootDir; - - @Value("/${minio.bucketName}/") - private String bucket; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java deleted file mode 100644 index 1a54aef..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/RecycleConfig.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -/** - * @description 垃圾回收机制配置常量 - * @date 2020-09-21 - */ -@Data -@Component -@ConfigurationProperties(prefix = "recycle.timeout") -public class RecycleConfig { - - /** - * 回收无效文件的默认有效时长 - */ - private Integer date; - - /** - * 用户上传文件至临时路径下后文件最大有效时长，以小时为单位 - */ - private Integer fileValid; - - /** - * 用户删除某一算法后，其算法文件最大有效时长，以天为单位 - */ - private Integer algorithmValid; - - /** - * 用户删除某一模型后，其模型文件最大有效时长，以天为单位 - */ - private Integer modelValid; - - /** - * 用户删除训练任务后，其训练管理文件最大有效时长，以天为单位 - */ - private Integer trainValid; - - /** - * 删除服务器无效文件(大文件) - * 示例：rsync --delete-before -d /空目录 /需要回收的源目录 - */ - public static final String DEL_COMMAND = "ssh %s@%s \"mkdir -p %s; rsync --delete-before -d %s %s; rmdir %s %s\""; -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java deleted file mode 100644 index f76f43d..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/RedisConfig.java +++ /dev/null @@ -1,214 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import cn.hutool.core.lang.Assert; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.parser.ParserConfig; -import com.alibaba.fastjson.serializer.SerializerFeature; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.dubhe.utils.StringUtils; -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.data.redis.RedisProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cache.Cache; -import org.springframework.cache.annotation.CachingConfigurerSupport; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.interceptor.CacheErrorHandler; -import org.springframework.cache.interceptor.KeyGenerator; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.cache.RedisCacheConfiguration; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.RedisOperations; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.serializer.RedisSerializationContext; -import org.springframework.data.redis.serializer.RedisSerializer; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -/** - * @description redis配置类 - * @date 2020-03-25 - */ -@Slf4j -@Configuration -@EnableCaching -@ConditionalOnClass(RedisOperations.class) -@EnableConfigurationProperties(RedisProperties.class) -public class RedisConfig extends CachingConfigurerSupport { - - /** - * 设置 redis 数据默认过期时间，默认2小时 - * 设置@cacheable 序列化方式 - */ - @Bean - public RedisCacheConfiguration redisCacheConfiguration() { - FastJsonRedisSerializer fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class); - RedisCacheConfiguration configuration = RedisCacheConfiguration.defaultCacheConfig(); - configuration = configuration.serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(fastJsonRedisSerializer)).entryTtl(Duration.ofHours(2)); - return configuration; - } - - @SuppressWarnings("all") - @Bean(name = "redisTemplate") - @ConditionalOnMissingBean(name = "redisTemplate") - public RedisTemplate redisTemplate(RedisConnectionFactory redisConnectionFactory) { - RedisTemplate template = new RedisTemplate<>(); - //序列化 - FastJsonRedisSerializer fastJsonRedisSerializer = new FastJsonRedisSerializer<>(Object.class); - // value值的序列化采用fastJsonRedisSerializer - template.setValueSerializer(fastJsonRedisSerializer); - template.setHashValueSerializer(fastJsonRedisSerializer); - // 全局开启AutoType，这里方便开发，使用全局的方式 - ParserConfig.getGlobalInstance().setAutoTypeSupport(true); - // 建议使用这种方式，小范围指定白名单 - // ParserConfig.getGlobalInstance().addAccept("org.dubhe.domain"); - // key的序列化采用StringRedisSerializer - template.setKeySerializer(new StringRedisSerializer()); - template.setHashKeySerializer(new StringRedisSerializer()); - template.setConnectionFactory(redisConnectionFactory); - return template; - } - - /** - * 自定义缓存key生成策略，默认将使用该策略 - */ - @Bean - @Override - public KeyGenerator keyGenerator() { - return (target, method, params) -> { - Map container = new HashMap<>(3); - Class targetClassClass = target.getClass(); - // 类地址 - container.put("class", targetClassClass.toGenericString()); - // 方法名称 - container.put("methodName", method.getName()); - // 包名称 - container.put("package", targetClassClass.getPackage()); - // 参数列表 - for (int i = 0; i < params.length; i++) { - container.put(String.valueOf(i), params[i]); - } - // 转为JSON字符串 - String jsonString = JSON.toJSONString(container); - // 做SHA256 Hash计算，得到一个SHA256摘要作为Key - return DigestUtils.sha256Hex(jsonString); - }; - } - - @Bean - @Override - public CacheErrorHandler errorHandler() { - // 异常处理，当Redis发生异常时，打印日志，但是程序正常走 - log.info("初始化 -> [{}]", "Redis CacheErrorHandler"); - return new CacheErrorHandler() { - @Override - public void handleCacheGetError(RuntimeException e, Cache cache, Object key) { - log.error("Redis occur handleCacheGetError：key -> [{}]", key, e); - } - - @Override - public void handleCachePutError(RuntimeException e, Cache cache, Object key, Object value) { - log.error("Redis occur handleCachePutError：key -> [{}]；value -> [{}]", key, value, e); - } - - @Override - public void handleCacheEvictError(RuntimeException e, Cache cache, Object key) { - log.error("Redis occur handleCacheEvictError：key -> [{}]", key, e); - } - - @Override - public void handleCacheClearError(RuntimeException e, Cache cache) { - log.error("Redis occur handleCacheClearError：", e); - } - }; - } - -} - -/** - * Value 序列化 - * - * @param - */ -class FastJsonRedisSerializer implements RedisSerializer { - - private Class clazz; - - FastJsonRedisSerializer(Class clazz) { - super(); - this.clazz = clazz; - } - - @Override - public byte[] serialize(T t) { - if (t == null) { - return new byte[0]; - } - return JSON.toJSONString(t, SerializerFeature.WriteClassName).getBytes(StandardCharsets.UTF_8); - } - - @Override - public T deserialize(byte[] bytes) { - if (bytes == null || bytes.length <= 0) { - return null; - } - String str = new String(bytes, StandardCharsets.UTF_8); - return JSON.parseObject(str, clazz); - } - -} - -/** - * 重写序列化器 - * - */ -class StringRedisSerializer implements RedisSerializer { - - private final Charset charset; - - StringRedisSerializer() { - this(StandardCharsets.UTF_8); - } - - private StringRedisSerializer(Charset charset) { - Assert.notNull(charset, "Charset must not be null!"); - this.charset = charset; - } - - @Override - public String deserialize(byte[] bytes) { - return (bytes == null ? null : new String(bytes, charset)); - } - - @Override - public byte[] serialize(Object object) { - String string = JSON.toJSONString(object); - if (StringUtils.isBlank(string)) { - return null; - } - string = string.replace("\"", ""); - return string.getBytes(charset); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java b/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java deleted file mode 100644 index e926b95..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/Resources.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.springframework.context.annotation.PropertySource; -import org.springframework.context.i18n.LocaleContextHolder; - -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.ResourceBundle; - -/** - * @description 加载配置 - * @date 2020-03-15 - */ -@PropertySource(value = {"classpath:config/thirdParty.properties", "classpath:i18n/messages*.properties"}) -public final class Resources { - /** - * 第三方登录配置 - */ - public static final ResourceBundle THIRDPARTY = ResourceBundle.getBundle("config/thirdParty"); - /** - * 国际化信息 - */ - private static final Map MESSAGES = new HashMap(); - - /** - * 国际化信息 - */ - public static String getMessage(String key, Object... params) { - Locale locale = LocaleContextHolder.getLocale(); - ResourceBundle message = MESSAGES.get(locale.getLanguage()); - if (message == null) { - synchronized (MESSAGES) { - message = MESSAGES.get(locale.getLanguage()); - if (message == null) { - message = ResourceBundle.getBundle("i18n/messages", locale); - MESSAGES.put(locale.getLanguage(), message); - } - } - } - if (params != null && params.length > 0) { - return String.format(message.getString(key), params); - } - return message.getString(key); - } - - /** - * 清除国际化信息 - */ - public static void flushMessage() { - MESSAGES.clear(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java deleted file mode 100644 index 5f8c817..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/SecurityConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; - -/** - * @description 关闭Security - * @date 2020-03-25 - */ -@Configuration -public class SecurityConfig extends WebSecurityConfigurerAdapter { - - @Override - protected void configure(HttpSecurity http) throws Exception { - http.authorizeRequests().antMatchers("/**").permitAll().and().csrf().disable(); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java deleted file mode 100644 index e4a4ebd..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/SwaggerConfig.java +++ /dev/null @@ -1,138 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import com.fasterxml.classmate.TypeResolver; -import com.google.common.base.Predicates; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.Ordered; -import org.springframework.data.domain.Pageable; -import springfox.documentation.builders.ApiInfoBuilder; -import springfox.documentation.builders.ParameterBuilder; -import springfox.documentation.builders.PathSelectors; -import springfox.documentation.schema.AlternateTypeRule; -import springfox.documentation.schema.AlternateTypeRuleConvention; -import springfox.documentation.schema.ModelRef; -import springfox.documentation.service.ApiInfo; -import springfox.documentation.service.Parameter; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; - -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import static com.google.common.collect.Lists.newArrayList; -import static springfox.documentation.schema.AlternateTypeRules.newRule; - -/** - * @description api页面 /swagger-ui.html - * @date 2020-03-25 - */ - -@Configuration -@EnableSwagger2 -public class SwaggerConfig { - - @Value("${jwt.header}") - private String tokenHeader; - - @Value("${jwt.token-start-with}") - private String tokenStartWith; - - @Value("${swagger.enabled}") - private Boolean enabled; - - @Value("${server.rest-version}") - private String restVersion; - - @Bean - @SuppressWarnings("all") - public Docket createRestApi() { - List pars = new ArrayList<>(); - pars.add(new ParameterBuilder().name(tokenHeader).description("token") - .modelRef(new ModelRef("string")) - .parameterType("header") - .defaultValue(tokenStartWith + " ") - .required(true) - .build()); - pars.add(new ParameterBuilder().name("version").description("版本") - .modelRef(new ModelRef("string")) - .parameterType("path") - .defaultValue(restVersion) - .required(false) - .build()); - return new Docket(DocumentationType.SWAGGER_2) - .enable(enabled) - .apiInfo(apiInfo()) - .select() - .paths(Predicates.not(PathSelectors.regex("/error.*"))) - .build().directModelSubstitute(Timestamp.class, Date.class) - .globalOperationParameters(pars); - } - - private ApiInfo apiInfo() { - return new ApiInfoBuilder() - .title("之江天枢一站式人工智能开源平台") - .version("1.0") - .build(); - } - -} - -/** - * 将Pageable转换展示在swagger中 - */ -@Configuration -class SwaggerDataConfig { - - @Bean - public AlternateTypeRuleConvention pageableConvention(final TypeResolver resolver) { - return new AlternateTypeRuleConvention() { - @Override - public int getOrder() { - return Ordered.HIGHEST_PRECEDENCE; - } - - @Override - public List rules() { - return newArrayList(newRule(resolver.resolve(Pageable.class), resolver.resolve(Page.class))); - } - }; - } - - @ApiModel - @Data - private static class Page { - @ApiModelProperty("页码 (0..N)") - private Integer page; - - @ApiModelProperty("每页显示的数目") - private Integer size; - - @ApiModelProperty("以下列格式排序标准：property[,asc | desc]。 默认排序顺序为升序。 支持多种排序条件：如：id,asc") - private List sort; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java deleted file mode 100644 index cd4c9ea..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainHarborConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -/** - * @description harbor相关配置 - * @date 2020-07-17 - */ -@Data -@Component -@ConfigurationProperties(prefix = "harbor") -public class TrainHarborConfig { - - private String address; - - private String username; - - private String password; - - private String modelName; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java deleted file mode 100644 index 0565cb1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainJobConfig.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -/** - * @description 训练常量 - * @date 2020-05-12 - */ -@Component -@Data -@ConfigurationProperties(prefix = "train-job") -public class TrainJobConfig { - - private String namespace; - - private String versionLabel; - - private String separator; - - private String podName; - - private String pythonFormat; - - private String manage; - - private String outPath; - - private String logPath; - - private String visualizedLogPath; - - private String dockerDatasetPath; - - private String dockerTrainPath; - - private String dockerOutPath; - - private String dockerLogPath; - - private String dockerDataset; - - private String dockerModelPath; - - private String dockerTeacherModelPath; - - private String dockerTeacherModelKey; - - private String dockerStudentModelKey; - - private String dockerStudentModelPath; - - private String atlasAnaconda; - - private String atlasPythonioencoding; - - private String dockerValDatasetPath; - - private String loadValDatasetKey; - - private String dockerVisualizedLogPath; - - private String loadPath; - - private String loadKey; - - private String eight; - - private String plusEight; - - private String nodeIps; - - private String nodeNum; - - private String gpuNumPerNode; - - public static final String TRAIN_ID = "trainId"; - - public static final String TRAIN_VERSION = "trainVersion"; - - public static final String RUN_TIME = "runtime"; - - public static final String TRAIN_STATUS = "trainStatus"; - - public static final String CREATE_TIME = "createTime"; - - public static final String ALGORITHM_NAME = "algorithmName"; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java deleted file mode 100644 index 93a377e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/TrainPoolConfig.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.config; - -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.LogUtil; -import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.AsyncConfigurer; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; - -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; - -/** - * @description 线程池配置类 - * @date 2020-07-17 - */ -@Configuration -public class TrainPoolConfig implements AsyncConfigurer { - - @Value("${basepool.corePoolSize:40}") - private Integer corePoolSize; - @Value("${basepool.maximumPoolSize:60}") - private Integer maximumPoolSize; - @Value("${basepool.keepAliveTime:120}") - private Integer keepAliveTime; - @Value("${basepool.blockQueueSize:20}") - private Integer blockQueueSize; - - /** - * 训练任务异步处理线程池 - * @return Executor 线程实例 - */ - @Bean("trainExecutor") - @Override - public Executor getAsyncExecutor() { - ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); - //核心线程数 - taskExecutor.setCorePoolSize(corePoolSize); - taskExecutor.setAllowCoreThreadTimeOut(true); - //最大线程数 - taskExecutor.setMaxPoolSize(maximumPoolSize); - //超时时间 - taskExecutor.setKeepAliveSeconds(keepAliveTime); - //配置队列大小 - taskExecutor.setQueueCapacity(blockQueueSize); - //配置线程池前缀 - taskExecutor.setThreadNamePrefix("async-train-"); - //拒绝策略 - taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy()); - taskExecutor.initialize(); - return taskExecutor; - } - - @Override - public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { - LogUtil.error(LogEnum.BIZ_TRAIN, "开始捕获训练管理异步任务异常信息-----》》》"); - return (ex, method, params) -> { - LogUtil.error(LogEnum.BIZ_TRAIN, "训练管理方法名{}的异步任务执行失败，参数信息：{},异常信息：{}", method.getName(), params, ex); - }; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java deleted file mode 100644 index d8bb2a5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.dubhe.interceptor.K8sCallBackPodInterceptor; -import org.dubhe.utils.K8sCallBackTool; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -import javax.annotation.Resource; - -/** - * @description Web Mvc Config - * - * @date 2020-05-28 - */ -@Configuration -public class WebMvcConfig implements WebMvcConfigurer { - - @Resource - private K8sCallBackPodInterceptor k8sCallBackPodInterceptor; - - @Override - public void addInterceptors(InterceptorRegistry registry) { - InterceptorRegistration registration = registry.addInterceptor(k8sCallBackPodInterceptor); - // 拦截配置 - registration.addPathPatterns(K8sCallBackTool.getK8sCallbackPaths()); - - } - - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java b/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java deleted file mode 100644 index 90f6a9e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/config/WebMvcRegistrationsConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.dubhe.handle.CustomRequestMappingHandlerMapping; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.web.servlet.WebMvcRegistrations; -import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; - -/** - * @description API版本控制 - * @date 2020-04-06 - */ -@SpringBootConfiguration -public class WebMvcRegistrationsConfig implements WebMvcRegistrations { - @Override - public RequestMappingHandlerMapping getRequestMappingHandlerMapping() { - return new CustomRequestMappingHandlerMapping(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java deleted file mode 100644 index bd95bce..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/AlgorithmSourceEnum.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Getter; - -/** - * @description 算法枚举类 - * @date 2020-05-12 - */ -@Getter -public enum AlgorithmSourceEnum { - - /** - * MINE 算法来源 我的算法 - */ - MINE(1, "MINE"), - /** - * PRE 算法来源 预置算法 - */ - PRE(2,"PRE"); - - private Integer status; - - private String message; - - AlgorithmSourceEnum(Integer status, String message) { - this.status = status; - this.message = message; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java deleted file mode 100644 index 3b6209e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ErrorMessageConstant.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.constant; - -/** - * @description 状态机异常量类 - * @date 2020-09-01 - */ -public class ErrorMessageConstant { - - private ErrorMessageConstant(){} - - /** - * 数据状态异常信息 - */ - public static final String FILE_CHANGE_ERR_MESSAGE = "当前文件状态不可变更"; - - /** - * 数据状态异常信息 - */ - public static final String DATASET_CHANGE_ERR_MESSAGE = "当前数据集状态不可变更"; - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java deleted file mode 100644 index b191317..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptConstant.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Data; -import org.springframework.stereotype.Component; - -/** - * @description 模型优化常量 - * @date 2020-05-25 - */ -@Component -@Data -public class ModelOptConstant { - - /** - * 模型优化任务结果字典 - */ - public static final String TASK_RESULT_DICT_NAME = "opt_result"; - /** - * 数据集挂载路径 - */ - public static final String DATASET_MOUNT_PATH = "/usr/local/dataset"; - /** - *算法挂载路径 - */ - public static final String ALGORITHM_MOUNT_PATH = "/usr/local/algorithm"; - /** - *输入模型挂载路径 - */ - public static final String INPUT_MODEL_BEFORE_MOUNT_PATH = "/usr/local/input/model"; - /** - *输出日志挂载路径 - */ - public static final String OUTPUT_LOG_MOUNT_PATH = "/usr/local/output/log"; - /** - *压缩前评估结果挂载路径 - */ - public static final String OUTPUT_RESULT_BEFORE_MOUNT_PATH = "/usr/local/output/result/before"; - /** - *输出模型挂载路径 - */ - public static final String OUTPUT_MODEL_MOUNT_PATH = "/usr/local/output/model"; - /** - *压缩后评估结果挂载路径 - */ - public static final String OUTPUT_RESULT_AFTER_MOUNT_PATH = "/usr/local/output/result/after"; - /** - * 使用的CPU配额 - */ - public static final int CPU_NUM = 2048; - /** - * 使用的GPU配额 - */ - public static final int GPU_NUM = 1; - /** - * 使用的内存配额 - */ - public static final int MEMORY_NUM = 8000; - /** - * CNN剪枝算法启动命令 - */ - public static final String OPT_START_SLIMMING_COMMAND = "cd /usr/local/algorithm && python run.py --prune_method=%s --model=%s --data_type=%s --dataset_dir=%s --model_save_dir=%s --log_dir=%s --before_result_dir=%s --after_result_dir=%s"; - /** - * 量化算法启动命令 - */ - public static final String OPT_START_QUANTIFY_COMMAND = "cd /usr/local/algorithm && python3 of_cnn_evaluate.py --model=%s --model_load_dir=%s --val_data_dir=%s --log_dir=%s --result_dir=%s --use_tensorrt=False --use_int8_online=False && python3 of_cnn_evaluate.py --model=%s --model_load_dir=%s --val_data_dir=%s --log_dir=%s --result_dir=%s --use_tensorrt=True --use_int8_online=True"; - /** - * 非内置运行命令 - */ - public static final String MY_OPT_COMMAND = "cd %s && %s"; - /** - * 模型优化结果日志路径 - */ - public static final String OPTIMIZE_LOG = "/log"; - /** - * 模型优化结果保存模型路径 - */ - public static final String OPTIMIZE_MODEL = "/model"; - /** - * 模型优化结果保存优化前模型评估参数 - */ - public static final String OPTIMIZE_JSON_BEFORE = "/before"; - /** - * 模型优化结果保存优化后模型评估参数 - */ - public static final String OPTIMIZE_JSON_AFTER = "/after"; - /** - * 模型优化结果JSON文件名称 - */ - public static final String OPTIMIZE_JSON_NAME = "/results_eval.json"; - /** - * 我的压缩算法根路径 - */ - public static final String MY_OPT_ALGORITHM_ROOT_PATH = "/model-opt/myAlgorithms/"; - /** - * 我的数据集 - */ - public static final String MY_OPT_DATASET_ROOT_PATH = "/model-opt/myDataset/"; - /** - * 算法准确度名称 - */ - public static final String ACCURACY = "accuracy"; - /** - * 复制模型后缀文件夹名称 - */ - public static final String COPY_MODEL_POSTFIX = "/model"; - /** - * 基于神经元权重剪枝 - */ - public static final String NEURONAL_PRUNING = "基于神经元权重剪枝"; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java deleted file mode 100644 index d16073b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptErrorEnum.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Getter; -import org.dubhe.exception.ErrorCode; - -/** - * @description 模型优化错误码 - * @date 2020-05-22 - */ -@Getter -public enum ModelOptErrorEnum implements ErrorCode { - - /** - * 该模型优化任务不存在 - */ - MODEL_OPT_TASK_ABSENT(1400, "该模型优化任务不存在"), - /** - * 该模型优化任务实例不存在 - */ - MODEL_OPT_TASK_INSTANCE_ABSENT(1401, "该模型优化任务实例不存在"), - /** - * 模型优化任务实例创建失败 - */ - MODEL_OPT_TASK_INSTANCE_CREATE_FAILED(1402, "模型优化任务实例创建失败"), - /** - * 模型优化任务名称已存在 - */ - MODEL_OPT_TASK_NAME_EXIST(1403, "模型优化任务名称已存在"), - /** - * 该任务已存在等待或进行中的实例 - */ - MODEL_OPT_TASK_INSTANCE_EXIST(1404, "该任务已存在等待或进行中的实例"), - /** - * 请先取消等待或进行中的实例 - */ - MODEL_OPT_TASK_DELETE_ERROR(1405, "请先取消等待或进行中的实例"), - - /** - * 请先添加模型优化结果字典 - */ - MODEL_OPT_TASK_RESULT_DICT_EMPTY(1406, "请先添加模型优化结果字典"), - - /** - * 模型压缩算法不存在 - */ - MODEL_OPT_ALGORITHM_NOT_EXIST(1407, "模型压缩算法不存在"), - - /** - * 任务未完成，不能下载 - */ - MODEL_OPT_TASK_UNFINISHED(1408, "任务未完成，不能下载"), - - /** - * 数据集不存在，请选择正确的数据集 - */ - MODEL_OPT_DATASET_ABSENT(1409, "数据集不存在，请选择正确的数据集"), - - /** - * 当前状态不能进行此操作 - */ - MODEL_OPT_TASK_INSTANCE_STATUS_ERROR(1410, "当前状态不能进行此操作"), - - /** - * 模型不存在 - */ - MODEL_OPT_MODEL_NOT_EXIST(1411, "模型不存在"), - - /** - * 请先登录 - */ - NO_USER_LOGGED_IN(1412, "请先登录"), - - /** - * 当前用户无此操作权限 - */ - NO_PERMISSION(1413, "当前用户无此操作权限"), - - /** - * 内部错误 - */ - INTERNAL_SERVER_ERROR(1414, "内部错误"), - - /** - * 字典参数不存在，请先添加字典 - */ - PARAM_DICT_NOT_EXIST(1415, "字典参数不存在，请先添加字典"), - - /** - * 无可用镜像，请先在字典中配置镜像 - */ - IMAGES_DICT_NOT_EXIST(1416, "无可用镜像，请先在字典中配置镜像"), - - /** - * 容器启动失败 - */ - CONTAINER_START_FAILED(1417, "容器启动失败"), - - /** - * 数据集名称已存在 - */ - DATASET_NAME_EXIST(1418, "数据集名称已存在"); - - private Integer code; - private String msg; - - ModelOptErrorEnum(Integer code, String msg) { - this.code = code; - this.msg = msg; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java deleted file mode 100644 index 235d74b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/ModelOptInstanceStatusEnum.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Getter; - -/** - * @description 模型优化常量 - * @date 2020-05-22 - */ -@Getter -public enum ModelOptInstanceStatusEnum { - - /** - * 模型优化任务状态 - */ - WAITING("-1", "等待中"), - RUNNING("0", "进行中"), - COMPLETED("1", "已完成"), - CANCELED("2", "已取消"), - EXEC_FAILED("3", "执行失败"), - - ; - private String value; - private String msg; - - ModelOptInstanceStatusEnum(String value, String msg) { - this.value = value; - this.msg = msg; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java deleted file mode 100644 index e7ee3d7..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/NumberConstant.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -/** - * @description 数字常量 - * @date 2020-06-09 - */ -public class NumberConstant { - - public final static int NUMBER_0 = 0; - public final static int NUMBER_1 = 1; - public final static int NUMBER_2 = 2; - public final static int NUMBER_3 = 3; - public final static int NUMBER_4 = 4; - public final static int NUMBER_5 = 5; - public final static int NUMBER_6 = 6; - public final static int NUMBER_8 = 8; - public final static int NUMBER_10 = 10; - public final static int NUMBER_25 = 25; - public final static int NUMBER_30 = 30; - public final static int NUMBER_50 = 50; - public final static int NUMBER_60 = 60; - public final static int NUMBER_1024 = 1024; - public final static int NUMBER_1000 = 1000; - public final static int NUMBER_100 = 100; - public final static int HOUR_SECOND = 60 * 60; - public final static int DAY_SECOND = 60 * 60 * 24; - public final static int WEEK_SECOND = 60 * 60 * 24 * 7; - public final static int MAX_PAGE_SIZE = 2000; - public final static int MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java deleted file mode 100644 index 82a28e3..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/PermissionConstant.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Data; -import org.springframework.stereotype.Component; - -/** - * @description 权限常量 - * @date 2020-05-25 - */ -@Component -@Data -public class PermissionConstant { - - /** - * 超级用户 - */ - public static final long ADMIN_USER_ID = 1L; - - /** - * 数据集模块类型 - */ - public static final Integer RESOURCE_DATA_MODEL = 1; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java b/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java deleted file mode 100644 index 62b0e88..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/Permissions.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -/** - * @description 权限标识，对应 menu 表中的 permission 字段 - * @date 2020-05-14 - */ -public final class Permissions { - - /** - * 数据管理 - */ - public static final String DATA = "data"; - public static final String DATA_DATASET = "data:dataset"; - - /** - * 模型开发 - */ - public static final String DEVELOPMENT = "development"; - public static final String DEVELOPMENT_NOTEBOOK = "development:notebook"; - public static final String DEVELOPMENT_ALGORITHM = "development:algorithm"; - - /** - * 训练管理 - */ - public static final String TRAINING = "training"; - public static final String TRAINING_IMAGE = "training:image"; - public static final String TRAINING_JOB = "training:job"; - - /** - * 模型管理 - */ - public static final String MODEL = "model"; - public static final String MODEL_MODEL = "model:model"; - public static final String MODEL_OPTIMIZE = "model:optimize"; - - /** - * 控制台 - */ - public static final String SYSTEM = "system"; - public static final String SYSTEM_USER = "system:user"; - public static final String SYSTEM_ROLE = "system:role"; - public static final String SYSTEM_MENU = "system:menu"; - public static final String SYSTEM_DICT = "system:dict"; - public static final String SYSTEM_NODE = "system:node"; - public static final String SYSTEM_LOG = "system:log"; - public static final String SYSTEM_TEAM = "system:team"; - - /** - * 云端Serving - */ - public static final String SERVING = "serving"; - public static final String SERVING_DEPLOYMENT = "serving:deployment"; - public static final String SERVING_BATCH = "serving:batch"; - - private Permissions() { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java b/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java deleted file mode 100644 index b9715eb..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/RedisConstants.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - - -import org.dubhe.base.MagicNumConstant; - -/** - * @description redis 常量 - * @date 2020-12-17 - */ -public class RedisConstants { - /** - * 分布式锁过期时间 - */ - public static final Long DELAY_CUD_RESOURCE_EXPIRE_TIME = MagicNumConstant.TEN_LONG; - /** - * 分布式锁 - */ - public static final String DELAY_CUD_RESOURCE_KEY = "k8sclient:task:k8s_delay_cud_resource"; - /** - * 延时启动队列 - */ - public static final String DELAY_APPLY_ZSET_KEY = "k8sclient:task:delay_apply_zset"; - /** - * 延时停止队列 - */ - public static final String DELAY_STOP_ZSET_KEY = "k8sclient:task:delay_stop_zset"; - /** - * 延时队列值 - */ - public static final String DELAY_ZSET_VALUE = "%s__%s"; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java deleted file mode 100644 index b03bfb6..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/StringConstant.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import java.util.regex.Pattern; - -/** - * @description 字符串constant - * @date 2020-05-14 - */ -public final class StringConstant { - - public static final String MSIE = "MSIE"; - public static final String MOZILLA = "Mozilla"; - public static final String REQUEST_METHOD_GET = "GET"; - - /** - * 字母、数字、英文横杠和下划线匹配 - */ - public static final String REGEXP = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$"; - /** - * 整数匹配 - */ - public static final Pattern PATTERN_NUM = Pattern.compile("^[-\\+]?[\\d]*$"); - - - /** - * 公共字段 - */ - public static final String CREATE_TIME = "createTime"; - public static final String UPDATE_TIME = "updateTime"; - public static final String UPDATE_USER_ID = "updateUserId"; - public static final String CREATE_USER_ID = "createUserId"; - public static final String ORIGIN_USER_ID = "originUserId"; - public static final String DELETED = "deleted"; - public static final String UTF8 = "utf-8"; - public static final String JSON_REQUEST = "application/json"; - public static final String K8S_CALLBACK_URI = "/api/k8s/callback/pod"; - public static final String K8S_CALLBACK_PATH_DEPLOYMENT = "/api/k8s/callback/deployment"; - public static final String MULTIPART = "multipart/form-data"; - /** - * 分页内容 - */ - public static final String RESULT = "result"; - /** - * 排序规则 - */ - public static final String SORT_ASC = "asc"; - - public static final String SORT_DESC = "desc"; - - public static final String QUERY = "query"; - - public static final String NGINX_LOWERCASE = "nginx"; - - public static final String TRUE_LOWERCASE = "true"; - - public static final String GRPC_CAPITALIZE = "GRPC"; - - - /** - * 测试环境 - */ - public static final String PROFILE_ACTIVE_TEST = "test"; - - private StringConstant() { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java deleted file mode 100644 index de5650a..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/SymbolConstant.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -/** - * @description 符号常量 - * @date 2020-5-29 - */ -public class SymbolConstant { - public static final String SLASH = "/"; - public static final String COMMA = ","; - public static final String COLON = ":"; - public static final String LINEBREAK = "\n"; - public static final String BLANK = ""; - public static final String QUESTION = "?"; - public static final String ZERO = "0"; - public static final String DOT = "."; - public static final String TOKEN = "token"; - public static final String GET = "get"; - public static final String SET = "set"; - public static final String HTTP = "http"; - public static final String GRPC = "grpc"; - public static final String BRACKETS = "{}"; - public static final String BACKSLASH = "\\"; - public static final String BACKSLASH_MARK= "\\\""; - public static final String DOUBLE_MARK= "\"\""; - public static final String MARK= "\""; - public static final String FLAG_EQUAL = "="; - public static final String LEFT_PARENTHESIS = "["; - public static final String RIGHT_PARENTHESIS = "]"; - public static final String APOSTROPHE = "'"; - public static final String HYPHEN = "-"; - public static final String EVENT_SEPARATOR = "&&"; - public static final String POST = "POST"; - - private SymbolConstant() { - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java deleted file mode 100644 index 1a85787..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/TrainAlgorithmConstant.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @description 算法常量 - * @date 2020-06-02 - */ -@Data -@Component -public class TrainAlgorithmConstant { - - @Value("${train-algorithm.is-train-out}") - private Boolean isTrainOut; - - @Value("${train-algorithm.is-train-log}") - private Boolean isTrainLog; - - @Value("${train-algorithm.is-visualized-log}") - private Boolean isVisualizedLog; - - @Value("${train-algorithm.algorithm-source}") - private Integer algorithmSource; - - @Value("${train-algorithm.fork}") - private Boolean fork; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java b/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java deleted file mode 100644 index 5364203..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/constant/UserAuxiliaryInfoConstant.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.constant; - -import lombok.Data; - -/** - * @description 算法用途 - * @date 2020-06-23 - */ -@Data -public class UserAuxiliaryInfoConstant { - - - - public static final String ALGORITHM_USAGE ="algorithem_usage"; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java deleted file mode 100644 index a0d6dc5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/CommonPermissionDataDTO.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.Set; - -/** - * @description 公共权限信息DTO - * @date 2020-09-24 - */ -@AllArgsConstructor -@NoArgsConstructor -@Builder -@Data -public class CommonPermissionDataDTO implements Serializable { - - /** - * 资源拥有者ID - */ - private Long id; - - /** - * 公共类型 - */ - private Boolean type; - /** - * 资源所属用户ids - */ - private Set resourceUserIds; - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java b/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java deleted file mode 100644 index cf506f5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/dto/UserDTO.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.domain.dto; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.domain.entity.Role; -import org.dubhe.domain.entity.UserAvatar; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.Date; -import java.util.List; - -/** - * @description 用户信息 - * @date 2020-06-29 - */ -@Data -public class UserDTO implements Serializable { - - @ApiModelProperty(hidden = true) - private Long id; - - private String username; - - private String nickName; - - private String sex; - - private String email; - - private String phone; - - private Boolean enabled; - - private String remark; - - @JsonIgnore - private String password; - - private Date lastPasswordResetTime; - - private Timestamp createTime; - - private List roles; - - private UserAvatar userAvatar; - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java deleted file mode 100644 index 1f42e5c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/LogInfo.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import cn.hutool.core.date.DateUtil; -import com.alibaba.fastjson.annotation.JSONField; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.base.MagicNumConstant; - -import java.io.Serializable; - -/** - * @description 日志对象封装类 - * @date 2020-06-29 - */ -@Data -@Accessors(chain = true) -public class LogInfo implements Serializable { - - private static final long serialVersionUID = 5250395474667395607L; - - @JSONField(ordinal = MagicNumConstant.ONE) - private String traceId; - - @JSONField(ordinal = MagicNumConstant.TWO) - private String type; - - @JSONField(ordinal = MagicNumConstant.THREE) - private String level; - - @JSONField(ordinal = MagicNumConstant.FOUR) - private String location; - - @JSONField(ordinal = MagicNumConstant.FIVE) - private String time = DateUtil.now(); - - @JSONField(ordinal = MagicNumConstant.SIX) - private Object info; - - public void setInfo(Object info) { - this.info = info; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java deleted file mode 100644 index 4136d11..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Menu.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.base.BaseEntity; - -import javax.validation.constraints.NotBlank; -import java.io.Serializable; -import java.util.Objects; - -/** - * @description 菜单实体 - * @date 2020-06-29 - */ -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@TableName("menu") -public class Menu extends BaseEntity implements Serializable { - - private static final long serialVersionUID = 3100515433018008777L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - @NotBlank - private String name; - - @TableField(value = "sort") - private Long sort = 999L; - - @TableField(value = "path") - private String path; - - @TableField(value = "component") - private String component; - - /** - * 类型，目录、菜单、按钮 - */ - @TableField(value = "type") - private Integer type; - - /** - * 权限 - */ - @TableField(value = "permission") - private String permission; - - @TableField(value = "component_name") - private String componentName; - - @TableField(value = "icon") - private String icon; - - /** - * 布局类型 - */ - @TableField(value = "layout") - private String layout; - - @TableField(value = "cache") - private Boolean cache; - - @TableField(value = "hidden") - private Boolean hidden; - - /** - * 上级菜单ID - */ - @TableField(value = "pid") - private Long pid; - - @TableField(value = "deleted",fill = FieldFill.INSERT) - private Boolean deleted; - - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Menu menu = (Menu) o; - return Objects.equals(id, menu.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - public @interface Update { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java deleted file mode 100644 index f50cd86..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Role.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.base.BaseEntity; -import java.io.Serializable; -import java.util.Objects; -import java.util.Set; - -/** - * @description 角色实体 - * @date 2020-06-29 - */ -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@TableName("role") -public class Role extends BaseEntity implements Serializable { - - private static final long serialVersionUID = -812009584744832371L; - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - @TableField(value = "name") - private String name; - - /** - * 权限 - */ - @TableField(value = "permission") - private String permission; - - @TableField(value = "remark") - private String remark; - - @TableField(exist = false) - private Set menus; - - @TableField(value = "deleted",fill = FieldFill.INSERT) - private Boolean deleted = false; - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Role role = (Role) o; - return Objects.equals(id, role.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - public @interface Update { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java deleted file mode 100644 index 18715c5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/RoleMenu.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - - -/** - * @description 角色菜单关系实体 - * @date 2020-06-29 - */ -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@TableName("roles_menus") -public class RoleMenu implements Serializable { - - private static final long serialVersionUID = -6296866205797727963L; - - @TableField(value = "menu_id") - private Long menuId; - - @TableField(value = "role_id") - private Long roleId; - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java deleted file mode 100644 index 71ad1ad..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/Team.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import java.sql.Timestamp; -import java.util.List; -import java.util.Objects; - -/** - * @description 团队实体 - * @date 2020-06-29 - */ -@TableName("team") -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class Team { - @TableId(value = "id", type = IdType.AUTO) - @NotNull(groups = Update.class) - private Long id; - - @TableField(value = "name") - @NotBlank - private String name; - - @TableField(value = "enabled") - @NotNull - private Boolean enabled; - - /** - * 团队成员 - */ - @TableField(exist = false) - private List teamUserList; - - @TableField(value = "create_time") - private Timestamp createTime; - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Team dept = (Team) o; - return Objects.equals(id, dept.id) && - Objects.equals(name, dept.name); - } - - @Override - public int hashCode() { - return Objects.hash(id, name); - } - - public @interface Update { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java deleted file mode 100644 index 59146f4..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/TeamUserRole.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 团队用户关系实体 - * @date 2020-06-29 - */ -@Data -@TableName("teams_users_roles") -public class TeamUserRole implements Serializable { - - @TableId(value = "id", type = IdType.AUTO) - @NotNull() - private Long id; - - /** - * 团队 - */ - @TableField(exist = false) - private Team team; - - /** - * 用户 - */ - @TableField(exist = false) - private User user; - - /** - * 角色 - */ - @TableField(exist = false) - private Role role; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java deleted file mode 100644 index fdbe3f7..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/User.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.base.BaseEntity; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.Date; -import java.util.List; -import java.util.Objects; - - -/** - * @description 用户实体 - * @date 2020-06-29 - */ -@Data -@TableName("user") -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class User extends BaseEntity implements Serializable { - - private static final long serialVersionUID = -3836401769559845765L; - - @TableId(value = "id", type = IdType.AUTO) - @NotNull(groups = Update.class) - private Long id; - - @NotBlank - @TableField(value = "username") - private String username; - - /** - * 用户昵称 - */ - @TableField(value = "nick_name") - @NotBlank - private String nickName; - - /** - * 性别 - */ - @TableField(value = "sex") - private String sex; - - @NotBlank - @TableField(value = "email") - private String email; - - @NotBlank - @TableField(value = "phone") - private String phone; - - @NotNull - @TableField(value = "enabled") - private Boolean enabled; - - @TableField(value = "password") - private String password; - - @TableField(value = "last_password_reset_time") - private Date lastPasswordResetTime; - - @TableField(value = "remark") - private String remark; - - @TableField(value = "avatar_id") - private Long avatarId; - - - @TableField(exist = false) - private UserAvatar userAvatar; - - - @NotEmpty - @TableField(exist = false) - private List roles; - - @TableField(value = "deleted",fill = FieldFill.INSERT) - private Boolean deleted = false; - - - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - User user = (User) o; - return Objects.equals(id, user.id) && - Objects.equals(username, user.username); - } - - @Override - public int hashCode() { - return Objects.hash(id, username); - } - - public @interface Update { - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java deleted file mode 100644 index b46c67b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserAvatar.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.domain.entity; - -import cn.hutool.core.util.ObjectUtil; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.base.BaseEntity; - -import java.io.Serializable; - - -/** - * @description 用户头像实体 - * @date 2020-06-29 - */ -@Data -@NoArgsConstructor -@TableName("user_avatar") -public class UserAvatar extends BaseEntity implements Serializable { - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - @TableField(value = "real_name") - private String realName; - - @TableField(value = "path") - private String path; - - @TableField(value = "size") - private String size; - - - public UserAvatar(UserAvatar userAvatar, String realName, String path, String size) { - this.id = ObjectUtil.isNotEmpty(userAvatar) ? userAvatar.getId() : null; - this.realName = realName; - this.path = path; - this.size = size; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java b/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java deleted file mode 100644 index 756326b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/domain/entity/UserRole.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - -/** - * @description 用户角色关系实体 - * @date 2020-06-29 - */ -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor -@TableName("users_roles") -public class UserRole implements Serializable { - - private static final long serialVersionUID = -6296866205797727963L; - - @TableField(value = "user_id") - private Long userId; - - @TableField(value = "role_id") - private Long roleId; - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java deleted file mode 100644 index e350450..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/GlobalRequestRecordDTO.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto; - -import lombok.Data; - -/** - * @description 全局请求日志信息 - * @date 2020-08-13 - */ -@Data -public class GlobalRequestRecordDTO { - /** - * 客户主机地址 - */ - private String clientHost; - /** - * 请求地址 - */ - private String uri; - /** - * 授权信息 - */ - private String authorization; - /** - * 用户名 - */ - private String username; - /** - * form参数 - */ - private String params; - /** - * 返回值类型 - */ - private String contentType; - /** - * 返回状态 - */ - private Integer status; - /** - * 时间耗费 - */ - private Long timeCost; - /** - * 请求方式 - */ - private String method; - /** - * 请求体body参数 - */ - private String requestBody; - /** - * 返回值json数据 - */ - private String responseBody; - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java deleted file mode 100644 index 9db46cd..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/StateChangeDTO.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.dto; - -import lombok.*; -import org.springframework.stereotype.Component; - -/** - * @description 执行状态机切换请求体 - * @date 2020-08-27 - */ -@Component -@Data -@Builder -@ToString -@NoArgsConstructor -@AllArgsConstructor -public class StateChangeDTO { - - /** - * 业务参数 eg： id - */ - private Object[] objectParam; - - /** - * 状态机类型 eg：dataStateMachine - */ - private String stateMachineType; - - /** - * 状态机执行事件 - */ - private String eventMethodName; - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java deleted file mode 100644 index 2daf37d..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AlgorithmK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @descripton 个性化参数实现 与 个性化参数个性化校验 - * - * @date 2020-05-28 - */ -@ApiModel(description = "k8s pod异步回调Algorithm") -@Data -public class AlgorithmK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { - @Override - public String toString() { - return super.toString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java deleted file mode 100644 index 7d9601c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/AllK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @descripton k8s方异步回调统一汇总类，即不管什么业务所有请求参数都放在这个类中 - * - * @date 2020-05-28 - */ -@ApiModel(description = "k8s方 pod异步回调汇总类") -@Data -public class AllK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO{ - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java deleted file mode 100644 index 6c9b13b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sDeploymentCallbackCreateDTO.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; - -/** - * @descripton 统一通用参数实现与校验 - * @date 2020-11-26 - */ -@ApiModel(description = "k8s deployment异步回调基类") -@Data -public class BaseK8sDeploymentCallbackCreateDTO { - @ApiModelProperty(required = true, value = "k8s namespace") - @NotBlank(message = "namespace 不能为空！") - private String namespace; - - @ApiModelProperty(required = true, value = "k8s resource name") - @NotBlank(message = "resourceName 不能为空！") - private String resourceName; - - @ApiModelProperty(required = true, value = "k8s deployment name") - @NotBlank(message = "deployment 不能为空！") - private String deploymentName; - - /** - * deployment已 Running的pod数 - */ - @ApiModelProperty(required = true, value = "k8s deployment readyReplicas") - @NotNull(message = "readyReplicas 不能为空！") - private Integer readyReplicas; - - /** - * deployment总pod数 - */ - @ApiModelProperty(required = true, value = "k8s deployment replicas") - @NotNull(message = "replicas 不能为空！") - private Integer replicas; - - public BaseK8sDeploymentCallbackCreateDTO() { - - } - - public BaseK8sDeploymentCallbackCreateDTO(String namespace, String resourceName, String deploymentName, Integer readyReplicas, Integer replicas) { - this.namespace = namespace; - this.resourceName = resourceName; - this.deploymentName = deploymentName; - this.readyReplicas = readyReplicas; - this.replicas = replicas; - } - - @Override - public String toString() { - return "BaseK8sDeploymentCallbackCreateDTO{" + - "namespace='" + namespace + '\'' + - ", resourceName='" + resourceName + '\'' + - ", deploymentName='" + deploymentName + '\'' + - ", readyReplicas='" + readyReplicas + '\'' + - ", replicas='" + replicas + '\'' + - '}'; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java deleted file mode 100644 index 82897de..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BaseK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotEmpty; - -/** - * @descripton 统一通用参数实现与校验 - * - * @date 2020-05-28 - */ -@ApiModel(description = "k8s pod异步回调基类") -@Data -public class BaseK8sPodCallbackCreateDTO { - - @ApiModelProperty(required = true,value = "k8s namespace") - @NotEmpty(message = "namespace 不能为空！") - private String namespace; - - @ApiModelProperty(required = true,value = "k8s resource name") - @NotEmpty(message = "resourceName 不能为空！") - private String resourceName; - - @ApiModelProperty(required = true,value = "k8s pod name") - @NotEmpty(message = "podName 不能为空！") - private String podName; - - @ApiModelProperty(required = true,value = "k8s pod parent type") - @NotEmpty(message = "podParentType 不能为空！") - private String podParentType; - - @ApiModelProperty(required = true,value = "k8s pod parent name") - @NotEmpty(message = "podParentName 不能为空！") - private String podParentName; - - @ApiModelProperty(value = "k8s pod phase",notes = "对应PodPhaseEnum") - @NotEmpty(message = "phase 不能为空！") - private String phase; - - @ApiModelProperty(value = "k8s pod containerStatuses state") - private String messages; - - public BaseK8sPodCallbackCreateDTO(){ - - } - - public BaseK8sPodCallbackCreateDTO(String namespace,String resourceName,String podName,String podParentType,String podParentName,String phase,String messages){ - this.namespace = namespace; - this.resourceName = resourceName; - this.podName = podName; - this.podParentType = podParentType; - this.podParentName = podParentName; - this.phase = phase; - this.messages = messages; - } - - @Override - public String toString() { - return "BaseK8sPodCallbackReq{" + - "namespace='" + namespace + '\'' + - ", resourceName='" + resourceName + '\'' + - ", podName='" + podName + '\'' + - ", podParentType='" + podParentType + '\'' + - ", podParentName='" + podParentName + '\'' + - ", phase='" + phase + '\'' + - ", messages=" + messages + - '}'; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java deleted file mode 100644 index 8d1e305..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/BatchServingK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @description k8s pod异步回调云端serving - * @date 2020-11-27 - */ -@ApiModel(description = "k8s pod异步回调云端serving") -@Data -public class BatchServingK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { - - @Override - public String toString() { - return super.toString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java deleted file mode 100644 index 770f963..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ModelOptK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @description 模型优化自定义回调参数类 - * @date 2020-06-03 - */ -@ApiModel(description = "k8s pod异步回调Algorithm") -@Data -public class ModelOptK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java deleted file mode 100644 index 1378cdd..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/NotebookK8sPodCallbackCreateDTO.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @descripton k8s pod异步回调Notebook - * - * @date 2020-05-28 - */ -@ApiModel(description = "k8s pod异步回调Notebook") -@Data -public class NotebookK8sPodCallbackCreateDTO extends BaseK8sPodCallbackCreateDTO { - - @Override - public String toString() { - return super.toString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java b/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java deleted file mode 100644 index 8611f62..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/dto/callback/ServingK8sDeploymentCallbackCreateDTO.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.dto.callback; - -import io.swagger.annotations.ApiModel; -import lombok.Data; - -/** - * @description k8s deployment异步回调云端serving - * @date 2020-09-16 - */ -@ApiModel(description = "k8s deployment异步回调云端serving") -@Data -public class ServingK8sDeploymentCallbackCreateDTO extends BaseK8sDeploymentCallbackCreateDTO { - - @Override - public String toString() { - return super.toString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java deleted file mode 100644 index 4ce8332..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/BizEnum.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -import java.util.HashMap; -import java.util.Map; - -/** - * @description 业务模块 - * @date 2020-05-25 - */ -@Getter -public enum BizEnum { - - /** - * 模型开发 - */ - NOTEBOOK("模型开发", "notebook", 0), - /** - * 算法管理 - */ - ALGORITHM("算法管理", "algorithm", 1), - /** - * 模型管理 - */ - MODEL("模型管理", "model", 2), - - /** - * 训练管理 - */ - TRAIN("训练管理", "train", 3), - /** - * 云端Serving-在线服务 - */ - SERVING("云端Serving", "serving", 4), - /** - * 批量服务 - */ - BATCH_SERVING("批量服务", "batchserving", 5), - /** - * 模型优化 - */ - MODEL_OPT("模型优化管理","modelopt",6), - ; - - /** - * 业务模块名称 - */ - private String bizName; - /** - * 业务模块名称 - */ - private String bizCode; - /** - * 业务源代号 - */ - private Integer createResource; - - BizEnum(String bizName, String bizCode, Integer createResource) { - this.createResource = createResource; - this.bizName = bizName; - this.bizCode = bizCode; - } - - private static final Map RESOURCE_ENUM_MAP = new HashMap() { - { - for (BizEnum enums : BizEnum.values()) { - put(enums.getCreateResource(), enums); - } - } - }; - - /** - * 根据createResource获取BizEnum - * @param createResource 业务源代号 - * @return 枚举对象 - */ - public static BizEnum getByCreateResource(int createResource) { - return RESOURCE_ENUM_MAP.get(createResource); - } - - /** - * 根据createResource获取code - * - * @param createResource 业务源代号 - * @return String 业务模块名称 - */ - public static String getCodeByResource(Integer createResource) { - for (BizEnum enums : BizEnum.values()) { - if (enums.getCreateResource().equals(createResource)) { - return enums.getBizCode(); - } - } - return null; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java deleted file mode 100644 index 9c0af4f..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/BizNfsEnum.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import java.util.HashMap; -import java.util.Map; - -/** - * @description 业务NFS路径枚举 - * @date 2020-05-13 - */ -public enum BizNfsEnum { - /** - * 模型开发 NFS 路径命名 - */ - NOTEBOOK(BizEnum.NOTEBOOK, "notebook"), - /** - * 算法管理 NFS 路径命名 - */ - ALGORITHM(BizEnum.ALGORITHM, "algorithm-manage"), - /** - * 模型管理 NFS 路径命名 - */ - MODEL(BizEnum.MODEL, "model"), - /** - * 模型优化 NFS 路径命名 - */ - MODEL_OPT(BizEnum.MODEL_OPT, "model-opt"), - ; - - BizNfsEnum(BizEnum bizEnum, String bizNfsPath) { - this.bizEnum = bizEnum; - this.bizNfsPath = bizNfsPath; - } - - /** - * 业务模块 - */ - private BizEnum bizEnum; - /** - * 业务模块nfs路径 - */ - private String bizNfsPath; - - - private static final Map RESOURCE_ENUM_MAP = new HashMap() { - { - for (BizNfsEnum enums : BizNfsEnum.values()) { - put(enums.getCreateResource(), enums); - } - } - }; - - /** - * 根据createResource获取BizNfsEnum - * - * @param createResource - * @return - */ - public static BizNfsEnum getByCreateResource(int createResource) { - return RESOURCE_ENUM_MAP.get(createResource); - } - - - public String getBizName() { - return bizEnum == null ? null : bizEnum.getBizName(); - } - - public Integer getCreateResource() { - return bizEnum == null ? null : bizEnum.getCreateResource(); - } - - public String getBizNfsPath() { - return bizNfsPath; - } - - public BizEnum getBizEnum() { - return bizEnum; - } - - public String getBizCode() { - return bizEnum == null ? null : bizEnum.getBizCode(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java deleted file mode 100644 index 5c5953b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/DatasetTypeEnum.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -/** - * @description 数据集类型 - * @date 2020-05-21 - */ -@Getter -public enum DatasetTypeEnum { - - /** - * 私有数据 - */ - PRIVATE(0, "私有数据"), - /** - * 团队数据 - */ - TEAM(1, "团队数据"), - /** - * 公开数据 - */ - PUBLIC(2, "公开数据"); - - DatasetTypeEnum(Integer value, String msg) { - this.value = value; - this.msg = msg; - } - - private Integer value; - private String msg; - - /** - * 数据集类型校验 用户web端接口调用时参数校验 - * - * @param value 数据集类型 - * @return - */ - public static boolean isValid(Integer value) { - for (DatasetTypeEnum datasetTypeEnum : DatasetTypeEnum.values()) { - if (datasetTypeEnum.value.equals(value)) { - return true; - } - } - return false; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java deleted file mode 100644 index 7a1912b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageSourceEnum.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.enums; - -/** - * @description 镜像来源枚举类 - * @date 2020-07-15 - */ -public enum ImageSourceEnum { - MINE(0, "我的镜像"), - PRE(1, "预置镜像"); - - - /** - * 编码 - */ - private Integer code; - - /** - * 描述 - */ - private String description; - - ImageSourceEnum(int code, String description) { - this.code = code; - this.description = description; - } - - public Integer getCode() { - return code; - } - - public String getDescription() { - return description; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java deleted file mode 100644 index fdc1414..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageStateEnum.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -/** - * @description 镜像运行状态枚举 - * @date 2020-07-15 - **/ -public enum ImageStateEnum { - - MAKING(0, "制作中"), - SUCCESS(1, "制作成功"), - FAIL(2, "制作失败"); - - - /** - * 编码 - */ - private Integer code; - - /** - * 描述 - */ - private String description; - - ImageStateEnum(int code, String description) { - this.code = code; - this.description = description; - } - - public Integer getCode() { - return code; - } - - public String getDescription() { - return description; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java deleted file mode 100644 index 18b881e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/ImageTypeEnum.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.enums; - - -import lombok.Getter; - -import java.util.HashMap; -import java.util.Map; - -/** - * @description 镜像项目枚举类 - * @date 2020-12-11 - */ -@Getter -public enum ImageTypeEnum { - - /** - * notebook镜像 - */ - NOTEBOOK("notebook镜像", "notebook", 0), - - /** - * 训练镜像 - */ - TRAIN("训练镜像", "train", 1); - - /** - * 镜像项目名称 - */ - private String name; - /** - * 镜像项目代码 - */ - private String code; - /** - * 镜像项目类型 - */ - private Integer type; - - ImageTypeEnum(String name, String code, Integer type) { - this.name = name; - this.code = code; - this.type = type; - } - - private static final Map RESOURCE_ENUM_MAP = new HashMap() { - { - for (ImageTypeEnum enums : ImageTypeEnum.values()) { - put(enums.getType(), enums); - } - } - }; - - /** - * 根据type获取ImageTypeEnum - * @param type - * @return 镜像项目枚举对象 - */ - public static ImageTypeEnum getType(int type) { - return RESOURCE_ENUM_MAP.get(type); - } - - /** - * 根据type获取code - * - * @param type 镜像项目类型 - * @return String 镜像项目代码 - */ - public static String getType(Integer type) { - for (ImageTypeEnum enums : ImageTypeEnum.values()) { - if (enums.getType().equals(type)) { - return enums.getCode(); - } - } - return null; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java deleted file mode 100644 index a266177..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/LogEnum.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -/** - * @description 日志类型枚举类 - * @date 2020-06-23 - */ -@Getter -public enum LogEnum { - - // 系统报错日志 - SYS_ERR, - // 用户请求日志 - REST_REQ, - //全局请求日志 - GLOBAL_REQ, - // 训练模块 - BIZ_TRAIN, - // 系统模块 - BIZ_SYS, - // 模型模块 - BIZ_MODEL, - // 模型优化 - MODEL_OPT, - // 数据集模块 - BIZ_DATASET, - // k8s模块 - BIZ_K8S, - //note book - NOTE_BOOK, - //NFS UTILS - NFS_UTIL, - //localFileUtil - LOCAL_FILE_UTIL, - //FILE UTILS - FILE_UTIL, - // 云端Serving模块 - SERVING, - //FILE UTILS - UPLOAD_TEMP, - //STATE MACHINE - STATE_MACHINE, - //全局垃圾回收 - GARBAGE_RECYCLE, - //IO UTIL - IO_UTIL, - //DATA_SEQUENCE - DATA_SEQUENCE, - // 日志切面 - LOG_ASPECT, - //镜像 - IMAGE, - //度量 - MEASURE; - ; - - /** - * 判断日志类型不能为空 - * - * @param logType 日志类型 - * @return boolean 返回类型 - */ - public static boolean isLogType(LogEnum logType) { - - if (logType != null) { - return true; - } - return false; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java deleted file mode 100644 index a4f0c55..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/ModelResourceEnum.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -/** - * @description 模型资源枚举类 - * @date 2020-11-19 - */ -@Getter -public enum ModelResourceEnum { - - /** - * 我的模型 - */ - MINE(0, "我的模型"), - /** - * 预置模型 - */ - PRESET(1, "预置模型"), - /** - * 炼知模型 - */ - ATLAS(2, "炼知模型"); - - private Integer type; - - private String description; - - ModelResourceEnum(Integer type, String description) { - this.type = type; - this.description = description; - } - - /** - * 根据类型获取枚举类对象 - * - * @param type 类型 - * @return 枚举类对象 - */ - public static ModelResourceEnum getType(Integer type) { - for (ModelResourceEnum modelResourceEnum : values()) { - if (modelResourceEnum.getType().compareTo(type) == 0) { - //获取指定的枚举 - return modelResourceEnum; - } - } - return null; - } -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java deleted file mode 100644 index 7d911ad..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/OperationTypeEnum.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; -import lombok.ToString; - -/** - * @Description 操作类型枚举 - * @Date 2020-08-24 - */ -@ToString -@Getter -public enum OperationTypeEnum { - /** - * SELECT 查询类型 - */ - SELECT("select", "查询"), - - /** - * UPDATE 修改类型 - */ - UPDATE("update", "修改"), - - /** - * DELETE 删除类型 - */ - DELETE("delete", "删除"), - - /** - * LIMIT 禁止操作类型 - */ - LIMIT("limit", "禁止操作"), - - /** - * INSERT 新增类型 - */ - INSERT("insert", "新增类型"), - - ; - - /** - * 操作类型值 - */ - private String type; - - /** - * 操作类型备注 - */ - private String desc; - - OperationTypeEnum(String type, String desc) { - this.type = type; - this.desc = desc; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java deleted file mode 100644 index 3639d35..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/RecycleResourceEnum.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -import java.util.HashSet; -import java.util.Set; - -/** - * @description 资源回收枚举类 - * @date 2020-10-10 - */ -@Getter -public enum RecycleResourceEnum { - - /** - * 数据集文件回收 - */ - DATASET_RECYCLE_FILE("datasetRecycleFile", "数据集文件回收"), - - /** - * 医学数据集文件回收 - */ - DATAMEDICINE_RECYCLE_FILE("dataMedicineRecycleFile", "数据集文件回收"), - - /** - * 数据集版本文件回收 - */ - DATASET_RECYCLE_VERSION_FILE("datasetRecycleVersionFile", "数据集版本文件回收"), - /** - * 云端Serving输入文件回收 - */ - SERVING_RECYCLE_FILE("servingRecycleFile", "云端Serving文件回收"), - - ; - - private String className; - - private String message; - - RecycleResourceEnum(String className, String message) { - this.className = className; - this.message = message; - } - - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java deleted file mode 100644 index 35d5c4b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/SwitchEnum.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -/** - * @Description 是否开关枚举 - * @Date 2020-06-01 - */ -public enum SwitchEnum { - /** - * OFF 否 - */ - OFF(0, "否"), - - /** - * ON 否 - */ - ON(1, "是"), - - ; - - private Integer value; - - private String desc; - - SwitchEnum(Integer value, String desc) { - this.value = value; - this.desc = desc; - } - - public Integer getValue() { - return this.value; - } - - public String getDesc() { - return desc; - } - - public void setDesc(String desc) { - this.desc = desc; - } - - public static SwitchEnum getEnumValue(Integer value) { - switch (value) { - case 0: - return OFF; - case 1: - return ON; - default: - return OFF; - } - } - - public static Boolean getBooleanValue(Integer value) { - switch (value) { - case 1: - return true; - case 0: - return false; - default: - return false; - } - } - - public static boolean isExist(Integer value) { - for (SwitchEnum itm : SwitchEnum.values()) { - if (value.compareTo(itm.getValue()) == 0) { - return true; - } - } - return false; - } - - - @Override - public String toString() { - return "[" + this.value + "]" + this.desc; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java b/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java deleted file mode 100644 index 024da9c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/enums/TrainJobStatusEnum.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -import java.util.HashSet; -import java.util.Set; - -/** - * @description 训练任务枚举类 - * @date 2020-04-27 - */ -@Getter -public enum TrainJobStatusEnum { - - /** - * PENDING - */ - PENDING(0, "PENDING"), - /** - * RUNNING - */ - RUNNING(1, "RUNNING"), - /** - * SUCCEEDED - */ - SUCCEEDED(2, "SUCCEEDED"), - /** - * FAILED - */ - FAILED(3, "FAILED"), - /** - * STOP - */ - STOP(4, "STOP"), - /** - * UNKNOWN - */ - UNKNOWN(5, "UNKNOWN"), - /** - * DELETED - */ - DELETED(6, "DELETED"), - - /** - * CREATE_FAILED - */ - CREATE_FAILED(7, "CREATE_FAILED"); - - private Integer status; - - private String message; - - TrainJobStatusEnum(Integer status, String message) { - this.status = status; - this.message = message; - } - - /** - * 根据信息获取枚举类对象 - * - * @param msg 信息 - * @return 枚举类对象 - */ - public static TrainJobStatusEnum getByMessage(String msg) { - for (TrainJobStatusEnum statusEnum : values()) { - if (statusEnum.message.equalsIgnoreCase(msg)) { - return statusEnum; - } - } - return UNKNOWN; - } - - /** - * 回调状态转换 若是DELETED则转换为STOP，避免状态不统一 - * @param phase k8s pod phase - * @return - */ - public static TrainJobStatusEnum transferStatus(String phase) { - TrainJobStatusEnum enums = getByMessage(phase); - if (enums != DELETED) { - return enums; - } - return STOP; - } - - /** - * 根据状态获取枚举类对象 - * - * @param status 状态 - * @return 枚举类对象 - */ - public static TrainJobStatusEnum getByStatus(Integer status) { - for (TrainJobStatusEnum statusEnum : values()) { - if (statusEnum.status.equals(status)) { - return statusEnum; - } - } - return UNKNOWN; - } - - - /** - * 结束状态枚举集合 - */ - public static final Set END_TRAIN_JOB_STATUS; - - static { - END_TRAIN_JOB_STATUS = new HashSet<>(); - END_TRAIN_JOB_STATUS.add(SUCCEEDED); - END_TRAIN_JOB_STATUS.add(FAILED); - END_TRAIN_JOB_STATUS.add(STOP); - END_TRAIN_JOB_STATUS.add(CREATE_FAILED); - END_TRAIN_JOB_STATUS.add(DELETED); - } - - public static boolean isEnd(String msg) { - return END_TRAIN_JOB_STATUS.contains(getByMessage(msg)); - } - - public static boolean isEnd(Integer status) { - return END_TRAIN_JOB_STATUS.contains(getByStatus(status)); - } - - public static boolean checkStopStatus(Integer num) { - return isEnd(num); - } - - public static boolean checkRunStatus(Integer num) { - return PENDING.getStatus().equals(num) || - RUNNING.getStatus().equals(num); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java deleted file mode 100644 index 5bf674c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/BaseErrorCode.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; - -/** - * @description 通用异常code - * @date 2020-03-26 - */ -@Getter -public enum BaseErrorCode implements ErrorCode { - - /** - * undefined error - */ - UNDEFINED(10000, "操作成功!"), - ERROR(10001, "操作失败!"), - ERROR_SYSTEM(10002, "系统繁忙!"), - UNAUTHORIZED(401, "无权访问!"), - /** - * system 模块异常码 - */ - SYSTEM_USERNAME_ALREADY_EXISTS(20000, "账号已存在!"), - SYSTEM_CODE_ALREADY_EXISTS(20001, "Code already exists!"), - SYSTEM_USER_IS_NOT_EXISTS(20002, "用户不存在!"), - SYSTEM_USER_ALREADY_REGISTER(20003, "账号已注册!"), - SYSTEM_USER_REGISTER_EMAIL_INFO_EXPIRED(20004, "邮箱验证码已过期!"), - SYSTEM_USER_EMAIL_ALREADY_EXISTS(20004, "该邮箱已被注册!"), - SYSTEM_USER_EMAIL_PASSWORD_ERROR(20005, "邮件密码错误!"), - SYSTEM_USER_EMAIL_CODE_CANNOT_EXCEED_TIMES(20006, "邮件发送不能超过三次!"), - SYSTEM_USER_EMAIL_OR_CODE_ERROR(20007, "邮箱地址或验证码错误、请重新输入!"), - SYSTEM_USER_IS_LOCKED(20008, "用户已锁定!"), - SYSTEM_USER_USERNAME_OR_PASSWORD_ERROR(20009, "账号或密码不正确!"), - SYSTEM_USER_USERNAME_IS_LOCKED(20010, "账号已锁定，6小时后解锁!"), - SYSTEM_USER_CANNOT_UPDATE_ADMIN(20011, "仅超级管理员可操作!"), - SYSTEM_USER_TOKEN_INFO_IS_NULL(20012, "登录信息不存在!"), - SYSTEM_USER_EMAIL_NOT_EXISTS(20013, "该邮箱未注册!"), - SYSTEM_USER_CANNOT_DELETE(20014, "系统默认用户不可删除!"), - SYSTEM_ROLE_CANNOT_DELETE(20015, "系统默认角色不可删除!"), - - DATASET_ADMIN_PERMISSION_ERROR(1310,"无此权限,请联系管理员"), - - ; - - - Integer code; - String msg; - - BaseErrorCode(Integer code, String msg) { - this.code = code; - this.msg = msg; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java deleted file mode 100644 index cee727d..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/BusinessException.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - -/** - * @description 业务异常 - * @date 2020-03-13 - */ -@Getter -public class BusinessException extends RuntimeException { - - private DataResponseBody responseBody; - - public BusinessException(String msg) { - super(msg); - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public BusinessException(String msg, Throwable cause) { - super(msg,cause); - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public BusinessException(Throwable cause) { - super(cause); - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); - } - - public BusinessException(Integer code, String msg, String info, Throwable cause) { - super(msg,cause); - if (info == null) { - this.responseBody = new DataResponseBody(code, msg); - } else { - this.responseBody = new DataResponseBody(code, msg + ":" + info); - } - } - - public BusinessException(ErrorCode errorCode, Throwable cause) { - this(errorCode.getCode(), errorCode.getMsg(), null, cause); - } - - public BusinessException(ErrorCode errorCode, String info, Throwable cause) { - this(errorCode.getCode(), errorCode.getMsg(), info, cause); - } - - public BusinessException(ErrorCode errorCode) { - this(errorCode, null); - } - - public BusinessException(Integer code,String msg) { - this.responseBody = new DataResponseBody(code, msg); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java deleted file mode 100644 index f2ec3e6..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/CaptchaException.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - -/** - * @description 验证码异常 - * @date 2020-02-23 - */ -@Getter -public class CaptchaException extends RuntimeException { - private static final long serialVersionUID = 1L; - - private DataResponseBody responseBody; - private Throwable cause; - - public CaptchaException(String msg) { - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public CaptchaException(String msg, Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public CaptchaException(Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java deleted file mode 100644 index b4790dd..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/DataSequenceException.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; - -/** - * @description 获取序列异常 - * @date 2020-09-23 - */ -@Getter -public class DataSequenceException extends BusinessException { - - private static final long serialVersionUID = 1L; - - public DataSequenceException(String msg) { - super(msg); - } - - public DataSequenceException(String msg, Throwable cause) { - super(msg,cause); - } - - public DataSequenceException(Throwable cause) { - super(cause); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java deleted file mode 100644 index 94de0f3..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/DockerBizException.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - -/** - * @Description docker 异常 - * @Date 2020-6-6 - **/ -@Getter -public class DockerBizException extends RuntimeException { - private static final long serialVersionUID = 1L; - - private DataResponseBody responseBody; - private Throwable cause; - - public DockerBizException(String msg) { - this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg); - } - - public DockerBizException(String msg, Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR, msg); - } - - public DockerBizException(Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.DOCKER_ERROR); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java b/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java deleted file mode 100644 index b8d367c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/ErrorCode.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - - -/** - * @description 异常code - * @date 2020-03-26 - */ -public interface ErrorCode { - - /** - * 错误码 - * @return code - */ - Integer getCode(); - - /** - * error info - * @return - */ - String getMsg(); - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java deleted file mode 100644 index 23668d9..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/LoginException.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.exception; - -import lombok.Getter; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - -/** - * @description 登录异常 - * @date 2020-02-23 - */ -@Getter -public class LoginException extends RuntimeException { - private DataResponseBody responseBody; - private Throwable cause; - - public LoginException(String msg) { - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public LoginException(String msg, Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST, msg); - } - - public LoginException(Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.BADREQUEST); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java deleted file mode 100644 index 2fa03d0..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/NfsBizException.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; - -/** - * Description : NFS utils 工具异常 - * - * @date 15.06.2020 - */ -@Getter -public class NfsBizException extends BusinessException { - - private static final long serialVersionUID = 1L; - - - public NfsBizException(Throwable cause){ - super(cause); - } - - public NfsBizException(String msg){ - super(msg); - } - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java deleted file mode 100644 index 3f23953..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/NotebookBizException.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; - -/** - * @description Notebook 业务处理异常 - * @date 2020.04.27 - */ -@Getter -public class NotebookBizException extends BusinessException{ - - private static final long serialVersionUID = 1L; - - public NotebookBizException(String msg) { - super(msg); - } - - public NotebookBizException(String msg, Throwable cause) { - super(msg,cause); - } - - public NotebookBizException(Throwable cause) { - super(cause); - } - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java deleted file mode 100644 index 774c9e2..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/StateMachineException.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.exception; - -import lombok.Getter; - -/** - * @description 状态机异常类 - * @date 2020-08-27 - */ -@Getter -public class StateMachineException extends BusinessException { - - private static final long serialVersionUID = 1L; - - /** - * 自定义状态机异常(抛出异常堆栈信息) - * - * @param cause - */ - public StateMachineException(Throwable cause){ - super(cause); - } - - /** - * 自定义状态机异常(抛出异常信息) - * - * @param msg - */ - public StateMachineException(String msg){ - super(msg); - } - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java b/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java deleted file mode 100644 index 54b4929..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/UnauthorizedException.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception; - -import lombok.Getter; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - -/** - * @description 权限异常 - * @date 2020-02-23 - */ -@Getter -public class UnauthorizedException extends RuntimeException { - - private DataResponseBody responseBody; - private Throwable cause; - - public UnauthorizedException(String msg) { - this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg); - } - - public UnauthorizedException(String msg, Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED, msg); - } - - public UnauthorizedException(Throwable cause) { - this.cause = cause; - this.responseBody = new DataResponseBody(ResponseCode.UNAUTHORIZED); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java b/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java deleted file mode 100644 index 41367b7..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/exception/handler/GlobalExceptionHandler.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.exception.handler; - -import lombok.extern.slf4j.Slf4j; -import org.apache.ibatis.exceptions.IbatisException; -import org.apache.shiro.ShiroException; -import org.apache.shiro.authc.AuthenticationException; -import org.apache.shiro.authc.IncorrectCredentialsException; -import org.apache.shiro.authc.LockedAccountException; -import org.apache.shiro.authc.UnknownAccountException; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.*; -import org.dubhe.utils.LogUtil; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.validation.BindException; -import org.springframework.validation.ObjectError; -import org.springframework.web.bind.MethodArgumentNotValidException; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -import java.util.Objects; - -/** - * @description 处理异常 - * @date 2020-02-23 - */ -@Slf4j -@RestControllerAdvice -public class GlobalExceptionHandler { - - /** - * 处理所有不可知的异常 - */ - @ExceptionHandler(Throwable.class) - public ResponseEntity handleException(Throwable e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR, - new DataResponseBody(ResponseCode.ERROR, e.getMessage())); - } - - /** - * UnauthorizedException - */ - @ExceptionHandler(UnauthorizedException.class) - public ResponseEntity badCredentialsException(UnauthorizedException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - String message = "坏的凭证".equals(e.getMessage()) ? "用户名或密码不正确" : e.getMessage(); - return buildResponseEntity(HttpStatus.UNAUTHORIZED, new DataResponseBody(ResponseCode.ERROR, message)); - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = BusinessException.class) - public ResponseEntity badRequestException(BusinessException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = IbatisException.class) - public ResponseEntity persistenceException(IbatisException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, e.getMessage())); - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = AuthenticationException.class) - public ResponseEntity badRequestException(AuthenticationException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问")); - } - - /** - * shiro 异常捕捉 - */ - @ExceptionHandler(value = ShiroException.class) - public ResponseEntity accountException(ShiroException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - ResponseEntity responseEntity; - if (e instanceof IncorrectCredentialsException) { - responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "密码不正确")); - } else if (e instanceof UnknownAccountException) { - responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "此账户不存在")); - } else if (e instanceof LockedAccountException) { - responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "未知的账号")); - } else if (e instanceof UnknownAccountException) { - responseEntity = buildResponseEntity(HttpStatus.OK, new DataResponseBody(ResponseCode.ERROR, "账户已被禁用")); - } else { - responseEntity = buildResponseEntity(HttpStatus.OK, - new DataResponseBody(ResponseCode.UNAUTHORIZED, "无权访问")); - } - return responseEntity; - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = LoginException.class) - public ResponseEntity loginException(LoginException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.UNAUTHORIZED, e.getResponseBody()); - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = CaptchaException.class) - public ResponseEntity captchaException(CaptchaException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); - } - - /** - * 处理自定义异常 - */ - @ExceptionHandler(value = NotebookBizException.class) - public ResponseEntity captchaException(NotebookBizException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - return buildResponseEntity(HttpStatus.OK, e.getResponseBody()); - } - - /** - * 处理所有接口数据验证异常 - */ - @ExceptionHandler(MethodArgumentNotValidException.class) - public ResponseEntity handleMethodArgumentNotValidException(MethodArgumentNotValidException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - String[] str = Objects.requireNonNull(e.getBindingResult().getAllErrors().get(0).getCodes())[1].split("\\."); - String message = e.getBindingResult().getAllErrors().get(0).getDefaultMessage(); - String msg = "不能为空"; - if (msg.equals(message)) { - message = str[1] + ":" + message; - } - return buildResponseEntity(HttpStatus.BAD_REQUEST, new DataResponseBody(ResponseCode.ERROR, message)); - } - - @ExceptionHandler(BindException.class) - public ResponseEntity bindException(BindException e) { - // 打印堆栈信息 - LogUtil.error(LogEnum.SYS_ERR, "引起异常的堆栈信息：{}", e); - ObjectError error = e.getAllErrors().get(0); - return buildResponseEntity(HttpStatus.BAD_REQUEST, - new DataResponseBody(ResponseCode.ERROR, error.getDefaultMessage())); - } - - /** - * 统一返回 - * - * @param httpStatus - * @param responseBody - * @return - */ - private ResponseEntity buildResponseEntity(HttpStatus httpStatus, DataResponseBody responseBody) { - return new ResponseEntity<>(responseBody, httpStatus); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java b/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java deleted file mode 100644 index c0b717f..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/factory/DataResponseFactory.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright 2019-2020 Zheng Jie - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dubhe.factory; - -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; - - -/** - * @description DataResponseBody 工厂类 - * @date 2020-03-16 - */ -public class DataResponseFactory { - - private DataResponseFactory(){ - - } - - /** - * 成功响应 - * - * @param - * @return - */ - public static DataResponseBody success(){ - return success(null,null); - } - - /** - * 成功响应 - * - * @param data - * @param - * @return - */ - public static DataResponseBody success(T data){ - return success(null,data); - } - - /** - * 成功响应 - * - * @param msg - * @return - */ - public static DataResponseBody successWithMsg(String msg){ - return success(msg,null); - } - - /** - * 成功响应 - * - * @param msg - * @param data - * @param - * @return - */ - public static DataResponseBody success(String msg, T data){ - return new DataResponseBody(ResponseCode.SUCCESS,msg,data); - } - - /** - * 失败响应 msg - * - * @param msg - * @return - */ - public static DataResponseBody failed(String msg){ - return failed(ResponseCode.ERROR,msg,null); - } - - /** - * 失败响应 - * - * @param failedCode - * @param msg - * @return - */ - public static DataResponseBody failed(Integer failedCode,String msg){ - return failed(failedCode,msg,null); - } - - /** - * 失败响应 - * - * @param failedCode - * @return - */ - public static DataResponseBody failed(Integer failedCode){ - return failed(failedCode,null,null); - } - - /** - * 失败响应 - * - * @param failedCode - * @param msg - * @param data - * @param - * @return - */ - public static DataResponseBody failed(Integer failedCode, String msg, T data){ - return new DataResponseBody(failedCode,msg,data); - } - - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java deleted file mode 100644 index 08dc43b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/BaseLogFilter.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.filter; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.filter.AbstractMatcherFilter; -import ch.qos.logback.core.spi.FilterReply; -import cn.hutool.core.util.StrUtil; -import org.slf4j.Marker; - -/** - * @description 自定义日志过滤器 - * @date 2020-07-21 - */ -public class BaseLogFilter extends AbstractMatcherFilter { - - Level level; - - /** - * 重写decide方法 - * - * @param iLoggingEvent event to decide upon. - * @return FilterReply - */ - @Override - public FilterReply decide(ILoggingEvent iLoggingEvent) { - if (!isStarted()) { - return FilterReply.NEUTRAL; - } - final String msg = iLoggingEvent.getMessage(); - //自定义级别 - if (checkLevel(iLoggingEvent) && msg != null && msg.startsWith(StrUtil.DELIM_START) && msg.endsWith(StrUtil.DELIM_END)) { - final Marker marker = iLoggingEvent.getMarker(); - if (marker != null && this.getName() != null && this.getName().contains(marker.getName())) { - return onMatch; - } - } - - return onMismatch; - } - - /** - * 检测日志级别 - * @param iLoggingEvent 日志事件 - * @return true 过滤当前级别 false 不过滤当前级别 - */ - protected boolean checkLevel(ILoggingEvent iLoggingEvent) { - return this.level != null - && iLoggingEvent.getLevel() != null - && iLoggingEvent.getLevel().toInt() == this.level.toInt(); - } - - public void setLevel(Level level) { - this.level = level; - } - - @Override - public void start() { - if (this.level != null) { - super.start(); - } - } -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java deleted file mode 100644 index c442842..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/ConsoleLogFilter.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.filter; - -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.spi.FilterReply; -import org.dubhe.utils.LogUtil; -import org.slf4j.MarkerFactory; - -/** - * @description 自定义日志过滤器 - * @date 2020-07-21 - */ -public class ConsoleLogFilter extends BaseLogFilter { - - @Override - public FilterReply decide(ILoggingEvent iLoggingEvent) { - if (!isStarted()) { - return FilterReply.NEUTRAL; - } - return checkLevel(iLoggingEvent) ? onMatch : onMismatch; - } - - protected boolean checkLevel(ILoggingEvent iLoggingEvent) { - - - return this.level != null - && iLoggingEvent.getLevel() != null - && iLoggingEvent.getLevel().toInt() >= this.level.toInt() - && !MarkerFactory.getMarker(LogUtil.K8S_CALLBACK_LEVEL).equals(iLoggingEvent.getMarker()) - && !MarkerFactory.getMarker(LogUtil.SCHEDULE_LEVEL).equals(iLoggingEvent.getMarker()) - && !"log4jdbc.log4j2".equals(iLoggingEvent.getLoggerName()); - } -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java b/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java deleted file mode 100644 index 4e3f09b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/filter/GlobalRequestLogFilter.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.filter; - -import ch.qos.logback.classic.spi.ILoggingEvent; - -/** - * @description 全局请求 日志过滤器 - * @date 2020-08-13 - */ -public class GlobalRequestLogFilter extends BaseLogFilter { - - - @Override - public boolean checkLevel(ILoggingEvent iLoggingEvent) { - return this.level != null; - } - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java b/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java deleted file mode 100644 index 6d928b1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/handle/ApiVersionCondition.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.handle; - -import org.springframework.web.servlet.mvc.condition.RequestCondition; - -import javax.servlet.http.HttpServletRequest; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @description API版本控制 - * @date 2020-04-06 - */ -public class ApiVersionCondition implements RequestCondition { - /** - * 路径中版本的前缀， 这里用 /v[1-9]/的形式 - */ - private final static Pattern VERSION_PREFIX_PATTERN = Pattern.compile("v(\\d+)/"); - private int apiVersion; - - public ApiVersionCondition(int apiVersion) { - this.apiVersion = apiVersion; - } - - @Override - public ApiVersionCondition combine(ApiVersionCondition other) { - // 采用最后定义优先原则，则方法上的定义覆盖类上面的定义 - return new ApiVersionCondition(other.getApiVersion()); - } - - @Override - public ApiVersionCondition getMatchingCondition(HttpServletRequest request) { - Matcher m = VERSION_PREFIX_PATTERN.matcher(request.getRequestURI()); - if (m.find()) { - Integer version = Integer.valueOf(m.group(1)); - if (version >= this.apiVersion) { - return this; - } - } - return null; - } - - - @Override - public int compareTo(ApiVersionCondition other, HttpServletRequest request) { - // 优先匹配最新的版本号 - return other.getApiVersion() - this.apiVersion; - } - public int getApiVersion() { - return apiVersion; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java b/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java deleted file mode 100644 index 64be203..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/handle/CustomRequestMappingHandlerMapping.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.handle; - -import lombok.Data; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.springframework.core.annotation.AnnotationUtils; -import org.springframework.stereotype.Component; -import org.springframework.web.servlet.mvc.condition.RequestCondition; -import org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping; - -import java.lang.reflect.Method; -import java.util.HashSet; -import java.util.Set; -/** - * @description 权限注解收集 - * @date 2020-04-06 - */ -@Component -@Data -public class CustomRequestMappingHandlerMapping extends RequestMappingHandlerMapping { - Set permissionsSet =new HashSet(); - @Override - protected RequestCondition getCustomTypeCondition(Class handlerType) { - ApiVersion apiVersion = AnnotationUtils.findAnnotation(handlerType, ApiVersion.class); - return createCondition(apiVersion); - } - - @Override - protected RequestCondition getCustomMethodCondition(Method method) { - RequiresPermissions requiresPermissions = AnnotationUtils.findAnnotation(method, RequiresPermissions.class); - if(requiresPermissions != null){ - if(requiresPermissions.value()!=null){ - for(int i=0;i createCondition(ApiVersion apiVersion) { - return apiVersion == null ? null : new ApiVersionCondition(apiVersion.value()); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java deleted file mode 100644 index 7b8d398..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/K8sCallBackPodInterceptor.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.interceptor; - -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.K8sCallBackTool; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** - * @description k8s pod 异步回调拦截器 - * - * @date 2020-05-28 - */ -@Component -public class K8sCallBackPodInterceptor extends HandlerInterceptorAdapter { - - @Autowired - private K8sCallBackTool k8sCallBackTool; - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { - String uri = request.getRequestURI(); - LogUtil.debug(LogEnum.BIZ_K8S,"接收到k8s异步请求,URI:{}",uri); - String k8sCallbackToken = request.getHeader(K8sCallBackTool.K8S_CALLBACK_TOKEN); - if (StringUtils.isBlank(k8sCallbackToken)){ - LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调没有配置【{}】,URI:{}",K8sCallBackTool.K8S_CALLBACK_TOKEN,uri); - return false; - } - boolean pass = k8sCallBackTool.validateToken(k8sCallbackToken); - if (!pass){ - LogUtil.warn(LogEnum.BIZ_K8S,"k8s异步回调token:【{}】 验证不通过,URI:{}",k8sCallbackToken,uri); - } - return pass; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java deleted file mode 100644 index 3654574..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/MySqlInterceptor.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.interceptor; - -import org.apache.ibatis.executor.statement.StatementHandler; -import org.apache.ibatis.mapping.BoundSql; -import org.apache.ibatis.mapping.MappedStatement; -import org.apache.ibatis.plugin.*; -import org.apache.ibatis.reflection.DefaultReflectorFactory; -import org.apache.ibatis.reflection.MetaObject; -import org.apache.ibatis.reflection.SystemMetaObject; -import org.dubhe.annotation.DataPermission; -import org.dubhe.base.DataContext; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.enums.OperationTypeEnum; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.SqlUtil; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Field; -import java.sql.Connection; -import java.util.Arrays; -import java.util.Objects; -import java.util.Properties; - - -/** - * @description mybatis拦截器 - * @date 2020-06-10 - */ -@Component -@Intercepts({@Signature(type = StatementHandler.class, method = "prepare", args = {Connection.class, Integer.class})}) -public class MySqlInterceptor implements Interceptor { - @Override - public Object intercept(Invocation invocation) throws Throwable { - - StatementHandler statementHandler = (StatementHandler) invocation.getTarget(); - MetaObject metaObject = MetaObject.forObject(statementHandler, SystemMetaObject.DEFAULT_OBJECT_FACTORY, - SystemMetaObject.DEFAULT_OBJECT_WRAPPER_FACTORY, new DefaultReflectorFactory()); - /* - * 先拦截到RoutingStatementHandler，里面有个StatementHandler类型的delegate变量，其实现类是BaseStatementHandler， - * 然后就到BaseStatementHandler的成员变量mappedStatement - */ - MappedStatement mappedStatement = (MappedStatement) metaObject.getValue("delegate.mappedStatement"); - //id为执行的mapper方法的全路径名，如com.uv.dao.UserDao.selectPageVo - String id = mappedStatement.getId(); - //sql语句类型 select、delete、insert、update - String sqlCommandType = mappedStatement.getSqlCommandType().toString(); - BoundSql boundSql = statementHandler.getBoundSql(); - - //获取到原始sql语句 - String sql = boundSql.getSql(); - String mSql = sql; - - //注解逻辑判断 添加注解了才拦截 - Class classType = Class.forName(mappedStatement.getId().substring(0, mappedStatement.getId().lastIndexOf("."))); - String mName = mappedStatement.getId().substring(mappedStatement.getId().lastIndexOf(".") + 1, mappedStatement.getId().length()); - UserDTO currentUserDto = JwtUtils.getCurrentUserDto(); - - //获取类注解 获取需要忽略拦截的方法名称 - DataPermission dataAnnotation = classType.getAnnotation(DataPermission.class); - if (!Objects.isNull(dataAnnotation)) { - - String[] ignores = dataAnnotation.ignoresMethod(); - //校验拦截忽略方法名 忽略新增方法 忽略回调/定时方法 - if ((!Objects.isNull(ignores) && Arrays.asList(ignores).contains(mName)) - || OperationTypeEnum.INSERT.getType().equals(sqlCommandType.toLowerCase()) - || Objects.isNull(currentUserDto) - || (!Objects.isNull(DataContext.get()) && DataContext.get().getType()) - ) { - return invocation.proceed(); - } else { - //拦截所有sql操作类型 - mSql = SqlUtil.buildTargetSql(sql, SqlUtil.getResourceIds()); - } - } - - //通过反射修改sql语句 - Field field = boundSql.getClass().getDeclaredField("sql"); - field.setAccessible(true); - field.set(boundSql, mSql); - return invocation.proceed(); - } - - @Override - public Object plugin(Object target) { - if (target instanceof StatementHandler) { - return Plugin.wrap(target, this); - } else { - return target; - } - } - - @Override - public void setProperties(Properties properties) { - - } - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java b/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java deleted file mode 100644 index 5176942..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/interceptor/PaginationInterceptor.java +++ /dev/null @@ -1,457 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.interceptor; - -import com.baomidou.mybatisplus.annotation.DbType; -import com.baomidou.mybatisplus.core.MybatisDefaultParameterHandler; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.core.metadata.OrderItem; -import com.baomidou.mybatisplus.core.parser.ISqlParser; -import com.baomidou.mybatisplus.core.parser.SqlInfo; -import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; -import com.baomidou.mybatisplus.core.toolkit.ExceptionUtils; -import com.baomidou.mybatisplus.core.toolkit.PluginUtils; -import com.baomidou.mybatisplus.core.toolkit.StringUtils; -import com.baomidou.mybatisplus.extension.handlers.AbstractSqlParserHandler; -import com.baomidou.mybatisplus.extension.plugins.pagination.DialectFactory; -import com.baomidou.mybatisplus.extension.plugins.pagination.DialectModel; -import com.baomidou.mybatisplus.extension.plugins.pagination.dialects.IDialect; -import com.baomidou.mybatisplus.extension.toolkit.JdbcUtils; -import com.baomidou.mybatisplus.extension.toolkit.SqlParserUtils; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.schema.Column; -import net.sf.jsqlparser.statement.select.*; -import org.apache.ibatis.executor.statement.StatementHandler; -import org.apache.ibatis.logging.Log; -import org.apache.ibatis.logging.LogFactory; -import org.apache.ibatis.mapping.*; -import org.apache.ibatis.plugin.*; -import org.apache.ibatis.reflection.MetaObject; -import org.apache.ibatis.reflection.SystemMetaObject; -import org.apache.ibatis.scripting.defaults.DefaultParameterHandler; -import org.apache.ibatis.session.Configuration; -import org.dubhe.annotation.DataPermission; -import org.dubhe.base.DataContext; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.enums.OperationTypeEnum; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.SqlUtil; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @description MybatisPlus 分页拦截器 - * @date 2020-10-09 - */ -@Intercepts({@Signature( - type = StatementHandler.class, - method = "prepare", - args = {Connection.class, Integer.class} -)}) -public class PaginationInterceptor extends AbstractSqlParserHandler implements Interceptor { - protected static final Log logger = LogFactory.getLog(PaginationInterceptor.class); - - /** - * COUNT SQL 解析 - */ - protected ISqlParser countSqlParser; - /** - * 溢出总页数，设置第一页 - */ - protected boolean overflow = false; - /** - * 单页限制 500 条，小于 0 如 -1 不受限制 - */ - protected long limit = 500L; - /** - * 数据类型 - */ - private DbType dbType; - /** - * 方言 - */ - private IDialect dialect; - /** - * 方言类型 - */ - @Deprecated - protected String dialectType; - /** - * 方言实现类 - */ - @Deprecated - protected String dialectClazz; - - public PaginationInterceptor() { - } - - /** - * 构建分页sql - * - * @param originalSql 原生sql - * @param page 分页参数 - * @return 构建后 sql - */ - public static String concatOrderBy(String originalSql, IPage page) { - if (CollectionUtils.isNotEmpty(page.orders())) { - try { - List orderList = page.orders(); - Select selectStatement = (Select) CCJSqlParserUtil.parse(originalSql); - List orderByElements; - List orderByElementsReturn; - if (selectStatement.getSelectBody() instanceof PlainSelect) { - PlainSelect plainSelect = (PlainSelect) selectStatement.getSelectBody(); - orderByElements = plainSelect.getOrderByElements(); - orderByElementsReturn = addOrderByElements(orderList, orderByElements); - plainSelect.setOrderByElements(orderByElementsReturn); - return plainSelect.toString(); - } - - if (selectStatement.getSelectBody() instanceof SetOperationList) { - SetOperationList setOperationList = (SetOperationList) selectStatement.getSelectBody(); - orderByElements = setOperationList.getOrderByElements(); - orderByElementsReturn = addOrderByElements(orderList, orderByElements); - setOperationList.setOrderByElements(orderByElementsReturn); - return setOperationList.toString(); - } - - if (selectStatement.getSelectBody() instanceof WithItem) { - return originalSql; - } - - return originalSql; - } catch (JSQLParserException var7) { - logger.error("failed to concat orderBy from IPage, exception=", var7); - } - } - - return originalSql; - } - - /** - * 添加分页排序规则 - * - * @param orderList 分页规则 - * @param orderByElements 分页排序元素 - * @return 分页规则 - */ - private static List addOrderByElements(List orderList, List orderByElements) { - orderByElements = CollectionUtils.isEmpty(orderByElements) ? new ArrayList(orderList.size()) : orderByElements; - List orderByElementList = (List) orderList.stream().filter((item) -> { - return StringUtils.isNotBlank(item.getColumn()); - }).map((item) -> { - OrderByElement element = new OrderByElement(); - element.setExpression(new Column(item.getColumn())); - element.setAsc(item.isAsc()); - element.setAscDescPresent(true); - return element; - }).collect(Collectors.toList()); - ((List) orderByElements).addAll(orderByElementList); - return (List) orderByElements; - } - - /** - * 执行sql查询逻辑 - * - * @param invocation mybatis 调用类 - * @return - * @throws Throwable - */ - @Override - public Object intercept(Invocation invocation) throws Throwable { - StatementHandler statementHandler = (StatementHandler) PluginUtils.realTarget(invocation.getTarget()); - MetaObject metaObject = SystemMetaObject.forObject(statementHandler); - this.sqlParser(metaObject); - MappedStatement mappedStatement = (MappedStatement) metaObject.getValue("delegate.mappedStatement"); - if (SqlCommandType.SELECT == mappedStatement.getSqlCommandType() && StatementType.CALLABLE != mappedStatement.getStatementType()) { - BoundSql boundSql = (BoundSql) metaObject.getValue("delegate.boundSql"); - Object paramObj = boundSql.getParameterObject(); - IPage page = null; - if (paramObj instanceof IPage) { - page = (IPage) paramObj; - } else if (paramObj instanceof Map) { - Iterator var8 = ((Map) paramObj).values().iterator(); - - while (var8.hasNext()) { - Object arg = var8.next(); - if (arg instanceof IPage) { - page = (IPage) arg; - break; - } - } - } - - if (null != page && page.getSize() >= 0L) { - if (this.limit > 0L && this.limit <= page.getSize()) { - this.handlerLimit(page); - } - - String originalSql = boundSql.getSql(); - - //注解逻辑判断 添加注解了才拦截 - Class classType = Class.forName(mappedStatement.getId().substring(0, mappedStatement.getId().lastIndexOf("."))); - String mName = mappedStatement.getId().substring(mappedStatement.getId().lastIndexOf(".") + 1, mappedStatement.getId().length()); - UserDTO currentUserDto = JwtUtils.getCurrentUserDto(); - - String sqlCommandType = mappedStatement.getSqlCommandType().toString(); - //获取类注解 获取需要忽略拦截的方法名称 - DataPermission dataAnnotation = classType.getAnnotation(DataPermission.class); - if (!Objects.isNull(dataAnnotation)) { - - String[] ignores = dataAnnotation.ignoresMethod(); - //校验拦截忽略方法名 忽略新增方法 忽略回调/定时方法 - if (!((!Objects.isNull(ignores) && Arrays.asList(ignores).contains(mName)) - || OperationTypeEnum.INSERT.getType().equals(sqlCommandType.toLowerCase()) - || Objects.isNull(currentUserDto) - || (!Objects.isNull(DataContext.get()) && DataContext.get().getType())) - - - ) { - originalSql = SqlUtil.buildTargetSql(originalSql, SqlUtil.getResourceIds()); - } - } - - Connection connection = (Connection) invocation.getArgs()[0]; - if (page.isSearchCount() && !page.isHitCount()) { - SqlInfo sqlInfo = SqlParserUtils.getOptimizeCountSql(page.optimizeCountSql(), this.countSqlParser, originalSql); - this.queryTotal(sqlInfo.getSql(), mappedStatement, boundSql, page, connection); - if (page.getTotal() <= 0L) { - return null; - } - } - - DbType dbType = Optional.ofNullable(this.dbType).orElse(JdbcUtils.getDbType(connection.getMetaData().getURL())); - IDialect dialect = Optional.ofNullable(this.dialect).orElse(DialectFactory.getDialect(dbType)); - String buildSql = concatOrderBy(originalSql, page); - DialectModel model = dialect.buildPaginationSql(buildSql, page.offset(), page.getSize()); - Configuration configuration = mappedStatement.getConfiguration(); - List mappings = new ArrayList(boundSql.getParameterMappings()); - Map additionalParameters = (Map) metaObject.getValue("delegate.boundSql.additionalParameters"); - model.consumers(mappings, configuration, additionalParameters); - metaObject.setValue("delegate.boundSql.sql", model.getDialectSql()); - metaObject.setValue("delegate.boundSql.parameterMappings", mappings); - return invocation.proceed(); - } else { - return invocation.proceed(); - } - } else { - return invocation.proceed(); - } - } - - /** - * 处理分页数量 - * - * @param page 分页参数 - */ - protected void handlerLimit(IPage page) { - page.setSize(this.limit); - } - - /** - * 查询总数量 - * - * @param sql sql语句 - * @param mappedStatement 映射语句包装类 - * @param boundSql sql包装类 - * @param page 分页参数 - * @param connection JDBC连接包装类 - */ - protected void queryTotal(String sql, MappedStatement mappedStatement, BoundSql boundSql, IPage page, Connection connection) { - try { - PreparedStatement statement = connection.prepareStatement(sql); - Throwable var7 = null; - - try { - DefaultParameterHandler parameterHandler = new MybatisDefaultParameterHandler(mappedStatement, boundSql.getParameterObject(), boundSql); - parameterHandler.setParameters(statement); - long total = 0L; - ResultSet resultSet = statement.executeQuery(); - Throwable var12 = null; - - try { - if (resultSet.next()) { - total = resultSet.getLong(1); - } - } catch (Throwable var37) { - var12 = var37; - throw var37; - } finally { - if (resultSet != null) { - if (var12 != null) { - try { - resultSet.close(); - } catch (Throwable var36) { - var12.addSuppressed(var36); - } - } else { - resultSet.close(); - } - } - - } - - page.setTotal(total); - if (this.overflow && page.getCurrent() > page.getPages()) { - this.handlerOverflow(page); - } - } catch (Throwable var39) { - var7 = var39; - throw var39; - } finally { - if (statement != null) { - if (var7 != null) { - try { - statement.close(); - } catch (Throwable var35) { - var7.addSuppressed(var35); - } - } else { - statement.close(); - } - } - - } - - } catch (Exception var41) { - throw ExceptionUtils.mpe("Error: Method queryTotal execution error of sql : \n %s \n", var41, new Object[]{sql}); - } - } - - /** - * 设置默认当前页 - * - * @param page 分页参数 - */ - protected void handlerOverflow(IPage page) { - page.setCurrent(1L); - } - - /** - * MybatisPlus拦截器实现自定义插件 - * - * @param target 拦截目标对象 - * @return - */ - @Override - public Object plugin(Object target) { - return target instanceof StatementHandler ? Plugin.wrap(target, this) : target; - } - - /** - * MybatisPlus拦截器实现自定义属性设置 - * - * @param prop 属性参数 - */ - @Override - public void setProperties(Properties prop) { - String dialectType = prop.getProperty("dialectType"); - String dialectClazz = prop.getProperty("dialectClazz"); - if (StringUtils.isNotBlank(dialectType)) { - this.setDialectType(dialectType); - } - - if (StringUtils.isNotBlank(dialectClazz)) { - this.setDialectClazz(dialectClazz); - } - - } - - /** - * 设置数据源类型 - * - * @param dialectType 数据源类型 - */ - @Deprecated - public void setDialectType(String dialectType) { - this.setDbType(DbType.getDbType(dialectType)); - } - - - /** - * 设置方言实现类配置 - * - * @param dialectClazz 方言实现类 - */ - @Deprecated - public void setDialectClazz(String dialectClazz) { - this.setDialect(DialectFactory.getDialect(dialectClazz)); - } - - /** - * 设置获取总数的sql解析器 - * - * @param countSqlParser 总数的sql解析器 - * @return 自定义MybatisPlus拦截器 - */ - public PaginationInterceptor setCountSqlParser(final ISqlParser countSqlParser) { - this.countSqlParser = countSqlParser; - return this; - } - - /** - * 溢出总页数，设置第一页 - * - * @param overflow 溢出总页数 - * @return 自定义MybatisPlus拦截器 - */ - public PaginationInterceptor setOverflow(final boolean overflow) { - this.overflow = overflow; - return this; - } - - /** - * 设置分页规则 - * - * @param limit 分页数量 - * @return 自定义MybatisPlus拦截器 - */ - public PaginationInterceptor setLimit(final long limit) { - this.limit = limit; - return this; - } - - /** - * 设置数据类型 - * - * @param dbType 数据类型 - * @return 自定义MybatisPlus拦截器 - */ - public PaginationInterceptor setDbType(final DbType dbType) { - this.dbType = dbType; - return this; - } - - /** - * 设置方言 - * - * @param dialect 方言 - * @return 自定义MybatisPlus拦截器 - */ - public PaginationInterceptor setDialect(final IDialect dialect) { - this.dialect = dialect; - return this; - } - - -} - diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java deleted file mode 100644 index 5399f7c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/AesUtil.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.util.HexUtil; -import org.dubhe.enums.LogEnum; - -import javax.crypto.Cipher; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.spec.SecretKeySpec; -import java.nio.charset.StandardCharsets; -import java.security.InvalidKeyException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -/** - * @description AES加解密工具 - * @date 2020-06-01 - */ -public class AesUtil { - - private static final String AES = "AES"; - - - private AesUtil(){ - - } - - - /** - * - * @param mode Cipher mode - * @param key 秘钥 - * @return Cipher - * @throws NoSuchAlgorithmException - * @throws NoSuchPaddingException - * @throws InvalidKeyException - */ - private static Cipher getCipher(int mode,String key) throws NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException { - MessageDigest md5Digest = MessageDigest.getInstance("MD5"); - SecretKeySpec secretKeySpec = new SecretKeySpec(md5Digest.digest(key.getBytes(StandardCharsets.UTF_8)), AES); - Cipher cipher = Cipher.getInstance(AES); - cipher.init(mode, secretKeySpec); - return cipher; - } - - /** - * 加密 - * - * @param data 原文 - * @param key 秘钥 - * @return String 密文 - */ - public static String encrypt(String data, String key) { - try { - Cipher cipher = getCipher(Cipher.ENCRYPT_MODE,key); - byte[] content = data.getBytes(StandardCharsets.UTF_8); - return new String(HexUtil.encodeHex(cipher.doFinal(content), false)); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 加密失败 ",e); - return null; - } - } - - /** - * 解密 - * @param hexData 十六进制密文 - * @param key 秘钥 - * @return String 密文 - */ - public static String decrypt(String hexData, String key) { - try { - Cipher cipher = getCipher(Cipher.DECRYPT_MODE,key); - byte[] content = HexUtil.decodeHex(hexData); - return new String(cipher.doFinal(content), StandardCharsets.UTF_8); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_SYS,"Md5Util 解密失败 ",e); - return null; - } - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java deleted file mode 100644 index e55822e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/DateUtil.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.sql.Timestamp; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.time.*; -import java.util.Date; - -/** - * @description 日期工具类 - * @date 2020-06-10 - */ -public class DateUtil { - - private DateUtil(){ - - } - - - /** - * 获取当前时间戳 - * - * @return - */ - public static Timestamp getCurrentTimestamp() { - return Timestamp.valueOf(LocalDateTime.now()); - } - - - - /** - * 获取六小时后时间 - * @return - */ - public static long getAfterSixHourTime(){ - long l1 = getTimestampOfDateTime(LocalDateTime.now()); - long milli = getTimestampOfDateTime(LocalDateTime.now().plusHours(6)); - return (milli-l1); - } - - - /** - * LocalDateTime -> long - * @param localDateTime - * @return - */ - public static long getTimestampOfDateTime(LocalDateTime localDateTime) { - ZoneId zone = ZoneId.systemDefault(); - Instant instant = localDateTime.atZone(zone).toInstant(); - return instant.toEpochMilli(); - } - - /** - * 获取第二天凌晨时间 - * @return - */ - public static long getSecondTime(){ - LocalDateTime localDateTime = LocalDateTime.now(); - long l1 = localDateTime.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli(); - - LocalDate localDate = LocalDate.now(); - LocalDate localDate1 = localDate.plusDays(1); - LocalDateTime localDateTime1 = localDate1.atStartOfDay(); - long milli = localDateTime1.atZone(ZoneId.of("Asia/Shanghai")).toInstant().toEpochMilli(); - System.out.println("当前时间到第二天凌晨的毫秒数"+(milli-l1)); - return (milli-l1); - } - - /** - * @return 当前字符串时间yyyy-MM-dd HH:mm:ss SSS - */ - public static String getCurrentTimeStr(){ - Date date = new Date(); - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS"); - return dateFormat.format(date); - } - - /** - * - * @return 当前字符串时间yyyyMMddHHmmss - */ - public static String getTimestampStr() { - SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmssSSS"); - return df.format(System.currentTimeMillis()); - } - - /** - * 获取当前时间与目标时间相差秒数 - * @param timestamp 目标时间 - * @return - */ - public static long getSeconds(Timestamp timestamp) { - LocalDateTime time = timestamp.toLocalDateTime(); - LocalDateTime now = LocalDateTime.now(); - Duration duration = Duration.between(time, now); - return duration.abs().getSeconds(); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java deleted file mode 100644 index b9f77f0..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/DubheDateUtil.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -/** - * @description 日期工具类 - * @date 2020-5-29 - **/ -public class DubheDateUtil { - - private static final int SIXTY = 60; - - /** - * 将时间 秒 转为 小时 - * - * @param second 秒数 - * @return - */ - public static String convert2Str(Long second) { - if (null == second || second < 1) { - return ""; - } - Long hh = second / (SIXTY * SIXTY); - second = second % (SIXTY * SIXTY); - Long mm = second / SIXTY; - Long ss = second % SIXTY; - - StringBuilder sb = new StringBuilder(); - sb.append(String.format("%02d", hh)).append(":").append(String.format("%02d", mm)).append(":").append(String.format("%02d", ss)); - return sb.toString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java deleted file mode 100644 index f05cbb1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/EncryptUtils.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.DESKeySpec; -import javax.crypto.spec.IvParameterSpec; -import java.nio.charset.StandardCharsets; - -/** - * @description 加密 - * @date 2020-03-13 - */ -public class EncryptUtils { - - private static String strParam = "Passw0rd"; - - private static Cipher cipher; - - private static IvParameterSpec iv = new IvParameterSpec(strParam.getBytes(StandardCharsets.UTF_8)); - - private static DESKeySpec getDesKeySpec(String source) throws Exception { - if (source == null || source.length() == 0) { - return null; - } - cipher = Cipher.getInstance("DES/CBC/PKCS5Padding"); - String strKey = "Passw0rd"; - return new DESKeySpec(strKey.getBytes(StandardCharsets.UTF_8)); - } - - /** - * 对称加密 - */ - public static String desEncrypt(String source) throws Exception { - DESKeySpec desKeySpec = getDesKeySpec(source); - SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); - SecretKey secretKey = keyFactory.generateSecret(desKeySpec); - cipher.init(Cipher.ENCRYPT_MODE, secretKey, iv); - return byte2hex( - cipher.doFinal(source.getBytes(StandardCharsets.UTF_8))).toUpperCase(); - } - - /** - * 对称解密 - */ - public static String desDecrypt(String source) throws Exception { - byte[] src = hex2byte(source.getBytes()); - DESKeySpec desKeySpec = getDesKeySpec(source); - SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("DES"); - SecretKey secretKey = keyFactory.generateSecret(desKeySpec); - cipher.init(Cipher.DECRYPT_MODE, secretKey, iv); - byte[] retByte = cipher.doFinal(src); - return new String(retByte); - } - - private static String byte2hex(byte[] inStr) { - String stmp; - StringBuilder out = new StringBuilder(inStr.length * 2); - for (byte b : inStr) { - stmp = Integer.toHexString(b & 0xFF); - if (stmp.length() == 1) { - // 如果是0至F的单位字符串，则添加0 - out.append("0").append(stmp); - } else { - out.append(stmp); - } - } - return out.toString(); - } - - private static byte[] hex2byte(byte[] b) { - int size = 2; - if ((b.length % size) != 0) { - throw new IllegalArgumentException("长度不是偶数"); - } - byte[] b2 = new byte[b.length / 2]; - for (int n = 0; n < b.length; n += size) { - String item = new String(b, n, 2); - b2[n / 2] = (byte) Integer.parseInt(item, 16); - } - return b2; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java deleted file mode 100644 index fe3a3b1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/FileUtil.java +++ /dev/null @@ -1,409 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.codec.Base64; -import cn.hutool.core.io.IoUtil; -import cn.hutool.core.util.CharsetUtil; -import cn.hutool.core.util.IdUtil; -import cn.hutool.poi.excel.BigExcelWriter; -import cn.hutool.poi.excel.ExcelUtil; -import org.apache.poi.util.IOUtils; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.springframework.web.multipart.MultipartFile; - -import javax.servlet.ServletOutputStream; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.security.MessageDigest; -import java.text.DecimalFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; - -/** - * @description File工具类，扩展 hutool 工具包 - * @date 2020-03-14 - */ -public class FileUtil extends cn.hutool.core.io.FileUtil { - - /** - * 定义GB的计算常量 - */ - private static final int GB = 1024 * 1024 * 1024; - /** - * 定义MB的计算常量 - */ - private static final int MB = 1024 * 1024; - /** - * 定义KB的计算常量 - */ - private static final int KB = 1024; - - /** - * 格式化小数 - */ - private static final DecimalFormat DF = new DecimalFormat("0.00"); - - /** - * MultipartFile转File - */ - public static File toFile(MultipartFile multipartFile) { - // 获取文件名 - String fileName = multipartFile.getOriginalFilename(); - // 获取文件后缀 - String prefix = "." + getExtensionName(fileName); - File file = null; - try { - // 用uuid作为文件名，防止生成的临时文件重复 - file = File.createTempFile(IdUtil.simpleUUID(), prefix); - // MultipartFile to File - multipartFile.transferTo(file); - } catch (IOException e) { - e.printStackTrace(); - } - return file; - } - - /** - * 获取文件扩展名，不带 . - */ - public static String getExtensionName(String filename) { - if ((filename != null) && (filename.length() > 0)) { - int dot = filename.lastIndexOf('.'); - if ((dot > -1) && (dot < (filename.length() - 1))) { - return filename.substring(dot + 1); - } - } - return filename; - } - - /** - * Java文件操作 获取不带扩展名的文件名 - */ - public static String getFileNameNoEx(String filename) { - if ((filename != null) && (filename.length() > 0)) { - int dot = filename.lastIndexOf('.'); - if ((dot > -1) && (dot < (filename.length()))) { - return filename.substring(0, dot); - } - } - return filename; - } - - /** - * 文件大小转换 - */ - public static String getSize(long size) { - String resultSize; - if (size / GB >= 1) { - //如果当前Byte的值大于等于1GB - resultSize = DF.format(size / (float) GB) + "GB "; - } else if (size / MB >= 1) { - //如果当前Byte的值大于等于1MB - resultSize = DF.format(size / (float) MB) + "MB "; - } else if (size / KB >= 1) { - //如果当前Byte的值大于等于1KB - resultSize = DF.format(size / (float) KB) + "KB "; - } else { - resultSize = size + "B "; - } - return resultSize; - } - - /** - * inputStream 转 File - */ - static File inputStreamToFile(InputStream ins, String name) throws Exception { - File file = new File(System.getProperty("java.io.tmpdir") + File.separator + name); - if (file.exists()) { - return file; - } - OutputStream os = new FileOutputStream(file); - int bytesRead; - int len = 8192; - byte[] buffer = new byte[len]; - while ((bytesRead = ins.read(buffer, 0, len)) != -1) { - os.write(buffer, 0, bytesRead); - } - os.close(); - ins.close(); - return file; - } - - /** - * 将文件名解析成文件的上传路径 - */ - public static File upload(MultipartFile file, String filePath) { - Date date = new Date(); - SimpleDateFormat format = new SimpleDateFormat("yyyyMMddhhmmssS"); - String name = getFileNameNoEx(file.getOriginalFilename()); - String suffix = getExtensionName(file.getOriginalFilename()); - String nowStr = "-" + format.format(date); - try { - String fileName = name + nowStr + "." + suffix; - String path = filePath + fileName; - // getCanonicalFile 可解析正确各种路径 - File dest = new File(path).getCanonicalFile(); - // 检测是否存在目录 - if (!dest.getParentFile().exists()) { - dest.getParentFile().mkdirs(); - } - // 文件写入 - file.transferTo(dest); - return dest; - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public static String fileToBase64(File file) throws Exception { - FileInputStream inputFile = new FileInputStream(file); - String base64; - byte[] buffer = new byte[(int) file.length()]; - inputFile.read(buffer); - inputFile.close(); - base64 = Base64.encode(buffer); - return base64.replaceAll("[\\s*\t\n\r]", ""); - } - - /** - * 导出excel - */ - public static void downloadExcel(List> list, HttpServletResponse response) throws IOException { - String tempPath = System.getProperty("java.io.tmpdir") + IdUtil.fastSimpleUUID() + ".xlsx"; - File file = new File(tempPath); - BigExcelWriter writer = ExcelUtil.getBigWriter(file); - // 一次性写出内容，使用默认样式，强制输出标题 - writer.write(list, true); - //response为HttpServletResponse对象 - response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;charset=utf-8"); - //test.xls是弹出下载对话框的文件名，不能为中文，中文请自行编码 - response.setHeader("Content-Disposition", "attachment;filename=file.xlsx"); - ServletOutputStream out = response.getOutputStream(); - // 终止后删除临时文件 - file.deleteOnExit(); - writer.flush(out, true); - //此处记得关闭输出Servlet流 - IoUtil.close(out); - } - - /** - * 下载文件 - */ - public static void download(String path, HttpServletResponse response) { - if (path == null) { - return; - } - FileInputStream fis = null; - ServletOutputStream out = null; - try { - File file = new File(path); - fis = new FileInputStream(file); - out = response.getOutputStream(); - IOUtils.copy(fis, out); - response.flushBuffer(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (fis != null) { - try { - fis.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - //此处记得关闭输出Servlet流 - IoUtil.close(out); - } - } - - public static String getFileType(String type) { - String documents = "txt doc pdf ppt pps xlsx xls docx"; - String music = "mp3 wav wma mpa ram ra aac aif m4a"; - String video = "avi mpg mpe mpeg asf wmv mov qt rm mp4 flv m4v webm ogv ogg"; - String image = "bmp dib pcp dif wmf gif jpg tif eps psd cdr iff tga pcd mpt png jpeg"; - if (image.contains(type)) { - return "图片"; - } else if (documents.contains(type)) { - return "文档"; - } else if (music.contains(type)) { - return "音乐"; - } else if (video.contains(type)) { - return "视频"; - } else { - return "其他"; - } - } - - public static void checkSize(long maxSize, long size) { - // 1M - int len = 1024 * 1024; - if (size > (maxSize * len)) { - throw new BusinessException("文件超出规定大小"); - } - } - - /** - * 判断两个文件是否相同 - */ - public static boolean check(File file1, File file2) { - String img1Md5 = getMd5(file1); - String img2Md5 = getMd5(file2); - return img1Md5.equals(img2Md5); - } - - /** - * 判断两个文件是否相同 - */ - public static boolean check(String file1Md5, String file2Md5) { - return file1Md5.equals(file2Md5); - } - - private static byte[] getByte(File file) { - // 得到文件长度 - byte[] b = new byte[(int) file.length()]; - try { - InputStream in = new FileInputStream(file); - try { - in.read(b); - } catch (IOException e) { - e.printStackTrace(); - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - return null; - } - return b; - } - - private static String getMd5(byte[] bytes) { - // 16进制字符 - char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - try { - MessageDigest mdTemp = MessageDigest.getInstance("MD5"); - mdTemp.update(bytes); - byte[] md = mdTemp.digest(); - int j = md.length; - char[] str = new char[j * 2]; - int k = 0; - // 移位 输出字符串 - for (byte byte0 : md) { - str[k++] = hexDigits[byte0 >>> 4 & 0xf]; - str[k++] = hexDigits[byte0 & 0xf]; - } - return new String(str); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - /** - * 下载文件 - * - * @param request / - * @param response / - * @param file / - */ - public static void downloadFile(HttpServletRequest request, HttpServletResponse response, File file, boolean deleteOnExit) { - response.setCharacterEncoding(request.getCharacterEncoding()); - response.setContentType("application/octet-stream"); - FileInputStream fis = null; - try { - fis = new FileInputStream(file); - response.setHeader("Content-Disposition", "attachment; filename=" + file.getName()); - IOUtils.copy(fis, response.getOutputStream()); - response.flushBuffer(); - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (fis != null) { - try { - fis.close(); - if (deleteOnExit) { - file.deleteOnExit(); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - public static String getMd5(File file) { - return getMd5(getByte(file)); - } - - - /** - * 生成文件 - * @param filePath 文件绝对路径 - * @param content 文件内容 - * @param append 文件是否是追加 - * @return - */ - public static boolean generateFile(String filePath,String content,boolean append){ - File file = new File(filePath); - FileOutputStream outputStream = null; - try { - if (!file.exists()){ - file.createNewFile(); - } - outputStream = new FileOutputStream(file,append); - outputStream.write(content.getBytes(CharsetUtil.defaultCharset())); - outputStream.flush(); - }catch (IOException e) { - LogUtil.error(LogEnum.FILE_UTIL,e); - return false; - }finally { - if (outputStream != null){ - try { - outputStream.close(); - } catch (IOException e) { - LogUtil.error(LogEnum.FILE_UTIL,e); - } - } - } - return true; - } - - - /** - * 压缩文件目录 - * - * @param zipDir 待压缩文件夹路径 - * @param zipFile 压缩完成zip文件绝对路径 - * @return - */ - public static boolean zipPath(String zipDir,String zipFile) { - if (zipDir == null) { - return false; - } - File zip = new File(zipFile); - cn.hutool.core.util.ZipUtil.zip(zip, CharsetUtil.defaultCharset(), true, - (f) -> !f.isDirectory(), - new File(zipDir).listFiles()); - return true; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java deleted file mode 100644 index 9e09cb1..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpClientUtils.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; -import org.apache.commons.io.IOUtils; -import org.dubhe.enums.LogEnum; - - - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URL; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import org.apache.commons.codec.binary.Base64; -import static org.dubhe.constant.StringConstant.UTF8; -import static org.dubhe.constant.SymbolConstant.BLANK; - -/** - * @description httpClient工具类，不校验SSL证书 - * @date 2020-05-21 - */ -public class HttpClientUtils { - - public static String sendHttps(String path) { - InputStream inputStream = null; - BufferedReader bufferedReader = null; - InputStreamReader inputStreamReader = null; - StringBuilder stringBuilder = new StringBuilder(); - String result = BLANK; - HttpsURLConnection con = null; - try { - con = getConnection(path); - con.connect(); - - /**将返回的输入流转换成字符串**/ - inputStream = con.getInputStream(); - inputStreamReader = new InputStreamReader(inputStream, UTF8); - bufferedReader = new BufferedReader(inputStreamReader); - - String str = null; - while ((str = bufferedReader.readLine()) != null) { - stringBuilder.append(str); - } - - result = stringBuilder.toString(); - LogUtil.info(LogEnum.BIZ_SYS,"Request path:{}, SUCCESS, result:{}", path, result); - - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, ERROR, exception:{}", path, e); - return result; - } finally { - closeResource(bufferedReader,inputStreamReader,inputStream,con); - } - - return result; - } - public static String sendHttpsDelete(String path,String username,String password) { - InputStream inputStream = null; - BufferedReader bufferedReader = null; - InputStreamReader inputStreamReader = null; - StringBuilder stringBuilder = new StringBuilder(); - String result = BLANK; - HttpsURLConnection con = null; - try { - con = getConnection(path); - String input =username+ ":" +password; - String encoding=Base64.encodeBase64String(input.getBytes()); - con.setRequestProperty(JwtUtils.AUTH_HEADER, "Basic " + encoding); - con.setRequestMethod("DELETE"); - con.connect(); - /**将返回的输入流转换成字符串**/ - inputStream = con.getInputStream(); - inputStreamReader = new InputStreamReader(inputStream, UTF8); - bufferedReader = new BufferedReader(inputStreamReader); - - String str = null; - while ((str = bufferedReader.readLine()) != null) { - stringBuilder.append(str); - } - - result = stringBuilder.toString(); - LogUtil.info(LogEnum.BIZ_SYS,"Request path:{}, SUCCESS, result:{}", path, result); - - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, ERROR, exception:{}", path, e); - return result; - } finally { - closeResource(bufferedReader,inputStreamReader,inputStream,con); - } - - return result; - } - - private static void closeResource(BufferedReader bufferedReader,InputStreamReader inputStreamReader,InputStream inputStream,HttpsURLConnection con) { - if (inputStream != null) { - IOUtils.closeQuietly(inputStream); - } - - if (inputStreamReader != null) { - IOUtils.closeQuietly(inputStreamReader); - } - if (bufferedReader != null) { - IOUtils.closeQuietly(bufferedReader); - } - if (con != null) { - con.disconnect(); - } - } - - - private static HttpsURLConnection getConnection(String path){ - HttpsURLConnection con = null; - try { - /**创建并初始化SSLContext对象**/ - TrustManager[] trustManagers = {new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - } - - @Override - public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }}; - SSLContext sslContext = SSLContext.getInstance("SSL", "SunJSSE"); - sslContext.init(null, trustManagers, new java.security.SecureRandom()); - - /**得到SSLSocketFactory对象**/ - SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); - - URL url = new URL(path); - con = (HttpsURLConnection) url.openConnection(); - con.setSSLSocketFactory(sslSocketFactory); - con.setUseCaches(false); - }catch (Exception e){ - LogUtil.error(LogEnum.BIZ_SYS,"Request path:{}, error, exception:{}", path, e); - } - return con; - - } - -} - - diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java deleted file mode 100644 index cb31df0..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/HttpUtils.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import lombok.extern.slf4j.Slf4j; - -/** - * @description HttpUtil - * @date 2020-04-30 - */ -@Slf4j -public class HttpUtils { - - private HttpUtils() { - - } - - /** - * 判断http请求是否成功 - * - * @param httpCode - * 1XX Informational(信息性状态码) - * 2XX Success(成功状态码) - * 3XX Redirection(重定向状态码) - * 4XX Client Error(客户端错误状态码) - * 5XX Server Error(服务器错误状态码) - * @return - */ - public static boolean isSuccess(String httpCode){ - if (StringUtils.isBlank(httpCode)){ - return false; - } - return httpCode.length() == 3 && httpCode.startsWith("2"); - } - - /** - * 判断http请求是否成功 - * - * @param httpCode - * 1XX Informational(信息性状态码) - * 2XX Success(成功状态码) - * 3XX Redirection(重定向状态码) - * 4XX Client Error(客户端错误状态码) - * 5XX Server Error(服务器错误状态码) - * @return - */ - public static boolean isSuccess(int httpCode) { - return isSuccess(String.valueOf(httpCode)); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java deleted file mode 100644 index e82e0be..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/IOUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.utils; - -import org.dubhe.enums.LogEnum; - -import java.io.Closeable; -import java.io.IOException; - -/** - * @description IO流操作工具类 - * @date 2020-10-14 - */ -public class IOUtil { - - /** - * 循环的依次关闭流 - * - * @param closeableList 要被关闭的流集合 - */ - public static void close(Closeable... closeableList) { - for (Closeable closeable : closeableList) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException e) { - LogUtil.error(LogEnum.IO_UTIL, "关闭流异常，异常信息：{}", e); - } - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java deleted file mode 100644 index a69eb78..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/JwcConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; -/** - * @description JWT配置类 - * @date 2020-04-15 - */ -@Configuration -public class JwcConfig { - @Value("${jwt.online-key}") - public String onlineKey; - @Autowired - public RedisUtils redisUtils; - @Value("${jwt.token-validity-in-seconds}") - public Long jwtExpiration; - @Value("${jwt.base64-secret}") - public String secret; -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java deleted file mode 100644 index 032f70a..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/JwtUtils.java +++ /dev/null @@ -1,203 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.auth0.jwt.JWT; -import com.auth0.jwt.JWTCreator.Builder; -import com.auth0.jwt.JWTVerifier; -import com.auth0.jwt.algorithms.Algorithm; -import com.auth0.jwt.exceptions.JWTDecodeException; -import com.auth0.jwt.interfaces.Claim; -import com.auth0.jwt.interfaces.DecodedJWT; -import org.apache.shiro.SecurityUtils; -import org.apache.shiro.crypto.SecureRandomNumberGenerator; -import org.apache.shiro.mgt.SecurityManager; -import org.apache.shiro.util.ThreadContext; -import org.dubhe.domain.dto.UserDTO; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.io.UnsupportedEncodingException; -import java.util.Calendar; -import java.util.Date; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * @description JWT - * @date 2020-03-14 - */ -@Component -public class JwtUtils { - - - /** - * 请求头 - */ - public static final String AUTH_HEADER = "Authorization"; - /** - * 过期时间20分钟 - */ - private static final long EXPIRE_TIME = 20 * 60 * 1000; - private static final String FIELD_NAME = "USER_NAME"; - private static JwcConfig jwcConfig; - private static RedisUtils redisUtils; - - - @Autowired - public void setJwcConfig(JwcConfig mtJwcConfig) { - jwcConfig = mtJwcConfig; - } - - @Autowired - public void setRedisUtils(RedisUtils myredisUtils) { - redisUtils = myredisUtils; - } - - /** - * 验证token是否正确 - */ - public static boolean verify(String token, String username) { - try { - Algorithm algorithm = null; - algorithm = Algorithm.HMAC256(jwcConfig.secret); - JWTVerifier verifier = JWT.require(algorithm).withClaim(FIELD_NAME, username).build(); - verifier.verify(token); - return true; - } catch (UnsupportedEncodingException exception) { - return false; - } - } - - /** - * 获得token中的自定义信息，无需secret解密也能获得 - */ - public static String getClaimFiled(String token, String filed) { - try { - DecodedJWT jwt = JWT.decode(token); - return jwt.getClaim(filed).asString(); - } catch (JWTDecodeException e) { - return null; - } - } - - /** - * 生成签名 - */ - public static String sign(String userName) { - try { - Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME); - Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret); - // 附带username， - String token = JWT.create().withClaim(FIELD_NAME, userName).withExpiresAt(date).sign(algorithm); - saveToken(userName, token); - return token; - } catch (UnsupportedEncodingException e) { - return null; - } - } - - /** - * 获取用户名 - */ - public static String getUserName(String token) { - return getClaimFiled(token, FIELD_NAME); - } - - /** - * 获取 token的签发时间 - */ - public static Date getIssuedAt(String token) { - try { - DecodedJWT jwt = JWT.decode(token); - return jwt.getIssuedAt(); - } catch (JWTDecodeException e) { - return null; - } - } - - /** - * 验证 token是否过期 - */ - public static boolean isTokenExpired(String token) { - Date now = Calendar.getInstance().getTime(); - DecodedJWT jwt = JWT.decode(token); - return jwt.getExpiresAt() == null || jwt.getExpiresAt().before(now); - } - - /** - * 刷新 token的过期时间 - */ - public static String refreshTokenExpired(String token) { - DecodedJWT jwt = JWT.decode(token); - Map claims = jwt.getClaims(); - try { - Date date = new Date(System.currentTimeMillis() + EXPIRE_TIME); - Algorithm algorithm = Algorithm.HMAC256(jwcConfig.secret); - Builder builer = JWT.create().withExpiresAt(date); - for (Entry entry : claims.entrySet()) { - builer.withClaim(entry.getKey(), entry.getValue().asString()); - } - String newToken = builer.sign(algorithm); - String userName = claims.get(FIELD_NAME).asString(); - String tokenKey = getUserTokenKey(userName); - if (redisUtils.get(tokenKey) == null) { - return null; - } - saveToken(userName, newToken); - return newToken; - } catch (UnsupportedEncodingException e) { - return null; - } - } - - /** - * 生成16位随机盐 - */ - public static String generateSalt() { - SecureRandomNumberGenerator secureRandom = new SecureRandomNumberGenerator(); - String hex = secureRandom.nextBytes(16).toHex(); - return hex; - } - - /** - * 获取当前用户 - */ - public static UserDTO getCurrentUserDto() { - Object principal = null; - SecurityManager manager = ThreadContext.getSecurityManager(); - if (!Objects.isNull(manager)) { - principal = SecurityUtils.getSubject().getPrincipal(); - } - return Objects.isNull(principal) ? null : (UserDTO) principal; - } - - /** - * 获取用户token key - */ - private static String getUserTokenKey(String userName) { - return jwcConfig.onlineKey + userName; - } - - private static void saveToken(String userName, String token) { - String tokenKey = getUserTokenKey(userName); - redisUtils.set(tokenKey, token, jwcConfig.jwtExpiration, TimeUnit.MINUTES); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java deleted file mode 100644 index 7dc31b9..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sCallBackTool.java +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.date.DateField; -import cn.hutool.core.date.DatePattern; -import cn.hutool.core.date.DateUtil; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -/** - * @description k8s命名相关工具类 token使用MD5加密，并设置超时时间 - * @date 2020-05-28 - */ -@Component -public class K8sCallBackTool { - - /** - * k8s 回调 token秘钥 - */ - @Value("${k8s.callback.token.secret-key}") - private String secretKey; - /** - * k8s 回调token超时时间 - */ - @Value("${k8s.callback.token.expire-seconds}") - private Integer expireSeconds; - /** - * k8s 回调域名或IP:Port - */ - @Value("${k8s.callback.url}") - private String url; - - /** - * k8s 回调token key - */ - public static final String K8S_CALLBACK_TOKEN = "k8sCallbackToken"; - /** - * 失败重试次数 - */ - private static final int RETRY_COUNT = 3; - - /** - * k8s 回调匹配地址 - */ - private static final List K8S_CALLBACK_PATH; - /** - * k8s 回调路径 - */ - private static final String K8S_CALLBACK_PATH_POD = "/api/k8s/callback/pod/"; - private static final String K8S_CALLBACK_PATH_DEPLOYMENT = "/api/k8s/callback/deployment/"; - - static { - K8S_CALLBACK_PATH = new ArrayList<>(); - // 添加需要token权限校验的地址（Shiro匿名访问的地址） - K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_POD + "**"); - K8S_CALLBACK_PATH.add(K8S_CALLBACK_PATH_DEPLOYMENT + "**"); - } - - /** - * 获取 k8s 回调匹配地址 - * - * @return List - */ - public static List getK8sCallbackPaths() { - return new ArrayList<>(K8S_CALLBACK_PATH); - } - - - /** - * 生成k8s回调 - * - * @return String - */ - public String generateToken() { - String expireTime = DateUtil.format( - DateUtil.offset(new Date(), DateField.SECOND, expireSeconds), - DatePattern.PURE_DATETIME_PATTERN - ); - return AesUtil.encrypt(expireTime, secretKey); - } - - /** - * 验证token - * - * @param token - * @return boolean - */ - public boolean validateToken(String token) { - String expireTime = AesUtil.decrypt(token, secretKey); - String nowTime = DateUtil.format( - new Date(), - DatePattern.PURE_DATETIME_PATTERN - ); - return expireTime.compareTo(nowTime) > 0; - } - - - /** - * 判断当前是否可以再次重试 - * - * @param retryTimes 第n次试图重试 - * @return boolean - */ - public boolean continueRetry(int retryTimes) { - return retryTimes <= RETRY_COUNT; - } - - /** - * 获取回调地址 - * - * @param podLabel - * @return String - */ - public String getPodCallbackUrl(String podLabel) { - return url + K8S_CALLBACK_PATH_POD + podLabel; - } - - /** - * 获取回调地址 - * - * @param businessLabel - * @return String - */ - public String getDeploymentCallbackUrl(String businessLabel) { - return url + K8S_CALLBACK_PATH_DEPLOYMENT + businessLabel; - } - - - /** - * 获取超时时间秒 - * - * @param timeoutSecond 超时秒数 - * @return Long - */ - public static Long getTimeoutSecondLong(int timeoutSecond) { - return Long.valueOf( - DateUtil.format( - DateUtil.offset( - new Date(), DateField.SECOND, timeoutSecond - ), - DatePattern.PURE_DATETIME_PATTERN - ) - ); - } - - /** - * 获取当前秒数 - * - * @return Long - */ - public static Long getCurrentSecondLong() { - return Long.valueOf( - DateUtil.format( - new Date(), - DatePattern.PURE_DATETIME_PATTERN - ) - ); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java deleted file mode 100644 index de3b916..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/K8sNameTool.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.date.DatePattern; -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.RandomUtil; -import org.dubhe.config.K8sNameConfig; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.BizNfsEnum; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.apache.commons.lang3.StringUtils; - -import java.util.Date; - -/** - * @description k8s命名相关工具类 - * @date 2020-05-13 - */ -@Component -public class K8sNameTool { - @Autowired - private K8sNameConfig k8sNameConfig; - /** - * 命名分隔符 - */ - private static final char SEPARATOR = '-'; - /** - * 文件分隔符 - */ - private static final String K8S_FILE_SEPARATOR = "/"; - /** - * 资源名称前缀 - */ - private static final String RESOURCE_NAME = "rn"; - /** - * 随机长度值 - */ - private static final int RANDOM_LENGTH = 4; - - /** - * 生成 ResourceName - * - * @param bizEnum 业务枚举 - * @param resourceInfo 资源备注信息（保证同业务下唯一并且命名规范） - * @return String - */ - public String generateResourceName(BizEnum bizEnum, String resourceInfo) { - return bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR + resourceInfo; - } - - /** - * 生成 Notebook的Namespace - * - * @param userId - * @return namespace - */ - public String generateNamespace(long userId) { - return this.k8sNameConfig.getNamespace() + SEPARATOR + userId; - } - - /** - * 从resourceName中获取资源信息 - * - * @param bizEnum 业务枚举 - * @param resourceName - * @return resourceInfo - */ - public String getResourceInfoFromResourceName(BizEnum bizEnum, String resourceName) { - if (StringUtils.isEmpty(resourceName) || !resourceName.contains(bizEnum.getBizCode() + SEPARATOR)) { - return null; - } - return resourceName.replace(bizEnum.getBizCode() + SEPARATOR + RESOURCE_NAME + SEPARATOR, SymbolConstant.BLANK); - } - - /** - * 从namespace 获取使用者ID - * - * @param namespace - * @return Long - */ - public Long getUserIdFromNamespace(String namespace) { - if (StringUtils.isEmpty(namespace) || !namespace.contains(this.k8sNameConfig.getNamespace() + SEPARATOR)) { - return null; - } - return Long.valueOf(namespace.replace(this.k8sNameConfig.getNamespace() + SEPARATOR, "")); - } - - - /** - * 生成业务模块相对路径 - * - * @param bizNfsEnum 业务NFS路径枚举 - * @return String 例如： /{biz}/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/ - */ - public String getNfsPath(BizNfsEnum bizNfsEnum, long userId) { - if (bizNfsEnum == null) { - return null; - } - return optimizationPath(K8S_FILE_SEPARATOR - + bizNfsEnum.getBizNfsPath() - + K8S_FILE_SEPARATOR - + userId - + K8S_FILE_SEPARATOR - + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH) - + K8S_FILE_SEPARATOR); - } - - /** - * 去除NFS根路径 - * - * @param nfsPath - * @return String - */ - public String removeNfsRootPath(String nfsPath) { - if (StringUtils.isBlank(nfsPath) || !nfsPath.startsWith(k8sNameConfig.getNfsRootPath())) { - return nfsPath; - } - return optimizationPath(K8S_FILE_SEPARATOR + nfsPath.replace(k8sNameConfig.getNfsRootPath(), "")); - } - - /** - * 路径添加bucket - * - * @param nfsPath - * @return String - */ - public String appendBucket(String nfsPath) { - return optimizationPath(K8S_FILE_SEPARATOR - + k8sNameConfig.getFileBucket() - + K8S_FILE_SEPARATOR - + nfsPath); - } - - /** - * 路径添加时间戳随机数 - * - * @param nfsPath - * @return String - */ - public String appendTimeStampAndRandomNum(String nfsPath) { - return optimizationPath(nfsPath - + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) + RandomUtil.randomString(RANDOM_LENGTH) - + K8S_FILE_SEPARATOR); - } - - /** - * nfs路径根据业务转换 - * - * @param nfsPath - * @param sourceBizNfsEnum 源业务 NFS Path - * @param targetBizNfsEnum 目标业务 NFS Path - * @return String - */ - public String convertNfsPath(String nfsPath, BizNfsEnum sourceBizNfsEnum, BizNfsEnum targetBizNfsEnum) { - if (!validateBizNfsPath(nfsPath, sourceBizNfsEnum) || targetBizNfsEnum == null) { - return nfsPath; - } - return optimizationPath(nfsPath.replace(K8S_FILE_SEPARATOR + sourceBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR - , K8S_FILE_SEPARATOR + targetBizNfsEnum.getBizNfsPath() + K8S_FILE_SEPARATOR)); - } - - /** - * 获取NFS绝对路径 - * - * @param nfsPath - * @return String - */ - public String getAbsoluteNfsPath(String nfsPath) { - if (StringUtils.isBlank(nfsPath)) { - return nfsPath; - } - return optimizationPath(k8sNameConfig.getNfsRootPath() - + K8S_FILE_SEPARATOR - + k8sNameConfig.getFileBucket() - + K8S_FILE_SEPARATOR - + nfsPath); - } - - /** - * 验证 nfsPath 是否是所属业务路径 - * - * @param nfsPath - * @param bizNfsEnum - * @return boolean - */ - public boolean validateBizNfsPath(String nfsPath, BizNfsEnum bizNfsEnum) { - return org.apache.commons.lang3.StringUtils.isNotBlank(nfsPath) - && bizNfsEnum != null - && nfsPath.contains(bizNfsEnum.getBizNfsPath()) - && !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getFileBucket() + K8S_FILE_SEPARATOR) - && !nfsPath.contains(K8S_FILE_SEPARATOR + k8sNameConfig.getNfsRootPath() + K8S_FILE_SEPARATOR); - } - - /** - * 路径优化 - * - * @param path - * @return String - */ - private String optimizationPath(String path) { - if (StringUtils.isBlank(path)) { - return path; - } - return path.replaceAll("///*", K8S_FILE_SEPARATOR); - } - - /** - * 获取k8s pod标签 - * - * @param bizEnum - * @return String - */ - public String getPodLabel(BizEnum bizEnum) { - return bizEnum == null ? null : bizEnum.getBizCode(); - } - - /** - * 获取数据集在镜像中路径 - * - * @return String - */ - public String getDatasetPath() { - return k8sNameConfig.getDatasetPath(); - } - - - /** - * 自送生成K8S名称，供K8S使用 - * - * @return String - */ - public String getK8sName() { - return DateUtil.format(new Date(), DatePattern.PURE_DATETIME_MS_FORMAT) - + RandomUtil.randomString(RANDOM_LENGTH); - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java deleted file mode 100644 index f3dcb62..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/KeyUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.text.SimpleDateFormat; -import java.util.Calendar; - -/** - * @description Key生成工具类 - * @date 2020-06-08 - */ -public class KeyUtil { - - /** - * 生成训练任务的key，目前规则为：train-{user_id}-{time} - * @param userId 用户id - * @return 训练任务的key - */ - public static String generateTrainKey(long userId) { - Calendar calendar = Calendar.getInstance(); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - return "train-" + userId + '-' + dateFormat.format(calendar.getTime()); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java deleted file mode 100644 index 5e77bc5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/LocalFileUtil.java +++ /dev/null @@ -1,402 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.utils; - -import cn.hutool.core.util.StrUtil; -import lombok.Getter; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipFile; -import org.apache.commons.io.IOUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.config.NfsConfig; -import org.dubhe.enums.LogEnum; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.FileCopyUtils; - -import java.io.*; -import java.util.Enumeration; -import java.util.Objects; -import java.util.zip.ZipEntry; - -/** - * @description 本地文件操作工具类 - * @date 2020-08-19 - */ -@Component -@Getter -public class LocalFileUtil { - - @Autowired - private NfsConfig nfsConfig; - - private static final String FILE_SEPARATOR = File.separator; - - private static final String ZIP = ".zip"; - - private static final String CHARACTER_GBK = "GBK"; - - private static final String OS_NAME = "os.name"; - - private static final String WINDOWS = "Windows"; - - /** - * nfs服务暴露的IP地址 - */ - @Value("${k8s.nfs}") - private String nfsIp; - - /** - * 文件存储服务器用户名 - */ - @Value("${data.server.userName}") - private String userName; - - /** - * 拷贝文件命令 - */ - public static final String COPY_COMMAND = "ssh %s@%s \"mkdir -p %s && cp -r %s %s && echo success\""; - /** - * 拷贝文件夹下文件命令 - */ - public static final String COPY_DIR_COMMAND = "ssh %s@%s \"mkdir -p %s && cp -rf %s* %s && echo success\""; - - @Value("${k8s.nfs-root-path}") - private String nfsRootPath; - - @Value("${k8s.nfs-root-windows-path}") - private String nfsRootWindowsPath; - - /** - * windows 与 linux 的路径兼容 - * - * @param path linux下的路径 - * @return path 兼容windows后的路径 - */ - private String compatiblePath(String path) { - if (path == null) { - return null; - } - if (System.getProperties().getProperty(OS_NAME).contains(WINDOWS)) { - path = path.replace(nfsRootPath, StrUtil.SLASH); - path = path.replace(StrUtil.SLASH, FILE_SEPARATOR); - path = nfsRootWindowsPath + path; - } - return path; - } - - - /** - * 本地解压zip包并删除压缩文件 - * - * @param sourcePath zip源文件 例如：/abc/z.zip - * @param targetPath 解压后的目标文件夹 例如：/abc/ - * @return boolean - */ - public boolean unzipLocalPath(String sourcePath, String targetPath) { - if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { - return false; - } - if (!sourcePath.toLowerCase().endsWith(ZIP)) { - return false; - } - //绝对路径 - String sourceAbsolutePath = nfsConfig.getRootDir() + sourcePath; - String targetPathAbsolutePath = nfsConfig.getRootDir() + targetPath; - ZipFile zipFile = null; - InputStream in = null; - OutputStream out = null; - File sourceFile = new File(compatiblePath(sourceAbsolutePath)); - File targetFileDir = new File(compatiblePath(targetPathAbsolutePath)); - if (!targetFileDir.exists()) { - boolean targetMkdir = targetFileDir.mkdirs(); - if (!targetMkdir) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}failed to create target folder before decompression", sourceAbsolutePath); - } - } - try { - zipFile = new ZipFile(sourceFile); - //判断压缩文件编码方式,并重新获取文件对象 - try { - zipFile.close(); - zipFile = new ZipFile(sourceFile, CHARACTER_GBK); - } catch (Exception e) { - zipFile.close(); - zipFile = new ZipFile(sourceFile); - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}the encoding mode of decompressed compressed file is changed to UTF-8:{}", sourceAbsolutePath, e); - } - ZipEntry entry; - Enumeration enumeration = zipFile.getEntries(); - while (enumeration.hasMoreElements()) { - entry = (ZipEntry) enumeration.nextElement(); - String entryName = entry.getName(); - File fileDir; - if (entry.isDirectory()) { - fileDir = new File(targetPathAbsolutePath + entry.getName()); - if (!fileDir.exists()) { - boolean fileMkdir = fileDir.mkdirs(); - if (!fileMkdir) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "failed to create folder {} while decompressing {}", fileDir, sourceAbsolutePath); - } - } - } else { - //若文件夹未创建则创建文件夹 - if (entryName.contains(FILE_SEPARATOR)) { - String zipDirName = entryName.substring(MagicNumConstant.ZERO, entryName.lastIndexOf(FILE_SEPARATOR)); - fileDir = new File(targetPathAbsolutePath + zipDirName); - if (!fileDir.exists()) { - boolean fileMkdir = fileDir.mkdirs(); - if (!fileMkdir) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "failed to create folder {} while decompressing {}", fileDir, sourceAbsolutePath); - } - } - } - in = zipFile.getInputStream((ZipArchiveEntry) entry); - out = new FileOutputStream(new File(targetPathAbsolutePath, entryName)); - IOUtils.copyLarge(in, out); - in.close(); - out.close(); - } - } - boolean deleteZipFile = sourceFile.delete(); - if (!deleteZipFile) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}compressed file deletion failed after decompression", sourceAbsolutePath); - } - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}decompression failed: {}", sourceAbsolutePath, e); - return false; - } finally { - //关闭未关闭的io流 - closeIoFlow(sourceAbsolutePath, zipFile, in, out); - } - - } - - /** - * 关闭未关闭的io流 - * - * @param sourceAbsolutePath 源路径 - * @param zipFile 压缩文件对象 - * @param in 输入流 - * @param out 输出流 - */ - private void closeIoFlow(String sourceAbsolutePath, ZipFile zipFile, InputStream in, OutputStream out) { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}input stream shutdown failed: {}", sourceAbsolutePath, e); - } - } - if (out != null) { - try { - out.close(); - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}output stream shutdown failed: {}", sourceAbsolutePath, e); - } - } - if (zipFile != null) { - try { - zipFile.close(); - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "{}input stream shutdown failed: {}", sourceAbsolutePath, e); - } - } - } - - - /** - * 复制单个文件到指定目录下 单个文件 - * - * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return boolean - */ - private boolean copyLocalFile(String sourcePath, String targetPath) { - if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourcePath = formatPath(sourcePath); - targetPath = formatPath(targetPath); - try (InputStream input = new FileInputStream(sourcePath); - FileOutputStream output = new FileOutputStream(targetPath)) { - FileCopyUtils.copy(input, output); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to copy file original path: {} ,target path： {} ,copyLocalFile:{} ", sourcePath, targetPath, e); - return false; - } - } - - /** - * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 - * - * 通过本地文件复制方式 - * - * @param sourcePath 需要复制的文件目录 例如：/abc/def - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return boolean - */ - public boolean copyPath(String sourcePath, String targetPath) { - if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourcePath = formatPath(sourcePath); - targetPath = formatPath(targetPath); - try { - return copyLocalPath(nfsConfig.getRootDir() + sourcePath, nfsConfig.getRootDir() + targetPath); - } catch (Exception e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, " failed to Copy file original path: {} ,target path： {} ,copyPath: {}", sourcePath, targetPath, e); - return false; - } - } - - /** - * 复制文件 到指定目录下 多个文件 包含目录与文件并存情况 - * - * @param sourcePath 需要复制的文件目录 例如：/abc/def - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return boolean - */ - private boolean copyLocalPath(String sourcePath, String targetPath) { - if (!StringUtils.isEmpty(sourcePath) && !StringUtils.isEmpty(targetPath)) { - sourcePath = formatPath(sourcePath); - if (sourcePath.endsWith(FILE_SEPARATOR)) { - sourcePath = sourcePath.substring(MagicNumConstant.ZERO, sourcePath.lastIndexOf(FILE_SEPARATOR)); - } - targetPath = formatPath(targetPath); - File sourceFile = new File(sourcePath); - if (sourceFile.exists()) { - File[] files = sourceFile.listFiles(); - if (files != null && files.length != 0) { - for (File file : files) { - try { - if (file.isDirectory()) { - File fileDir = new File(targetPath + FILE_SEPARATOR + file.getName()); - if (!fileDir.exists()) { - fileDir.mkdirs(); - } - copyLocalPath(sourcePath + FILE_SEPARATOR + file.getName(), targetPath + FILE_SEPARATOR + file.getName()); - } - if (file.isFile()) { - File fileTargetPath = new File(targetPath); - if (!fileTargetPath.exists()) { - fileTargetPath.mkdirs(); - } - copyLocalFile(file.getAbsolutePath(), targetPath + FILE_SEPARATOR + file.getName()); - } - } catch (Exception e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "failed to copy folder original path: {} , target path ： {} ,copyLocalPath: {}", sourcePath, targetPath, e); - return false; - } - } - } - return true; - } - } - return false; - } - - /** - * 替换路径中多余的 "/" - * - * @param path - * @return String - */ - public String formatPath(String path) { - if (!StringUtils.isEmpty(path)) { - return path.replaceAll("///*", FILE_SEPARATOR); - } - return path; - } - - /** - * 拷贝文件 - * - * @param sourcePath 需要复制的文件 例如：/abc/def/cc.txt - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return - */ - public boolean copyFile(String sourcePath, String targetPath) { - //绝对路径 - String sourceAbsolutePath = formatPath(nfsConfig.getRootDir() + sourcePath); - String targetPathAbsolutePath = formatPath(nfsConfig.getRootDir() + targetPath); - String[] command = new String[]{"/bin/sh", "-c", String.format(COPY_COMMAND, userName, nfsIp, targetPathAbsolutePath, sourceAbsolutePath, targetPathAbsolutePath)}; - boolean flag = false; - Process process; - try { - process = Runtime.getRuntime().exec(command); - if (isCopySuccess(process)) { - flag = true; - } - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "copy file failed, filePath:{}, targetPath:{}", sourcePath, targetPath, e); - } - return flag; - } - - /** - * 拷贝文件夹下内容 - * @param sourcePath 需要复制的文件目录 例如：/abc/dd - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return - */ - public boolean copyDir(String sourcePath, String targetPath) { - //绝对路径 - String sourceAbsolutePath = formatPath(nfsConfig.getRootDir() + sourcePath); - String targetPathAbsolutePath = formatPath(nfsConfig.getRootDir() + targetPath); - String[] command = new String[]{"/bin/sh", "-c", String.format(COPY_DIR_COMMAND, userName, nfsIp, targetPathAbsolutePath, sourceAbsolutePath, targetPathAbsolutePath)}; - boolean flag = false; - Process process; - try { - process = Runtime.getRuntime().exec(command); - if (isCopySuccess(process)) { - flag = true; - } - } catch (IOException e) { - LogUtil.error(LogEnum.LOCAL_FILE_UTIL, "copy file failed, filePath:{}, targetPath:{}", sourcePath, targetPath, e); - } - return flag; - } - - /** - * 判断拷贝结果 - * - * @param process - * @return - */ - public boolean isCopySuccess(Process process) { - boolean flag = false; - try (InputStream stream = process.getInputStream(); - InputStreamReader iReader = new InputStreamReader(stream); - BufferedReader bReader = new BufferedReader(iReader)) { - String line; - while (Objects.nonNull(line = bReader.readLine())) { - boolean temp = line.contains("success"); - if (temp) { - flag = true; - } - } - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "Read stream failed : {}", e); - } - return flag; - } - -} \ No newline at end of file diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java deleted file mode 100644 index cc336cc..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/LogUtil.java +++ /dev/null @@ -1,320 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import ch.qos.logback.classic.Level; -import com.alibaba.fastjson.JSON; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.dubhe.aspect.LogAspect; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.domain.entity.LogInfo; -import org.dubhe.enums.LogEnum; -import org.slf4j.MDC; -import org.slf4j.MarkerFactory; -import org.slf4j.helpers.MessageFormatter; - -import java.util.Arrays; -import java.util.UUID; - -/** - * @description 日志工具类 - * @date 2020-06-29 - */ -@Slf4j -public class LogUtil { - - private static final String TRACE_TYPE = "TRACE_TYPE"; - - public static final String SCHEDULE_LEVEL = "SCHEDULE"; - - public static final String K8S_CALLBACK_LEVEL = "K8S_CALLBACK"; - - private static final String GLOBAL_REQUEST_LEVEL = "GLOBAL_REQUEST"; - - private static final String TRACE_LEVEL = "TRACE"; - - private static final String DEBUG_LEVEL = "DEBUG"; - - private static final String INFO_LEVEL = "INFO"; - - private static final String WARN_LEVEL = "WARN"; - - private static final String ERROR_LEVEL = "ERROR"; - - - public static void startScheduleTrace() { - MDC.put(TRACE_TYPE, SCHEDULE_LEVEL); - } - - public static void startK8sCallbackTrace() { - MDC.put(TRACE_TYPE, K8S_CALLBACK_LEVEL); - } - - public static void cleanTrace() { - MDC.clear(); - } - - /** - * info级别的日志 - * - * @param logType 日志类型 - * @param object 打印的日志参数 - * @return void - */ - - public static void info(LogEnum logType, Object... object) { - - logHandle(logType, Level.INFO, object); - } - - /** - * debug级别的日志 - * - * @param logType 日志类型 - * @param object 打印的日志参数 - * @return void - */ - public static void debug(LogEnum logType, Object... object) { - logHandle(logType, Level.DEBUG, object); - } - - /** - * error级别的日志 - * - * @param logType 日志类型 - * @param object 打印的日志参数 - * @return void - */ - public static void error(LogEnum logType, Object... object) { - errorObjectHandle(object); - logHandle(logType, Level.ERROR, object); - } - - /** - * warn级别的日志 - * - * @param logType 日志类型 - * @param object 打印的日志参数 - * @return void - */ - public static void warn(LogEnum logType, Object... object) { - logHandle(logType, Level.WARN, object); - } - - /** - * trace级别的日志 - * - * @param logType 日志类型 - * @param object 打印的日志参数 - * @return void - */ - public static void trace(LogEnum logType, Object... object) { - logHandle(logType, Level.TRACE, object); - } - - /** - * 日志处理 - * - * @param logType 日志类型 - * @param level 日志级别 - * @param object 打印的日志参数 - * @return void - */ - private static void logHandle(LogEnum logType, Level level, Object[] object) { - - LogInfo logInfo = generateLogInfo(logType, level, object); - - switch (logInfo.getLevel()) { - case TRACE_LEVEL: - log.trace(MarkerFactory.getMarker(TRACE_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case DEBUG_LEVEL: - log.debug(MarkerFactory.getMarker(DEBUG_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case GLOBAL_REQUEST_LEVEL: - logInfo.setLevel(null); - logInfo.setType(null); - logInfo.setLocation(null); - log.info(MarkerFactory.getMarker(GLOBAL_REQUEST_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case SCHEDULE_LEVEL: - log.info(MarkerFactory.getMarker(SCHEDULE_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case K8S_CALLBACK_LEVEL: - log.info(MarkerFactory.getMarker(K8S_CALLBACK_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case INFO_LEVEL: - log.info(MarkerFactory.getMarker(INFO_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case WARN_LEVEL: - log.warn(MarkerFactory.getMarker(WARN_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - case ERROR_LEVEL: - log.error(MarkerFactory.getMarker(ERROR_LEVEL), logJsonStringLengthLimit(logInfo)); - break; - default: - } - - } - - - /** - * 日志信息组装的内部方法 - * - * @param logType 日志类型 - * @param level 日志级别 - * @param object 打印的日志参数 - * @return LogInfo - */ - private static LogInfo generateLogInfo(LogEnum logType, Level level, Object[] object) { - - - LogInfo logInfo = new LogInfo(); - // 日志类型检测 - if (!LogEnum.isLogType(logType)) { - level = Level.ERROR; - object = new Object[MagicNumConstant.ONE]; - object[MagicNumConstant.ZERO] = "日志类型【".concat(String.valueOf(logType)).concat("】不正确！"); - logType = LogEnum.SYS_ERR; - } - - // 获取trace_id - if (StringUtils.isEmpty(MDC.get(LogAspect.TRACE_ID))) { - MDC.put(LogAspect.TRACE_ID, UUID.randomUUID().toString()); - } - // 设置logInfo的level,type,traceId属性 - logInfo.setLevel(level.levelStr) - .setType(logType.toString()) - .setTraceId(MDC.get(LogAspect.TRACE_ID)); - - - //自定义日志级别 - //LogEnum、 MDC中的 TRACE_TYPE 做日志分流标识 - if (Level.INFO.toInt() == level.toInt()) { - if (LogEnum.GLOBAL_REQ.equals(logType)) { - //info全局请求 - logInfo.setLevel(GLOBAL_REQUEST_LEVEL); - } else if (LogEnum.BIZ_K8S.equals(logType)) { - logInfo.setLevel(K8S_CALLBACK_LEVEL); - } else { - //schedule定时等 链路记录 - String traceType = MDC.get(TRACE_TYPE); - if (StringUtils.isNotBlank(traceType)) { - logInfo.setLevel(traceType); - } - } - } - - // 设置logInfo的堆栈信息 - setLogStackInfo(logInfo); - // 设置logInfo的info信息 - setLogInfo(logInfo, object); - // 截取logInfo的长度并转换成json字符串 - return logInfo; - } - - /** - * 设置loginfo的堆栈信息 - * - * @param logInfo 日志对象 - * @return void - */ - private static void setLogStackInfo(LogInfo logInfo) { - StackTraceElement[] elements = Thread.currentThread().getStackTrace(); - if (elements.length >= MagicNumConstant.SIX) { - StackTraceElement element = elements[MagicNumConstant.FIVE]; - logInfo.setLocation(String.format("%s#%s:%s", element.getClassName(), element.getMethodName(), element.getLineNumber())); - } - } - - /** - * 限制log日志的长度并转换成json - * - * @param logInfo 日志对象 - * @return String - */ - private static String logJsonStringLengthLimit(LogInfo logInfo) { - try { - - String jsonString = JSON.toJSONString(logInfo); - if (StringUtils.isBlank(jsonString)) { - return ""; - } - if (jsonString.length() > MagicNumConstant.TEN_THOUSAND) { - String trunk = logInfo.getInfo().toString().substring(MagicNumConstant.ZERO, MagicNumConstant.NINE_THOUSAND); - logInfo.setInfo(trunk); - jsonString = JSON.toJSONString(logInfo); - } - return jsonString; - - } catch (Exception e) { - logInfo.setLevel(Level.ERROR.levelStr).setType(LogEnum.SYS_ERR.toString()) - .setInfo("cannot serialize exception: " + ExceptionUtils.getStackTrace(e)); - return JSON.toJSONString(logInfo); - } - } - - /** - * 设置日志对象的info信息 - * - * @param logInfo 日志对象 - * @param object 打印的日志参数 - * @return void - */ - private static void setLogInfo(LogInfo logInfo, Object[] object) { - - if (object.length > MagicNumConstant.ONE) { - logInfo.setInfo(MessageFormatter.arrayFormat(object[MagicNumConstant.ZERO].toString(), - Arrays.copyOfRange(object, MagicNumConstant.ONE, object.length)).getMessage()); - - } else if (object.length == MagicNumConstant.ONE && object[MagicNumConstant.ZERO] instanceof Exception) { - logInfo.setInfo((ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ZERO]))); - log.error((ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ZERO]))); - } else if (object.length == MagicNumConstant.ONE) { - logInfo.setInfo(object[MagicNumConstant.ZERO] == null ? "" : object[MagicNumConstant.ZERO]); - } else { - logInfo.setInfo(""); - } - - } - - /** - * 处理Exception的情况 - * - * @param object 打印的日志参数 - * @return void - */ - private static void errorObjectHandle(Object[] object) { - - if (object.length == MagicNumConstant.TWO && object[MagicNumConstant.ONE] instanceof Exception) { - log.error(String.valueOf(object[MagicNumConstant.ZERO]), (Exception) object[MagicNumConstant.ONE]); - object[MagicNumConstant.ONE] = ExceptionUtils.getStackTrace((Exception) object[MagicNumConstant.ONE]); - - } else if (object.length >= MagicNumConstant.THREE) { - log.error(String.valueOf(object[MagicNumConstant.ZERO]), - Arrays.copyOfRange(object, MagicNumConstant.ONE, object.length)); - for (int i = 0; i < object.length; i++) { - if (object[i] instanceof Exception) { - object[i] = ExceptionUtils.getStackTrace((Exception) object[i]); - } - - } - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java deleted file mode 100644 index cf97b0f..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/MathUtils.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.NumberConstant; - -/** - * @description 计算工具类 - * @date 2020-06-04 - */ -public class MathUtils { - - private static String FLOAT_DIVISION_FORMATE = "%1$0"; - - /** - * 字符串整数加法 - * num1+num2 - * @param num1 - * @param num2 - * @return - */ - public static String add(String num1, String num2) { - return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO : Integer.valueOf(num1)) + (!RegexUtil.isDigits(num2) ? MagicNumConstant.ZERO : Integer.valueOf(num2))); - } - - /** - * 字符串整数减法 - * num1 - num2 - * @param num1 - * @param num2 - * @return - */ - public static String reduce(String num1, String num2) { - return String.valueOf((!RegexUtil.isDigits(num1) ? MagicNumConstant.ZERO : Integer.valueOf(num1)) - (!RegexUtil.isDigits(num2) ? MagicNumConstant.ZERO : Integer.valueOf(num2))); - } - - /** - * - * 浮点数除法 num1/num2 - * @param num1 - * @param num2 - * @param decimal 结果小数位数 - * @return - */ - public static Float floatDivision(String num1, String num2, Integer decimal) { - if (!RegexUtil.isFloat(num1) || !RegexUtil.isFloat(num2)) { - return null; - } - if (Float.valueOf(num2).equals(0f)) { - return null; - } - if (decimal != null && decimal > NumberConstant.NUMBER_0) { - Integer d = Integer.valueOf(MagicNumConstant.ONE + String.format(FLOAT_DIVISION_FORMATE + decimal + "d", NumberConstant.NUMBER_0)); - return (float) (Math.round((Float.valueOf(num1) / Float.valueOf(num2)) * d)) / d; - } else { - return Float.valueOf(num1) / Float.valueOf(num2); - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java deleted file mode 100644 index 4461288..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/MinioUtil.java +++ /dev/null @@ -1,343 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.io.IoUtil; -import cn.hutool.http.HttpRequest; -import com.alibaba.fastjson.JSONObject; -import io.minio.CopyConditions; -import io.minio.MinioClient; -import io.minio.PutObjectOptions; -import io.minio.Result; -import io.minio.errors.*; -import io.minio.messages.DeleteError; -import io.minio.messages.Item; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.apache.commons.lang.StringUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.NumberConstant; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import javax.annotation.PostConstruct; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.*; - -/** - * @description Minio工具类 - * @date 2020-05-09 - */ -@Service -public class MinioUtil { - - @Value("${minio.url}") - private String url; - @Value("${minio.accessKey}") - private String accessKey; - @Value("${minio.secretKey}") - private String secretKey; - - private MinioClient client; - - @PostConstruct - public void init() { - try { - client = new MinioClient(url, accessKey, secretKey); - } catch (InvalidEndpointException e) { - LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint invalid. e, {}", e); - } catch (InvalidPortException e) { - LogUtil.warn(LogEnum.BIZ_DATASET, "MinIO endpoint port invalid. e, {}", e); - } - } - - /** - * 写文件 - * - * @param bucket 桶名称 - * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt - * @param content file content. can not be null - */ - public void writeString(String bucket, String fullFilePath, String content) throws Exception { - boolean isExist = client.bucketExists(bucket); - if (!isExist) { - client.makeBucket(bucket); - } - InputStream inputStream = IoUtil.toUtf8Stream(content); - PutObjectOptions options = new PutObjectOptions(inputStream.available(), MagicNumConstant.NEGATIVE_ONE); - client.putObject(bucket, fullFilePath, inputStream, options); - } - - /** - * 写文件 - * - * @param bucket 桶名称 - * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt - * @param inputStream file inputStream. can not be null - */ - public void writeInputStream(String bucket, String fullFilePath, InputStream inputStream) throws Exception { - boolean isExist = client.bucketExists(bucket); - if (!isExist) { - client.makeBucket(bucket); - } - PutObjectOptions options = new PutObjectOptions(inputStream.available(), MagicNumConstant.NEGATIVE_ONE); - client.putObject(bucket, fullFilePath, inputStream, options); - } - - /** - * 读取文件 - * - * @param bucket 桶 - * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt - * @return String - */ - public String readString(String bucket, String fullFilePath) throws Exception { - try (InputStream is = client.getObject(bucket, fullFilePath)) { - return IoUtil.read(is, Charset.defaultCharset()); - } - } - - /** - * 文件删除 - * - * @param bucket 桶 - * @param fullFilePath 文件存储的全路径，包括文件名，非'/'开头. e.g. dataset/12/annotation/test.txt - */ - public void del(String bucket, String fullFilePath) throws Exception { - Iterable> items = client.listObjects(bucket, fullFilePath); - Set files = new HashSet<>(); - for (Result item : items) { - files.add(item.get().objectName()); - } - Iterable> results = client.removeObjects(bucket, files); - for (Result result : results) { - result.get(); - } - } - - /** - * 批量删除文件 - * - * @param bucket 桶 - * @param objectNames 对象名称 - */ - public void delFiles(String bucket, List objectNames) throws Exception { - Iterable> results = client.removeObjects(bucket, objectNames); - for (Result result : results) { - result.get(); - } - } - - /** - * 获取对象名称 - * - * @param bucketName 桶名称 - * @param prefix 前缀 - * @return List 对象名称列表 - * @throws Exception - */ - public List getObjects(String bucketName, String prefix) throws Exception { - List fileNames = new ArrayList<>(); - Iterable> results = client.listObjects(bucketName, prefix); - for (Result result : results) { - Item item = result.get(); - fileNames.add(item.objectName()); - } - return fileNames; - } - - /** - * 获取路径下文件数量 - * - * @param bucketName 桶名称 - * @param prefix 前缀 - * @return InputStream 文件流 - * @throws Exception - */ - public int getCount(String bucketName, String prefix) throws Exception { - int count = NumberConstant.NUMBER_0; - Iterable> results = client.listObjects(bucketName, prefix); - for (Result result : results) { - count++; - } - return count; - } - - /** - * 获取文件流 - * - * @param bucket 桶 - * @param objectName 对象名称 - * @return InputStream 文件流 - * @throws Exception - */ - public InputStream getObjectInputStream(String bucket, String objectName) throws Exception { - return client.getObject(bucket, objectName); - } - - /** - * 文件夹复制 - * - * @param bucket 桶 - * @param sourceFiles 源文件 - * @param targetDir 目标文件夹 - */ - public void copyDir(String bucket, List sourceFiles, String targetDir) { - sourceFiles.forEach(sourceFile -> { - InputStream inputStream = null; - try { - String sourceObjectName = sourceFile; - String targetObjectName = targetDir + "/" + StringUtils.substringAfterLast(sourceObjectName, "/"); - inputStream = client.getObject(bucket, sourceObjectName); - byte[] buf = new byte[512]; - int bytesRead; - int count = MagicNumConstant.ZERO; - while ((bytesRead = inputStream.read(buf, MagicNumConstant.ZERO, buf.length)) >= MagicNumConstant.ZERO) { - count += bytesRead; - } - PutObjectOptions options = new PutObjectOptions(count, MagicNumConstant.ZERO); - client.putObject(bucket, targetObjectName, client.getObject(bucket, sourceObjectName), options); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file copy exception, {}", e); - } finally { - try { - if (inputStream != null) { - inputStream.close(); - } - } catch (IOException e) { - LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file read stream closed failed, {}", e); - } - } - }); - } - - /** - * minio拷贝操作 - * - * @Param bucket 桶名 - * @Param sourceFiles 需要复制的标注文件名 - * @Param targetDir 目标文件夹路径 - */ - public void copyObject(String bucket, List sourceFiles, String targetDir) { - CopyConditions copyConditions = new CopyConditions(); - sourceFiles.forEach(sourceFile -> { - try { - String targetName = targetDir + "/" + org.dubhe.utils.StringUtils.substringAfterLast(sourceFile, "/"); - client.copyObject(bucket, targetName, null, null, bucket, sourceFile, null, copyConditions); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_DATASET, "MinIO file copy failed, {}", e); - } - }); - } - - @Data - @Service - public class MinioWebTokenBody { - - @Value("${minioweb.GetToken.url}") - private String tokenUrl; - @Value("${minioweb.GetToken.param.id}") - private int id; - @Value("${minioweb.GetToken.param.jsonrpc}") - private String jsonrpc; - @Value("${minioweb.GetToken.param.method}") - private String method; - @Value("${minioweb.zip.url}") - private String zipUrl; - - /** - * 生成文件下载请求参数方法 - * - * @param bucketName 桶名称 - * @param prefix 前缀 - * @param objects 对象名称 - * @return MinioDownloadDto 下载请求参数 - */ - public MinioDownloadDto getDownloadParam(String bucketName, String prefix, List objects, String zipName) { - String paramTemplate = "{\"id\":%d,\"jsonrpc\":\"%s\",\"params\":{\"username\":\"%s\",\"password\":\"%s\"},\"method\":\"%s\"}"; - String downloadBodyTemplate = "{\"bucketName\":\"%s\",\"prefix\":\"%s\",\"objects\":[%s]}"; - String param = String.format(paramTemplate, id, jsonrpc, accessKey, secretKey, method); - String result = HttpRequest.post(url + tokenUrl).contentType("application/json").body(param).execute().body(); - String token = JSONObject.parseObject(result).getJSONObject("result").getString("token"); - return new MinioDownloadDto(token, String.format(downloadBodyTemplate, bucketName, prefix, getStrFromList(objects)), zipName); - } - - public String getStrFromList(List objects) { - List result = new ArrayList<>(); - objects.stream().forEach(s -> { - result.add("\"" + s + "\""); - }); - return StringUtils.join(result, ","); - } - - } - - @ApiModel - @Data - public class MinioDownloadDto { - @ApiModelProperty("下载压缩包请求token") - private String token; - @ApiModelProperty("下载压缩包请求参数") - private String body; - @ApiModelProperty("下载压缩包请求需要的header") - private Map headers; - @ApiModelProperty("下载压缩包文件名称") - private String zipName; - - public MinioDownloadDto() { - } - - public MinioDownloadDto(String token, String body, String zipName) { - this.token = token; - this.body = body; - this.zipName = zipName; - Map headers = new HashMap<>(); - headers.put("Content-Type", "text/plain;charset=UTF-8"); - this.headers = headers; - } - - } - - - /** - * 生成一个给HTTP PUT请求用的presigned URL。浏览器/移动端的客户端可以用这个URL进行上传， - * 即使其所在的存储桶是私有的。这个presigned URL可以设置一个失效时间，默认值是7天 - * - * @param bucketName 存储桶名称 - * @param objectName 存储桶里的对象名称 - * @param expires 失效时间（以秒为单位），默认是7天，不得大于七天 - * @return String - */ - public String getEncryptedPutUrl(String bucketName, String objectName, Integer expires) { - if (StringUtils.isEmpty(objectName)) { - throw new BusinessException("object name cannot be empty"); - } - try { - return client.presignedPutObject(bucketName, objectName, expires); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_DATASET, e.getMessage()); - throw new BusinessException("MinIO an error occurred, please contact the administrator"); - } - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java deleted file mode 100644 index f47f304..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsFactory.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3; -import com.emc.ecs.nfsclient.rpc.CredentialUnix; -import org.apache.commons.pool2.PooledObject; -import org.apache.commons.pool2.PooledObjectFactory; -import org.apache.commons.pool2.impl.DefaultPooledObject; -import org.dubhe.config.NfsConfig; -import org.dubhe.enums.LogEnum; -import org.springframework.stereotype.Component; - -import java.io.IOException; - -/** - * @description NFS工厂类 - * @date 2020-05-13 - */ -@Component -public class NfsFactory implements PooledObjectFactory { - - private final NfsConfig nfsConfig; - - public NfsFactory(NfsConfig nfsConfig) { - this.nfsConfig = nfsConfig; - } - - @Override - public PooledObject makeObject() { - Nfs3 nfs3 = null; - try { - nfs3 = new Nfs3(nfsConfig.getNfsIp(), nfsConfig.getRootDir(), new CredentialUnix(0, 0, null), 3); - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "创建NFS对象失败: ", e); - } - return new DefaultPooledObject<>(nfs3); - } - - @Override - public void destroyObject(PooledObject pooledObject) { - LogUtil.info(LogEnum.NFS_UTIL, "销毁NFS对象: ", pooledObject.getObject()); - } - - @Override - public boolean validateObject(PooledObject pooledObject) { - LogUtil.info(LogEnum.NFS_UTIL, "验证NFS对象: ", pooledObject.getObject()); - return true; - } - - @Override - public void activateObject(PooledObject pooledObject) { - LogUtil.info(LogEnum.NFS_UTIL, "激活NFS对象: ", pooledObject.getObject()); - - } - - @Override - public void passivateObject(PooledObject pooledObject) { - LogUtil.info(LogEnum.NFS_UTIL, "钝化NFS对象: ", pooledObject.getObject()); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java deleted file mode 100644 index d6e031d..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsPool.java +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.emc.ecs.nfsclient.nfs.nfs3.Nfs3; -import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.dubhe.enums.LogEnum; -import org.springframework.stereotype.Component; - -/** - * @description NFS3 连接池 - * @date 2020-05-18 - */ -@Component -public class NfsPool { - /** - * NFS工厂对象 - */ - private NfsFactory nfsFactory; - /** - * GenericObjectPool对象 - */ - private final GenericObjectPool genericObjectPool; - /** - * 最大总共连接数 - */ - public static final int MAX_TOTAL = 300; - /** - * 最小连接数 - */ - public static final int MIN = 20; - /** - * 最大连接数 - */ - public static final int MAX = 300; - /** - * 最大等待时间 单位毫秒 - */ - public static final int MAX_WAIT_TIME = 3000; - - /** - * 初始化连接池 - * - * @param nfsFactory - */ - public NfsPool(NfsFactory nfsFactory) { - this.nfsFactory = nfsFactory; - GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); - poolConfig.setMaxTotal(MAX_TOTAL); - poolConfig.setMinIdle(MIN); - poolConfig.setMaxIdle(MAX); - poolConfig.setMaxWaitMillis(MAX_WAIT_TIME); - this.genericObjectPool = new GenericObjectPool<>(nfsFactory, poolConfig); - } - - /** - * 从连接池中取连接 - * - * @return nfs3 - */ - public Nfs3 getNfs() { - try { - LogUtil.info(LogEnum.NFS_UTIL,"NFS线程 活跃数量：{} ,空闲数量: {} , 等待队列数量 ： {}",genericObjectPool.getNumActive(),genericObjectPool.getNumIdle(),genericObjectPool.getNumWaiters()); - return genericObjectPool.borrowObject(); - } catch (Exception e) { - LogUtil.error(LogEnum.NFS_UTIL, "获取NFS连接失败: {} ", e); - return null; - } - } - - /** - * 释放连接到连接池 - * - * @param nfs3 - */ - public void revertNfs(Nfs3 nfs3) { - try { - if(nfs3 != null){ - LogUtil.info(LogEnum.NFS_UTIL,"成功释放对象 : {} ", nfs3); - genericObjectPool.returnObject(nfs3); - } - } catch (Exception e) { - LogUtil.error(LogEnum.NFS_UTIL, " 释放NFS连接失败: ", e); - } - } - - /** - * 销毁公共池 - */ - public void destroyPool() { - try { - genericObjectPool.close(); - } catch (Exception e) { - LogUtil.error(LogEnum.NFS_UTIL, "销毁NFS连接失败: ", e); - } - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java deleted file mode 100644 index 459b11b..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NfsUtil.java +++ /dev/null @@ -1,776 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.util.StrUtil; -import com.emc.ecs.nfsclient.nfs.io.Nfs3File; -import com.emc.ecs.nfsclient.nfs.io.NfsFileInputStream; -import com.emc.ecs.nfsclient.nfs.io.NfsFileOutputStream; -import lombok.Getter; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import org.apache.commons.io.IOUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.config.NfsConfig; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.NfsBizException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -/** - * @description NFS Util - * ！！！注意： 使用NFSFile 对象 业务上自行需要关闭 NFS 对象 - * @date 2020-05-12 - */ -@Component -@Getter -public class NfsUtil { - - private final NfsConfig nfsConfig; - - private static final String FILE_SEPARATOR = "/"; - - private static final String ZIP = ".zip"; - - private static final String CHARACTER_GBK = "GBK"; - - private static final String CHARACTER_UTF_8 = "UTF-8"; - - private static final String UNDER_LINE = "_"; - - @Autowired - private NfsPool nfsPool; - - - public NfsUtil(NfsConfig nfsConfig) { - this.nfsConfig = nfsConfig; - } - - private String getReplaceRootPathRegex() { - String rootPath = nfsConfig.getRootDir(); - return "^" + rootPath.substring(MagicNumConstant.ZERO, rootPath.length() - MagicNumConstant.ONE); - } - - /** - * 获取NFS3File 对象 - * - * @param path - * @return Nfs3File - */ - public Nfs3File getNfs3File(String path) { - if (StringUtils.isEmpty(formatPath(path))) { - LogUtil.error(LogEnum.NFS_UTIL, "传入的NFS3初始化路径 {} ,无法初始化NFS3 ", path); - throw new NfsBizException("初始化路径：" + path + " 不合法"); - } - Nfs3File nfs3File; - try { - nfs3File = new Nfs3File(nfsPool.getNfs(), path); - LogUtil.info(LogEnum.NFS_UTIL, "成功获取NFS3File对象 ： {} ", nfs3File.getName()); - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "获取NFS3File对象失败 ：", e); - throw new NfsBizException("未获取到NFS连接或者NFS连接池已满" + e.getMessage()); - } - return nfs3File; - } - - /** - * 获取NFS3File 对象文件的输入流 - * - * @param nfs3File - * @return BufferedInputStream - */ - public BufferedInputStream getInputStream(Nfs3File nfs3File) { - BufferedInputStream stream = null; - try { - if (!nfs3File.isFile()) { - throw new NfsBizException("此路径下查找到的对象不是文件类型，请检查文件类型是否正确！"); - } - stream = new BufferedInputStream(new NfsFileInputStream(nfs3File)); - } catch (IOException e) { - throw new NfsBizException("nfs获取对象输出流失败！"); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - return stream; - - } - - /** - * 获取NFS3File 对象文件的输入流 - * - * @param path - * @return BufferedInputStream - */ - public BufferedInputStream getInputStream(String path) { - Nfs3File nfs3File = getNfs3File(formatPath(path)); - if (nfs3File == null) { - throw new NfsBizException("此路径" + path + "下没有文件可以加载！"); - } - return getInputStream(nfs3File); - } - - /** - * 校验文件或文件夹是否不存在 - * - * @param path 文件路径 - * @return boolean - */ - public boolean fileOrDirIsEmpty(String path) { - if (!StringUtils.isEmpty(path)) { - path = formatPath(path.startsWith(nfsConfig.getRootDir()) ? path.replaceFirst(nfsConfig.getRootDir(), StrUtil.SLASH) : path); - Nfs3File nfs3File = getNfs3File(path); - try { - if (nfs3File.exists()) { - return false; - } - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "判断NFS File异常: ", e); - return true; - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - return true; - - } - - - /** - * 创建指定NFS目录 - * - * @param dir 需要创建的目录 例如：/abc/def - * @return boolean - */ - public boolean createDir(String dir) { - if (!StringUtils.isEmpty(dir)) { - dir = formatPath(dir); - String[] paths = dir.substring(MagicNumConstant.ONE).split(FILE_SEPARATOR); - StringBuilder sbPath = new StringBuilder(); - for (String path : paths) { - sbPath.append(FILE_SEPARATOR).append(path); - Nfs3File nfs3File = getNfs3File(sbPath.toString()); - try { - if (!nfs3File.exists()) { - nfs3File.mkdirs(); - } - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "创建NFS目录失败:", e); - return false; - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - return true; - } - return false; - } - - /** - * 创建多个指定目录 - * - * @param removeNfsRootPath - * @param paths - * @return boolean - */ - private boolean createDir(boolean removeNfsRootPath, String... paths) { - for (String path : paths) { - if (path == null) { - continue; - } - String formatPath = path; - if (removeNfsRootPath) { - formatPath = formatPath.replaceAll(this.getReplaceRootPathRegex(), ""); - } - boolean res = createDir(formatPath); - if (!res) { - return false; - } - } - return true; - } - - - /** - * 创建指定NFS目录 - * - * @param paths 路径 例如：/nfs/abc/def/ /abc/def/ - * @param nfsRootPath 是否包含nfs根目录 true:包含 false:不包含 - * @return boolean - */ - public boolean createDirs(boolean nfsRootPath, String... paths) { - if (null == paths || paths.length < MagicNumConstant.ONE) { - return true; - } - for (String path : paths) { - if (path == null) { - continue; - } - String formatPath = path; - if (nfsRootPath) { - formatPath = formatPath.replaceAll(this.getReplaceRootPathRegex(), ""); - } - if (!createDir(formatPath)) { - return false; - } - } - return true; - } - - /** - * 指定目录NFS中创建文件 - * - * @param dir 需要创建的目录 例如：/abc/def - * @param fileName 需要创建的文件 例如：dd.txt - * @return boolean - */ - public boolean createFile(String dir, String fileName) { - if (!StringUtils.isEmpty(dir) && !StringUtils.isEmpty(fileName)) { - dir = formatPath(dir); - Nfs3File nfs3File = getNfs3File(dir + FILE_SEPARATOR + fileName); - try { - if (!nfs3File.exists()) { - nfs3File.mkdirs(); - } - nfs3File.createNewFile(); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "创建NFS文件失败: ", e); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - return false; - } - - /** - * 删除NFS目录 或者文件 - * - * @param dirOrFile 需要删除的目录 或者文件 例如：/abc/def 或者 /abc/def/dd.txt - * @return boolean - */ - public boolean deleteDirOrFile(String dirOrFile) { - if (!StringUtils.isEmpty(dirOrFile)) { - dirOrFile = formatPath(dirOrFile); - try { - List nfs3FileList = getNfs3File(dirOrFile).listFiles(); - //删除目录下的子文件 - if (!CollectionUtils.isEmpty(nfs3FileList)) { - for (Nfs3File nfs3File : nfs3FileList) { - if (nfs3File.isDirectory()) { - deleteDirOrFile(nfs3File.getPath()); - } else if (nfs3File.isFile()) { - try { - nfs3File.delete(); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - } - } - Nfs3File sourceNfsFile = getNfs3File(dirOrFile); - try { - sourceNfsFile.delete(); - } finally { - nfsPool.revertNfs(sourceNfsFile.getNfs()); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.NFS_UTIL, "删除NFS目录失败:", e); - } - } - return false; - } - - - /** - * 上传文件到 NFS 指定目录 - * - * @param sourceFile 本地文件 包含路径 例如：/abc/def/gg.txt - * @param targetDir 指定目录 例如：/abc/def - * @return boolean - */ - public boolean uploadFileToNfs(String sourceFile, String targetDir) { - if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetDir)) { - return false; - } - sourceFile = formatPath(sourceFile); - targetDir = formatPath(targetDir); - //本地文件对象 - File localFile = new File(sourceFile); - Nfs3File nfs3File = getNfs3File(targetDir + FILE_SEPARATOR + localFile.getName()); - try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localFile)); - BufferedOutputStream outputStream = new BufferedOutputStream(new NfsFileOutputStream(nfs3File))) { - IOUtils.copyLarge(inputStream, outputStream); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "上传失败: ", e); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - return false; - } - - /** - * 下载NFS文件到本地目录 - * - * @param sourceFile 指定文件 例如：/abc/def/dd.txt - * @param targetPath 目标目录 例如: /abc/dd - * @return boolean - */ - public boolean downFileFormNfs(String sourceFile, String targetPath) { - if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourceFile = formatPath(sourceFile); - targetPath = formatPath(targetPath); - Nfs3File nfsFile = getNfs3File(sourceFile); - if (nfsFile != null) { - try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(nfsFile)); - OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(new File(targetPath + FILE_SEPARATOR + nfsFile.getName())))) { - IOUtils.copyLarge(inputStream, outputStream); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "下载失败:", e); - } finally { - nfsPool.revertNfs(nfsFile.getNfs()); - } - } - return false; - } - - - /** - * NFS 复制文件 到指定目录下 单个文件 - * - * @param sourceFile 需要复制的文件 例如：/abc/def/dd.txt - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return boolean - */ - public boolean copyFile(String sourceFile, String targetPath) { - if (StringUtils.isEmpty(sourceFile) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourceFile = formatPath(sourceFile); - targetPath = formatPath(targetPath); - Nfs3File sourceNfsFile = null; - Nfs3File targetNfsFileNew = null; - try { - sourceNfsFile = getNfs3File(sourceFile); - targetNfsFileNew = getNfs3File(targetPath + FILE_SEPARATOR + sourceNfsFile.getName()); - if (!targetNfsFileNew.exists()) { - createDir(targetPath); - } - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "创建目标文件失败： ", e); - } - try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)); - OutputStream outputStream = new BufferedOutputStream(new NfsFileOutputStream(targetNfsFileNew))) { - targetNfsFileNew.createNewFile(); - IOUtils.copyLarge(inputStream, outputStream); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "复制失败:", e); - return false; - } finally { - nfsPool.revertNfs(sourceNfsFile.getNfs()); - nfsPool.revertNfs(targetNfsFileNew.getNfs()); - } - } - - - /** - * NFS 复制目录到指定目录下 多个文件 包含目录与文件并存情况 - * - * 通过NFS文件复制方式 可能存在NFS RPC协议超时情况 - * - * @param sourcePath 需要复制的文件目录 例如：/abc/def - * @param targetPath 需要放置的目标目录 例如：/abc/dd - * @return boolean - */ - public boolean copyNfsPath(String sourcePath, String targetPath) { - if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourcePath = formatPath(sourcePath); - targetPath = formatPath(targetPath); - try { - Nfs3File sourceNfsFile = getNfs3File(sourcePath); - List nfs3FileList = sourceNfsFile.listFiles(); - if (CollectionUtils.isEmpty(nfs3FileList)) { - createDir(targetPath + sourcePath.substring(sourcePath.lastIndexOf(FILE_SEPARATOR))); - } else { - for (Nfs3File nfs3File : nfs3FileList) { - if (nfs3File.isDirectory()) { - String newTargetPath = nfs3File.getPath().substring(nfs3File.getPath().lastIndexOf(FILE_SEPARATOR)); - Nfs3File newNfs3File = getNfs3File(newTargetPath); - try { - if (!newNfs3File.exists()) { - createDir(targetPath + newTargetPath); - } - copyNfsPath(nfs3File.getPath(), targetPath + newTargetPath); - } finally { - nfsPool.revertNfs(newNfs3File.getNfs()); - } - } - if (nfs3File.isFile()) { - copyFile(sourcePath + FILE_SEPARATOR + nfs3File.getName(), targetPath); - } - } - } - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "复制失败: ", e); - return false; - } - } - - /** - * zip解压并删除压缩文件 - * 当压缩包文件较多时，可能会因为RPC超时而解压失败 - * 该方法已废弃，请使用org.dubhe.utils.LocalFileUtil类的unzipLocalPath方法来替代 - * @param sourcePath zip源文件 例如：/abc/z.zip - * @param targetPath 解压后的目标文件夹 例如：/abc/ - * @return boolean - */ - @Deprecated - public boolean unzip(String sourcePath, String targetPath) { - if (StringUtils.isEmpty(sourcePath) || StringUtils.isEmpty(targetPath)) { - return false; - } - sourcePath = formatPath(sourcePath); - targetPath = formatPath(targetPath); - if (!sourcePath.toLowerCase().endsWith(ZIP)) { - return false; - } - ArchiveInputStream zIn = null; - Nfs3File sourceNfsFile = getNfs3File(sourcePath); - try { - zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_GBK, false, true); - //判断压缩文件编码方式,并重新获取NFS对象流 - try { - zIn.getNextEntry(); - zIn.close(); - zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_GBK, false, true); - } catch (Exception e) { - zIn.close(); - zIn = new ZipArchiveInputStream(new BufferedInputStream(new NfsFileInputStream(sourceNfsFile)), CHARACTER_UTF_8, false, true); - } - ZipEntry entry; - while ((entry = (ZipEntry) zIn.getNextEntry()) != null) { - if (entry.isDirectory()) { - createDir(targetPath + FILE_SEPARATOR + entry.getName()); - } else { - //若文件夹未创建则创建文件夹 - if (entry.getName().contains(FILE_SEPARATOR)) { - String entryName = entry.getName(); - String zipDirName = entryName.substring(MagicNumConstant.ZERO, entryName.lastIndexOf(FILE_SEPARATOR)); - createDir(targetPath + FILE_SEPARATOR + zipDirName); - } - Nfs3File nfs3File = getNfs3File(targetPath + FILE_SEPARATOR + entry.getName()); - try { - if (!nfs3File.exists()) { - nfs3File.createNewFile(); - } - BufferedOutputStream bos = new BufferedOutputStream(new NfsFileOutputStream(nfs3File)); - IOUtils.copyLarge(zIn, bos); - bos.flush(); - bos.close(); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - } - sourceNfsFile.delete(); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "解压失败: ", e); - return false; - } finally { - nfsPool.revertNfs(sourceNfsFile.getNfs()); - if (zIn != null) { - try { - zIn.close(); - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "输入流关闭失败: ", e); - } - } - } - - } - - /** - * NFS 解压压缩包 包含目录与子目录 - * - * @param sourcePath 需要复制的文件 例如：/abc/def/aaa.rar - * @return boolean - */ - public boolean unZip(String sourcePath) { - sourcePath = formatPath(sourcePath); - if (StringUtils.isEmpty(sourcePath)) { - return false; - } - String fileDir = sourcePath.substring(MagicNumConstant.ZERO, sourcePath.lastIndexOf(FILE_SEPARATOR)); - ZipEntry zipEntry = null; - try (ZipInputStream zipInputStream = new ZipInputStream(new NfsFileInputStream(getNfs3File(sourcePath)))) { - while ((zipEntry = zipInputStream.getNextEntry()) != null) { - if (zipEntry.isDirectory()) { - createDir(fileDir + FILE_SEPARATOR + zipEntry.getName()); - continue; - } - Nfs3File targetNfsFileNew = getNfs3File(fileDir + FILE_SEPARATOR + zipEntry.getName()); - try (BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(new NfsFileOutputStream(targetNfsFileNew))) { - targetNfsFileNew.createNewFile(); - IOUtils.copyLarge(zipInputStream, bufferedOutputStream); - } finally { - nfsPool.revertNfs(targetNfsFileNew.getNfs()); - } - } - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "解压文件失败 : ", e); - return false; - } - } - - /** - * 压缩NFS 目录 或者文件 到压缩包 - * - * @param dirOrFile 目录或者文件 例如： /abc/def/aaa.txt , /abc/def - * @param zipName 压缩包名称 例如： aa,bb,cc - * @return boolean - */ - public boolean zipDirOrFile(String dirOrFile, String zipName) { - Nfs3File nfs3File = getNfs3File(formatPath(dirOrFile)); - try (ZipOutputStream zipOutputStream = getFileZipOutputStream(getNfsFilePath(formatPath(dirOrFile)), zipName)) { - zipFiles(zipOutputStream, nfs3File); - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "压缩文件失败 : ", e); - return false; - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - - /** - * 获取NFS 文件压缩目录 - * - * @param dirOrFile - * @return String - */ - private String getNfsFilePath(String dirOrFile) throws IOException { - Nfs3File nfs3File = getNfs3File(formatPath(dirOrFile)); - if (nfs3File.isFile()) { - nfsPool.revertNfs(nfs3File.getNfs()); - return dirOrFile.substring(MagicNumConstant.ZERO, dirOrFile.lastIndexOf(FILE_SEPARATOR)); - } - return dirOrFile; - } - - /** - * 根据文件路劲 获取Zip文件流 - * - * @param dirOrFile - * @param zipName - * @return ZipOutputStream - */ - private ZipOutputStream getFileZipOutputStream(String dirOrFile, String zipName) throws IOException { - Nfs3File targetNfsFileNew = getNfs3File(getNfsFilePath(formatPath(dirOrFile)) + FILE_SEPARATOR + zipName + ZIP); - targetNfsFileNew.createNewFile(); - return new ZipOutputStream(new NfsFileOutputStream(targetNfsFileNew)); - } - - /** - * 压缩文件和文件夹 - * - * @param zipOutputStream - * @param nfs3File - * @return boolean - */ - public boolean zipFiles(ZipOutputStream zipOutputStream, Nfs3File nfs3File) { - try { - if (nfs3File.isFile()) { - compressZip(zipOutputStream, nfs3File, ""); - } else { - List nfs3FileList = nfs3File.listFiles(); - if (!CollectionUtils.isEmpty(nfs3FileList)) { - for (Nfs3File nfs3FileChildren : nfs3FileList) { - zipFiles(zipOutputStream, nfs3FileChildren); - } - } - } - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "压缩文件失败 : ", e); - return false; - } - } - - /** - * 单个文件压缩 - * - * @param zipOutputStream - * @param nfs3File - */ - public void compressZip(ZipOutputStream zipOutputStream, Nfs3File nfs3File, String childPath) { - try (InputStream inputStream = new BufferedInputStream(new NfsFileInputStream(nfs3File))) { - if (StringUtils.isEmpty(childPath)) { - zipOutputStream.putNextEntry(new ZipEntry(nfs3File.getName())); - } else { - zipOutputStream.putNextEntry(new ZipEntry(childPath + FILE_SEPARATOR + nfs3File.getName())); - } - byte[] buffer = new byte[1024 * 10]; - int length; - while ((length = inputStream.read(buffer, MagicNumConstant.ZERO, buffer.length)) != -1) { - zipOutputStream.write(buffer, MagicNumConstant.ZERO, length); - } - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "解压单个文件异常： ", e); - } finally { - nfsPool.revertNfs(nfs3File.getNfs()); - } - } - - /** - * 复制文件夹(或文件)到另一个文件夹 - * - * @param sourcePath 复制文件夹 /abc/def 复制文件：/abc/def/dd.txt - * @param targetPath /abc/dd/def - * @return boolean - */ - public boolean copyDirs(String sourcePath, String targetPath) { - Nfs3File sourceNfsFile = getNfs3File(formatPath(sourcePath)); - try { - if (!sourceNfsFile.exists()) { - LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", sourcePath); - return false; - } - if (sourceNfsFile.isFile()) { - return copyFile(sourcePath, targetPath); - } else if (sourceNfsFile.isDirectory()) { - targetPath = targetPath + FILE_SEPARATOR + sourceNfsFile.getName(); - boolean bool = createDir(formatPath(targetPath)); - if (!bool) { - LogUtil.error(LogEnum.NFS_UTIL, "{}文件夹创建失败... ", targetPath); - return false; - } - List files = sourceNfsFile.listFiles(); - for (Nfs3File file : files) { - copyDirs(file.getPath(), targetPath); - } - } - return true; - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "copyDirs失败, sourcePath为 , targetPath为 , 失败原因 ", sourcePath, targetPath, e); - } finally { - nfsPool.revertNfs(sourceNfsFile.getNfs()); - } - return false; - } - - /** - * 找到倒数第二新的文件夹 - * - * @param parentPath 父文件夹 - * @return - */ - public String find2ndNewDir(String parentPath) { - Nfs3File parentNfsFile = getNfs3File(formatPath(parentPath)); - try { - if (!parentNfsFile.exists() || parentNfsFile.isFile()) { - LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", parentPath); - return ""; - } - List files = parentNfsFile.listFiles(); - List dirs = new ArrayList<>(); - for (Nfs3File file : files) { - if (file.isDirectory()) { - dirs.add(file); - } - } - if (dirs.size() < MagicNumConstant.TWO) { - return ""; - } - dirs.sort((o1, o2) -> { - try { - return (int) (o2.lastModified() - o1.lastModified()); - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "执行异常: {} ", e); - return MagicNumConstant.ZERO; - } - }); - return dirs.get(MagicNumConstant.ONE).getName(); - - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "findSecNewDir失败, parentPath为{}, 失败原因{}", parentPath, e); - } finally { - nfsPool.revertNfs(parentNfsFile.getNfs()); - } - return ""; - } - - /** - * 重命名文件夹 - * - * @param sourcePath 原文件夹 - * @param targetPath 目标文件夹 - * @return - */ - public void renameDir(String sourcePath, String targetPath) { - Nfs3File sourceNfsFile = getNfs3File(formatPath(sourcePath)); - Nfs3File targetNfsFile = getNfs3File(formatPath(targetPath)); - try { - if (!sourceNfsFile.exists()) { - LogUtil.error(LogEnum.NFS_UTIL, "sourcePath不存在, 如下{} ", sourcePath); - } - sourceNfsFile.rename(targetNfsFile); - } catch (IOException e) { - LogUtil.error(LogEnum.NFS_UTIL, "renameDir失败, sourcePath为{}, targetPath为{}, 失败原因{}", sourcePath, targetPath, e); - } finally { - nfsPool.revertNfs(sourceNfsFile.getNfs()); - nfsPool.revertNfs(targetNfsFile.getNfs()); - } - } - - - /** - * 替换路劲中多余的 "/" - * - * @param path - * @return String - */ - public String formatPath(String path) { - if (!StringUtils.isEmpty(path)) { - return path.replaceAll("///*", FILE_SEPARATOR); - } - return path; - } - - public String getAbsolutePath(String relativePath) { - return nfsConfig.getRootDir() + nfsConfig.getBucket() + relativePath; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java deleted file mode 100644 index 00365d6..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/NumberUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.dubhe.exception.NotebookBizException; - -import java.util.regex.Pattern; - -/** - * @description 数字验证工具 - * @date 2020-05-18 - */ -public class NumberUtil { - - private static final String REGEX = "^[0-9]*$"; - - private NumberUtil() { - - } - - /** - * 判断是否为数字格式不限制位数 - * - * @param object 待校验参数 - */ - public static void isNumber(Object object) { - if (!((Pattern.compile(REGEX)).matcher(String.valueOf(object)).matches())) { - throw new NotebookBizException("parameter is incorrect"); - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java deleted file mode 100644 index 7a2cb7c..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/PageUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.baomidou.mybatisplus.core.metadata.IPage; - -import java.util.*; -import java.util.function.Function; - -/** - * @description 分页工具 - * @date 2020-03-13 - */ -public class PageUtil extends cn.hutool.core.util.PageUtil { - - /** - * List 分页 - */ - public static List toPage(int page, int size, List list) { - int fromIndex = page * size; - int toIndex = page * size + size; - if (fromIndex > list.size()) { - return new ArrayList(); - } else if (toIndex >= list.size()) { - return list.subList(fromIndex, list.size()); - } else { - return list.subList(fromIndex, toIndex); - } - } - - /** - * Page 数据处理，预防redis反序列化报错 - */ - public static Map toPage(IPage page) { - return toPage(page, page.getRecords()); - } - - /** - * 自定义分页 - */ - public static Map toPage(IPage page, Function function) { - return toPage(page, function.apply(page.getRecords())); - } - - /** - * 自定义分页 - */ - public static Map toPage(IPage page, Collection data) { - Map map = new LinkedHashMap<>(2); - map.put("result", data); - map.put("page", buildPagination(page)); - return map; - } - - private static Map buildPagination(IPage page) { - Map map = new HashMap<>(2); - map.put("current", page.getCurrent()); - map.put("size", page.getSize()); - map.put("total", page.getTotal()); - return map; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java deleted file mode 100644 index fe29ab3..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ReadFile.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * @description 读文件工具类 - * @date 2020-5-6 - */ -public class ReadFile { - - public static List read(String path, Integer start, Integer length){ - if (start < 0) { - start = 0; - } - List ansList = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(path))){ - for (int i = 0; i < start+length; i++) { - String line = br.readLine(); - if (line == null) { - break; - } - if (start <= i && i < start+length) { - ansList.add(line); - } - } - } catch (IOException ex) { - ex.printStackTrace(); - return ansList; - } - return ansList; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java deleted file mode 100644 index d7f59d5..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/RedisUtils.java +++ /dev/null @@ -1,847 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.dubhe.base.MagicNumConstant; -import org.dubhe.enums.LogEnum; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.redis.connection.RedisConnection; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.Cursor; -import org.springframework.data.redis.core.RedisConnectionUtils; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.core.ScanOptions; -import org.springframework.data.redis.core.script.DefaultRedisScript; -import org.springframework.data.redis.core.script.RedisScript; -import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.*; -import java.util.concurrent.TimeUnit; - -/** - * @description redis工具类 - * @date 2020-03-13 - */ -@Component -@SuppressWarnings({"unchecked", "all"}) -public class RedisUtils { - - private RedisTemplate redisTemplate; - @Value("${jwt.online-key}") - private String onlineKey; - - public RedisUtils(RedisTemplate redisTemplate) { - this.redisTemplate = redisTemplate; - } - - // =============================common============================ - - /** - * 指定缓存失效时间 - * - * @param key 键 - * @param time 时间(秒) - */ - public boolean expire(String key, long time) { - try { - if (time > 0) { - redisTemplate.expire(key, time, TimeUnit.SECONDS); - } - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils expire key {} time {} error:{}", key, time, e.getMessage(), e); - return false; - } - return true; - } - - /** - * 根据 key 获取过期时间 - * - * @param key 键 不能为null - * @return 时间(秒) 返回0代表为永久有效 - */ - public long getExpire(Object key) { - return redisTemplate.getExpire(key, TimeUnit.SECONDS); - } - - /** - * 查找匹配key - * - * @param pattern key - * @return List 匹配的key集合 - */ - public List scan(String pattern) { - ScanOptions options = ScanOptions.scanOptions().match(pattern).build(); - RedisConnectionFactory factory = redisTemplate.getConnectionFactory(); - RedisConnection rc = Objects.requireNonNull(factory).getConnection(); - Cursor cursor = rc.scan(options); - List result = new ArrayList<>(); - while (cursor.hasNext()) { - result.add(new String(cursor.next())); - } - try { - RedisConnectionUtils.releaseConnection(rc, factory, true); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils scan pattern {} error:{}", pattern, e.getMessage(), e); - } - return result; - } - - /** - * 分页查询 key - * - * @param patternKey key - * @param page 页码 - * @param size 每页数目 - * @return 匹配到的key集合 - */ - public List findKeysForPage(String patternKey, int page, int size) { - ScanOptions options = ScanOptions.scanOptions().match(patternKey).build(); - RedisConnectionFactory factory = redisTemplate.getConnectionFactory(); - RedisConnection rc = Objects.requireNonNull(factory).getConnection(); - Cursor cursor = rc.scan(options); - List result = new ArrayList<>(size); - int tmpIndex = 0; - int fromIndex = page * size; - int toIndex = page * size + size; - while (cursor.hasNext()) { - if (tmpIndex >= fromIndex && tmpIndex < toIndex) { - result.add(new String(cursor.next())); - tmpIndex++; - continue; - } - // 获取到满足条件的数据后,就可以退出了 - if (tmpIndex >= toIndex) { - break; - } - tmpIndex++; - cursor.next(); - } - try { - RedisConnectionUtils.releaseConnection(rc, factory, true); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils findKeysForPage patternKey {} page {} size {} error:{}", patternKey, page, size, e.getMessage(), e); - } - return result; - } - - /** - * 判断key是否存在 - * - * @param key 键 - * @return true 存在 false不存在 - */ - public boolean hasKey(String key) { - try { - return redisTemplate.hasKey(key); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils hasKey key {} error:{}", key, e.getMessage(), e); - return false; - } - } - - /** - * 删除缓存 - * - * @param key 可以传一个值 或多个 - */ - public void del(String... key) { - if (key != null && key.length > 0) { - if (key.length == 1) { - redisTemplate.delete(key[0]); - } else { - redisTemplate.delete(CollectionUtils.arrayToList(key)); - } - } - } - - /** - * - * @param script 脚本字符串 - * @param key 键 - * @param args 脚本其他参数 - * @return - */ - public Object executeRedisScript(String script, String key, Object... args) { - try { - RedisScript redisScript = new DefaultRedisScript<>(script, Long.class); - redisTemplate.setValueSerializer(new Jackson2JsonRedisSerializer<>(Object.class)); - return redisTemplate.execute(redisScript, Collections.singletonList(key), args); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "executeRedisScript script {} key {} expireTime {} args {} error:{}", script, key, args, e); - return MagicNumConstant.ZERO_LONG; - } - } - - /** - * - * @param script 脚本字符串 - * @param key 键 - * @param args 脚本其他参数 - * @return - */ - public Object executeRedisObjectScript(String script, String key, Object... args) { - try { - RedisScript redisScript = new DefaultRedisScript<>(script, Object.class); - redisTemplate.setValueSerializer(new Jackson2JsonRedisSerializer<>(Object.class)); - return redisTemplate.execute(redisScript, Collections.singletonList(key), args); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "executeRedisObjectScript script {} key {} expireTime {} args {} error:{}", script, key, args, e); - return null; - } - } - - // ============================String============================= - - /** - * 普通缓存获取 - * - * @param key 键 - * @return key对应的value值 - */ - public Object get(String key) { - - return key == null ? null : redisTemplate.opsForValue().get(key); - } - - /** - * 批量获取 - * - * @param keys key集合 - * @return key集合对应的value集合 - */ - public List multiGet(List keys) { - Object obj = redisTemplate.opsForValue().multiGet(Collections.singleton(keys)); - return null; - } - - /** - * 普通缓存放入 - * - * @param key 键 - * @param value 值 - * @return true成功 false失败 - */ - public boolean set(String key, Object value) { - try { - redisTemplate.opsForValue().set(key, value); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils set key {} value {} error:{}", key, value, e.getMessage(), e); - return false; - } - } - - /** - * 普通缓存放入并设置时间 - * - * @param key 键 - * @param value 值 - * @param time 时间(秒) time要大于0 如果time小于等于0 将设置无限期 - * @return true成功 false 失败 - */ - public boolean set(String key, Object value, long time) { - try { - if (time > 0) { - redisTemplate.opsForValue().set(key, value, time, TimeUnit.SECONDS); - } else { - set(key, value); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils set key {} value {} time {} error:{}", key, value, time, e.getMessage(), e); - return false; - } - } - - /** - * 普通缓存放入并设置时间 - * - * @param key 键 - * @param value 值 - * @param time 时间 - * @param timeUnit 类型 - * @return true成功 false 失败 - */ - public boolean set(String key, Object value, long time, TimeUnit timeUnit) { - try { - if (time > 0) { - redisTemplate.opsForValue().set(key, value, time, timeUnit); - } else { - set(key, value); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils set key {} value {} time {} timeUnit {} error:{}", key, value, time, timeUnit, e.getMessage(), e); - return false; - } - } - - //===============================Lock================================= - - /** - * 加锁 - * @param key 键 - * @param requestId 请求id用以释放锁 - * @param expireTime 超时时间（秒） - * @return - */ - public boolean getDistributedLock(String key, String requestId, long expireTime) { - String script = "if redis.call('setNx',KEYS[1],ARGV[1]) == 1 then if redis.call('get',KEYS[1]) == ARGV[1] then return redis.call('expire',KEYS[1],ARGV[2]) else return 0 end else return 0 end"; - Object result = executeRedisScript(script, key, requestId, expireTime); - return result != null && result.equals(MagicNumConstant.ONE_LONG); - } - - /** - * 释放锁 - * @param key 键 - * @param requestId 请求id用以释放锁 - * @return - */ - public boolean releaseDistributedLock(String key, String requestId) { - String script = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; - Object result = executeRedisScript(script, key, requestId); - return result != null && result.equals(MagicNumConstant.ONE_LONG); - } - - // ================================Map================================= - - /** - * HashGet - * - * @param key 键 不能为null - * @param item 项 不能为null - * @return 值 - */ - public Object hget(String key, String item) { - return redisTemplate.opsForHash().get(key, item); - } - - /** - * 获取hashKey对应的所有键值 - * - * @param key 键 - * @return 对应的多个键值 - */ - public Map hmget(String key) { - return redisTemplate.opsForHash().entries(key); - - } - - /** - * HashSet - * - * @param key 键 - * @param map 对应多个键值 - * @return true 成功 false 失败 - */ - public boolean hmset(String key, Map map) { - try { - redisTemplate.opsForHash().putAll(key, map); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils hmset key {} map {} error:{}", key, map, e.getMessage(), e); - return false; - } - } - - /** - * HashSet 并设置时间 - * - * @param key 键 - * @param map 对应多个键值 - * @param time 时间(秒) - * @return true成功 false失败 - */ - public boolean hmset(String key, Map map, long time) { - try { - redisTemplate.opsForHash().putAll(key, map); - if (time > 0) { - expire(key, time); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils hmset key {} map {} time {} error:{}", key, map, time, e.getMessage(), e); - return false; - } - } - - /** - * 向一张hash表中放入数据,如果不存在将创建 - * - * @param key 键 - * @param item 项 - * @param value 值 - * @return true 成功 false失败 - */ - public boolean hset(String key, String item, Object value) { - try { - redisTemplate.opsForHash().put(key, item, value); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils hset key {} item {} value {} error:{}", key, item, value, e.getMessage(), e); - return false; - } - } - - /** - * 向一张hash表中放入数据,如果不存在将创建 - * - * @param key 键 - * @param item 项 - * @param value 值 - * @param time 时间(秒) 注意:如果已存在的hash表有时间,这里将会替换原有的时间 - * @return true 成功 false失败 - */ - public boolean hset(String key, String item, Object value, long time) { - try { - redisTemplate.opsForHash().put(key, item, value); - if (time > 0) { - expire(key, time); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils hset key {} item {} value {} time {} error:{}", key, item, value, time, e.getMessage(), e); - return false; - } - } - - /** - * 删除hash表中的值 - * - * @param key 键 不能为null - * @param item 项 可以使多个 不能为null - */ - public void hdel(String key, Object... item) { - redisTemplate.opsForHash().delete(key, item); - } - - /** - * 判断hash表中是否有该项的值 - * - * @param key 键 不能为null - * @param item 项 不能为null - * @return true 存在 false不存在 - */ - public boolean hHasKey(String key, String item) { - return redisTemplate.opsForHash().hasKey(key, item); - } - - /** - * hash递增 如果不存在,就会创建一个 并把新增后的值返回 - * - * @param key 键 - * @param item 项 - * @param by 要增加几(大于0) - * @return - */ - public double hincr(String key, String item, double by) { - return redisTemplate.opsForHash().increment(key, item, by); - } - - /** - * hash递减 - * - * @param key 键 - * @param item 项 - * @param by 要减少记(小于0) - * @return - */ - public double hdecr(String key, String item, double by) { - return redisTemplate.opsForHash().increment(key, item, -by); - } - - // ============================set============================= - - /** - * 根据key获取Set中的所有值 - * - * @param key 键 - * @return - */ - public Set sGet(String key) { - try { - return redisTemplate.opsForSet().members(key); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils sGet key {} error:{}", key, e.getMessage(), e); - return null; - } - } - - /** - * 根据value从一个set中查询,是否存在 - * - * @param key 键 - * @param value 值 - * @return true 存在 false不存在 - */ - public boolean sHasKey(String key, Object value) { - try { - return redisTemplate.opsForSet().isMember(key, value); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils sHasKey key {} value {} error:{}", key, value, e.getMessage(), e); - return false; - } - } - - /** - * 将数据放入set缓存 - * - * @param key 键 - * @param values 值 可以是多个 - * @return 成功个数 - */ - public long sSet(String key, Object... values) { - try { - return redisTemplate.opsForSet().add(key, values); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils sSet key {} values {} error:{}", key, values, e.getMessage(), e); - return 0; - } - } - - /** - * 将set数据放入缓存 - * - * @param key 键 - * @param time 时间(秒) - * @param values 值 可以是多个 - * @return 成功个数 - */ - public long sSetAndTime(String key, long time, Object... values) { - try { - Long count = redisTemplate.opsForSet().add(key, values); - if (time > 0) { - expire(key, time); - } - return count; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils sSetAndTime key {} time {} values {} error:{}", key, time, values, e.getMessage(), e); - return 0; - } - } - - /** - * 获取set缓存的长度 - * - * @param key 键 - * @return - */ - public long sGetSetSize(String key) { - try { - return redisTemplate.opsForSet().size(key); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils sGetSetSize key {} error:{}", key, e.getMessage(), e); - return 0; - } - } - - /** - * 移除值为value的 - * - * @param key 键 - * @param values 值 可以是多个 - * @return 移除的个数 - */ - public long setRemove(String key, Object... values) { - try { - Long count = redisTemplate.opsForSet().remove(key, values); - return count; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils setRemove key {} values {} error:{}", key, values, e.getMessage(), e); - return 0; - } - } - - // ===============================sorted set================================= - - /** - * 将zSet数据放入缓存 - * - * @param key - * @param time - * @param values - * @return Boolean - */ - public Boolean zSet(String key, long time, Object value) { - try { - Boolean success = redisTemplate.opsForZSet().add(key, value, System.currentTimeMillis()); - if (success) { - expire(key, time); - } - return success; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zSet key {} time {} value {} error:{}", key, time, value, e.getMessage(), e); - return false; - } - } - - /** - * 将zSet数据放入缓存 - * @param key 健 - * @param score 分数 - * @param value 值 - * @return - */ - public Boolean zAdd(String key,Long score,Object value){ - try{ - if (StringUtils.isEmpty(key) || score == null || value == null){ - return false; - } - return redisTemplate.opsForZSet().add(key, value, score); - }catch (Exception e){ - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zAdd key {} score {} value {} error:{}", key, score, value, e.getMessage(), e); - return false; - } - } - - /** - * 返回有序集合所有成员，从大到小排序 - * - * @param key - * @return Set - */ - public Set zGet(String key) { - try { - return redisTemplate.opsForZSet().reverseRange(key, Long.MIN_VALUE, Long.MAX_VALUE); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zGet key {} error:{}", key, e.getMessage(), e); - return null; - } - } - - /** - * 弹出有序集合 score 在 [min,max] 内由小到大从 offset 取 count 个 - * @param key 健 - * @param min score最小值 - * @param max score最大值 - * @param offset 起始下标 - * @param count 偏移量 - * @return - */ - public List zRangeByScorePop(String key,double min,double max,long offset,long count){ - try{ - String script = "local elementSet = redis.call('ZRANGEBYSCORE',KEYS[1],ARGV[1],ARGV[2],'LIMIT',ARGV[3],ARGV[4]) if elementSet ~= false and #elementSet ~= 0 then redis.call('ZREM' , KEYS[1] , elementSet[1]) end return elementSet"; - Object result = executeRedisObjectScript(script, key, min, max,offset,count); - return (List) result; - }catch (Exception e){ - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils zRangeByScorePop key {} min {} max {} offset {} count {} error:{}", key,min, max, offset, count, e.getMessage(), e); - return new ArrayList<>(); - } - } - - /** - * 弹出有序集合 score 在 [min,max] 内由小到大从 0 取 1 个 - * @param key 健 - * @param min score最小值 - * @param max score最大值 - * @return - */ - public List zRangeByScorePop(String key,double min, double max){ - return zRangeByScorePop( key,min, max,0,1); - } - - /** - * 弹出有序集合 score 在 [0,max] 内由小到大从 offset 取 count 个 - * @param key 健 - * @param max score最大值 - * @return - */ - public List zRangeByScorePop(String key,double max){ - return zRangeByScorePop( key,0, max,0,1); - } - - - // ===============================list================================= - - /** - * 获取list缓存的内容 - * - * @param key 键 - * @param start 开始 - * @param end 结束 0 到 -1代表所有值 - * @return - */ - public List lGet(String key, long start, long end) { - try { - return redisTemplate.opsForList().range(key, start, end); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lGetIndex key {} start {} end {} error:{}", key, start, end, e.getMessage(), e); - return null; - } - } - - /** - * 获取list缓存的长度 - * - * @param key 键 - * @return - */ - public long lGetListSize(String key) { - try { - return redisTemplate.opsForList().size(key); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lGetListSize key {} error:{}", key, e.getMessage(), e); - return 0; - } - } - - /** - * 通过索引 获取list中的值 - * - * @param key 键 - * @param index 索引 index>=0时， 0 表头，1 第二个元素，依次类推；index<0时，-1，表尾，-2倒数第二个元素，依次类推 - * @return - */ - public Object lGetIndex(String key, long index) { - try { - return redisTemplate.opsForList().index(key, index); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lGetIndex key {} index {} error:{}", key, index, e.getMessage(), e); - return null; - } - } - - /** - * 将list放入缓存 - * - * @param key 键 - * @param value 值 - * @return - */ - public boolean lSet(String key, Object value) { - try { - redisTemplate.opsForList().rightPush(key, value); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lSet key {} value {} error:{}", key, value, e.getMessage(), e); - return false; - } - } - - /** - * 将list放入缓存 - * - * @param key 键 - * @param value 值 - * @param time 时间(秒) - * @return 是否存储成功 - */ - public boolean lSet(String key, Object value, long time) { - try { - redisTemplate.opsForList().rightPush(key, value); - if (time > 0) { - expire(key, time); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lSet key {} value {} time {} error:{}", key, value, time, e.getMessage(), e); - return false; - } - } - - /** - * 将list放入缓存 - * - * @param key 键 - * @param value 值 - * @return 是否存储成功 - */ - public boolean lSet(String key, List value) { - try { - redisTemplate.opsForList().rightPushAll(key, value); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lSet key {} value {} error:{}", key, value, e.getMessage(), e); - return false; - } - } - - /** - * 将list放入缓存 - * - * @param key 键 - * @param value 值 - * @param time 时间(秒) - * @return 是否存储成功 - */ - public boolean lSet(String key, List value, long time) { - try { - redisTemplate.opsForList().rightPushAll(key, value); - if (time > 0) { - expire(key, time); - } - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lSet key {} value {} time {} error:{}", key, value, time, e.getMessage(), e); - return false; - } - } - - /** - * 根据索引修改list中的某条数据 - * - * @param key 键 - * @param index 索引 - * @param value 值 - * @return 更新数据标识 - */ - public boolean lUpdateIndex(String key, long index, Object value) { - try { - redisTemplate.opsForList().set(key, index, value); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lUpdateIndex key {} index {} value {} error:{}", key, index, value, e.getMessage(), e); - return false; - } - } - - /** - * 移除N个值为value - * - * @param key 键 - * @param count 移除多少个 - * @param value 值 - * @return 移除的个数 - */ - public long lRemove(String key, long count, Object value) { - try { - return redisTemplate.opsForList().remove(key, count, value); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lRemove key {} count {} value {} error:{}", key, count, value, e.getMessage(), e); - return 0; - } - } - - /** - * 队列从左弹出数据 - * - * @param key - * @return key对应的value值 - */ - public Object lpop(String key) { - try { - return redisTemplate.opsForList().leftPop(key); - } catch (Exception e) { - LogUtil.error(LogEnum.SYS_ERR, "RedisUtils lRemove key {} error:{}", key, e.getMessage(), e); - return null; - } - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java deleted file mode 100644 index 3f0aeae..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ReflectionUtils.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - -/** - * @description 反射工具类 - * @date 2020-05-29 - **/ -public class ReflectionUtils { - - /** - * 获取所有字段集合 - * - * @param clazz 反射对象 - * @return List 字段集合 - **/ - public static List getFieldNames(Class clazz) { - Field[] fields = clazz.getDeclaredFields(); - List fieldNameList = new ArrayList<>(); - for (Field field : fields) { - fieldNameList.add(field.getName()); - } - return fieldNameList; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java deleted file mode 100644 index a85034e..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/RegexUtil.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @description 正则匹配工具类 - * @date 2020-04-23 - */ -@Slf4j -public class RegexUtil { - private static final String DIGIT = "^[0-9]*$"; - private static final String FLOAT = "^[-+]?[0-9]*\\.?[0-9]+$"; - - /** - * 字符串匹配 - * - * @param str 待匹配文本 - * @param regex 正则表达式 - * @return 返回str中匹配regex的第一个子串 - */ - public static String getMatcher(String str, String regex) { - try { - if (StringUtils.isEmpty(str) || StringUtils.isEmpty(regex)) { - return ""; - } - Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); - Matcher matcher = p.matcher(str); - matcher.find(); - return matcher.group(); - } catch (IllegalStateException e) { - log.error(e.getMessage(), e); - return ""; - } - } - - /** - * 数字匹配 - * - * @param str - * @return - */ - public static boolean isDigits(String str) { - if (StringUtils.isEmpty(str)) { - return false; - } - return str.matches(DIGIT); - } - - /** - * 浮点数匹配 - * - * @param str - * @return - */ - public static boolean isFloat(String str) { - if (StringUtils.isEmpty(str)) { - return false; - } - return str.matches(FLOAT); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java deleted file mode 100644 index 501f555..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/SpringContextHolder.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeansException; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -/** - * @description 上下文工具类 - * @date 2020-03-25 - */ -@Slf4j -public class SpringContextHolder implements ApplicationContextAware, DisposableBean { - - private static ApplicationContext applicationContext = null; - - /** - * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. - */ - @SuppressWarnings("unchecked") - public static T getBean(String name) { - assertContextInjected(); - return (T) applicationContext.getBean(name); - } - - /** - * 从静态变量applicationContext中取得Bean, 自动转型为所赋值对象的类型. - */ - public static T getBean(Class requiredType) { - assertContextInjected(); - return applicationContext.getBean(requiredType); - } - - /** - * 检查ApplicationContext不为空. - */ - private static void assertContextInjected() { - if (applicationContext == null) { - throw new IllegalStateException("applicaitonContext属性未注入, 请在applicationContext" + - ".xml中定义SpringContextHolder或在SpringBoot启动类中注册SpringContextHolder."); - } - } - - /** - * 清除SpringContextHolder中的ApplicationContext为Null. - */ - private static void clearHolder() { - log.debug("清除SpringContextHolder中的ApplicationContext:" - + applicationContext); - applicationContext = null; - } - - @Override - public void destroy() { - SpringContextHolder.clearHolder(); - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - if (SpringContextHolder.applicationContext != null) { - log.warn("SpringContextHolder中的ApplicationContext被覆盖, 原有ApplicationContext为:" + SpringContextHolder.applicationContext); - } - SpringContextHolder.applicationContext = applicationContext; - } - - /** - * 获取当前环境 - * - * @return - */ - public static String getActiveProfile(){ - return applicationContext.getEnvironment().getActiveProfiles()[0]; - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java deleted file mode 100644 index cbee763..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/SqlUtil.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.dubhe.base.BaseService; -import org.dubhe.base.DataContext; - -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -/** - * @description sql语句转换的工具类 - * @date 2020-07-06 - */ - -public class SqlUtil { - - /** - * 将数组转换成in('1','2')的形式 - * - * @param objs - * @return - */ - public static String integerlistToString(Integer[] objs) { - - if (objs != null && objs.length > 0) { - StringBuilder sb = new StringBuilder("("); - for (Object obj : objs) { - if (obj != null) { - sb.append("'" + obj.toString() + "',"); - } - } - sb.deleteCharAt(sb.length() - 1); - sb.append(")"); - return sb.toString(); - } - return ""; - } - - - /** - * 获取资源拥有着ID - * - * @return 资源拥有者id集合 - */ - public static Set getResourceIds() { - if (!Objects.isNull(DataContext.get())) { - return DataContext.get().getResourceUserIds(); - } - Set ids = new HashSet<>(); - Long id = JwtUtils.getCurrentUserDto().getId(); - ids.add(id); - return ids; - - } - - - /** - * 构建目标sql语句 - * - * @param originSql 原生sql - * @param resourceUserIds 所属资源用户ids - * @return 目标sql - */ - public static String buildTargetSql(String originSql, Set resourceUserIds) { - if (BaseService.isAdmin()) { - return originSql; - } - String sqlWhereBefore = org.dubhe.utils.StringUtils.substringBefore(originSql.toLowerCase(), "where"); - String sqlWhereAfter = org.dubhe.utils.StringUtils.substringAfter(originSql.toLowerCase(), "where"); - StringBuffer buffer = new StringBuffer(); - //操作的sql拼接 - String targetSql = buffer.append(sqlWhereBefore).append(" where ").append(" origin_user_id in (") - .append(org.dubhe.utils.StringUtils.join(resourceUserIds, ",")).append(") and ").append(sqlWhereAfter).toString(); - - return targetSql; - } - - - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java deleted file mode 100644 index cd04585..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/StateMachineProxyUtil.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.utils; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.apache.poi.ss.formula.functions.T; -import org.dubhe.dto.StateChangeDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.StateMachineException; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; -import org.springframework.util.ReflectionUtils; - -/** - * @description 代理执行状态机 - * @date 2020-08-27 - */ -@Component -public class StateMachineProxyUtil { - - /** - * 代理执行单个状态机的状态切换 - * - * @param stateChangeDTO 数据集状态切换信息 - * @param objectService 服务类 - */ - public static void proxyExecutionSingleState(StateChangeDTO stateChangeDTO,Object objectService) { - checkSingleParam(stateChangeDTO); - //获取全局状态机中的指定状态机 - Field field = ReflectionUtils.findField(objectService.getClass(), stateChangeDTO.getStateMachineType()); - if (field == null) { - throw new StateMachineException("在全局状态机中未找到指定状态机"); - } - //获取需要执行的状态机对象 - Object stateMachineObject = SpringContextHolder.getBean(field.getName()); - try { - //获取目标执行方法的参数类型 - List> paramTypesList = getMethodParamTypes(stateMachineObject, stateChangeDTO.getEventMethodName()); - //构造目标执行方法 - Method method = ReflectionUtils.findMethod(stateMachineObject.getClass(), stateChangeDTO.getEventMethodName(), paramTypesList.toArray(new Class[paramTypesList.size()])); - if (stateChangeDTO.getObjectParam().length != paramTypesList.size()) { - LogUtil.error(LogEnum.STATE_MACHINE, " 目标执行方法参数 {} 与传入的数量不一致 {} ", paramTypesList.size(), stateChangeDTO.getObjectParam().length); - } else { - ReflectionUtils.invokeMethod(method, stateMachineObject, stateChangeDTO.getObjectParam()); - } - } catch (ClassNotFoundException e) { - LogUtil.error(LogEnum.STATE_MACHINE, "未找到指定类： {} ", e); - } - } - - /** - * 代理执行多个状态机的状态切换 - * - * @param stateChangeDTOList 多个状态机切换信息 - * @param objectService 服务类 - */ - public static void proxyExecutionRelationState(List stateChangeDTOList,Object objectService) { - if (!CollectionUtils.isEmpty(stateChangeDTOList)) { - for (StateChangeDTO stateChangeDTO : stateChangeDTOList) { - proxyExecutionSingleState(stateChangeDTO,objectService); - } - } - } - - /** - * 校验参数是否正常 - * - * @param stateChangeDTO 数据集状态切换信息 - */ - public static void checkSingleParam(StateChangeDTO stateChangeDTO) { - if (StringUtils.isEmpty(stateChangeDTO.getStateMachineType())) { - throw new StateMachineException("未指定状态机类"); - } - if (StringUtils.isEmpty(stateChangeDTO.getEventMethodName())) { - throw new StateMachineException("未指定状态机需要执行事件"); - } - } - - /** - * 根据方法名获取所有参数的类型 - * - * @param classInstance 类实例 - * @param methodName 方法名 - * @return List> 对象集合 - * @throws ClassNotFoundException - */ - public static List> getMethodParamTypes(Object classInstance, String methodName) throws ClassNotFoundException { - List> paramTypes = new ArrayList<>(); - Method[] methods = classInstance.getClass().getMethods(); - for (Method method : methods) { - if (method.getName().equals(methodName)) { - Class[] params = method.getParameterTypes(); - for (Class classParamType : params) { - paramTypes.addAll(Collections.singleton((Class) Class.forName(classParamType.getName()))); - } - break; - } - } - return paramTypes; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java b/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java deleted file mode 100644 index 28c8b87..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/StringUtils.java +++ /dev/null @@ -1,348 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import eu.bitwalker.useragentutils.Browser; -import eu.bitwalker.useragentutils.UserAgent; -import org.apache.commons.lang.RandomStringUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.SymbolConstant; - -import javax.servlet.http.HttpServletRequest; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @description 字符串工具类, 继承org.apache.commons.lang3.StringUtils类 - * @date 2020-03-25 - */ -public class StringUtils extends org.apache.commons.lang3.StringUtils { - - private static final char SEPARATOR = '_'; - - private static final String UNKNOWN = "unknown"; - - private static Pattern linePattern = Pattern.compile("_(\\w)"); - - - /** - * 驼峰命名法工具 - * - * @return toCamelCase(" hello_world ") == "helloWorld" - * toCapitalizeCamelCase("hello_world") == "HelloWorld" - * toUnderScoreCase("helloWorld") = "hello_world" - */ - public static String toCamelCase(String s) { - if (s == null) { - return null; - } - - s = s.toLowerCase(); - - StringBuilder sb = new StringBuilder(s.length()); - boolean upperCase = false; - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - - if (c == SEPARATOR) { - upperCase = true; - } else if (upperCase) { - sb.append(Character.toUpperCase(c)); - upperCase = false; - } else { - sb.append(c); - } - } - - return sb.toString(); - } - - /** - * 驼峰命名法工具 - * - * @return toCamelCase(" hello_world ") == "helloWorld" - * toCapitalizeCamelCase("hello_world") == "HelloWorld" - * toUnderScoreCase("helloWorld") = "hello_world" - */ - public static String toCapitalizeCamelCase(String s) { - if (s == null) { - return null; - } - s = toCamelCase(s); - return s.substring(0, 1).toUpperCase() + s.substring(1); - } - - /** - * 驼峰命名法工具 - * - * @return toCamelCase(" hello_world ") == "helloWorld" - * toCapitalizeCamelCase("hello_world") == "HelloWorld" - * toUnderScoreCase("helloWorld") = "hello_world" - */ - static String toUnderScoreCase(String s) { - if (s == null) { - return null; - } - - StringBuilder sb = new StringBuilder(); - boolean upperCase = false; - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - - boolean nextUpperCase = true; - - if (i < (s.length() - 1)) { - nextUpperCase = Character.isUpperCase(s.charAt(i + 1)); - } - - if ((i > 0) && Character.isUpperCase(c)) { - if (!upperCase || !nextUpperCase) { - sb.append(SEPARATOR); - } - upperCase = true; - } else { - upperCase = false; - } - - sb.append(Character.toLowerCase(c)); - } - - return sb.toString(); - } - - - /** - * 获取ip地址 - * - * @param request - * @return - */ - public static String getIp(HttpServletRequest request) { - String ip = request.getHeader("x-forwarded-for"); - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - ip = request.getHeader("Proxy-Client-IP"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - ip = request.getHeader("WL-Proxy-Client-IP"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - ip = request.getRemoteAddr(); - } - String comma = ","; - String localhost = "127.0.0.1"; - if (ip.contains(comma)) { - ip = ip.split(",")[0]; - } - if (localhost.equals(ip)) { - // 获取本机真正的ip地址 - try { - ip = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - e.printStackTrace(); - } - } - return ip; - } - - public static String getBrowser(HttpServletRequest request) { - UserAgent userAgent = UserAgent.parseUserAgentString(request.getHeader("User-Agent")); - Browser browser = userAgent.getBrowser(); - return browser.getName(); - } - - - /** - * 获得当天是周几 - */ - public static String getWeekDay() { - String[] weekDays = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; - Calendar cal = Calendar.getInstance(); - cal.setTime(new Date()); - - int w = cal.get(Calendar.DAY_OF_WEEK) - 1; - if (w < 0) { - w = 0; - } - return weekDays[w]; - } - - /** - * 字符串匹配工具类(正则表达式) - * - * @param str - * @param regexp - * @return - */ - public static boolean stringMatch(String str, String regexp) { - Pattern pattern = Pattern.compile(regexp); - return pattern.matcher(str).find(); - } - - /** - * 字符串长度不够补位 - * - * @param sourceStr 源字符串 - * @param length 最终长度 - * @param type 补位方式 0:左边 1:右边 - * @return - */ - public static String stringFillIn(String sourceStr, int length, int type) { - if (sourceStr.length() > length) { - return sourceStr; - } - StringBuilder stringBuilder = new StringBuilder(""); - for (int i = 0; i < length - sourceStr.length(); i++) { - stringBuilder.append("0"); - } - - if (type == MagicNumConstant.ZERO) { - return stringBuilder.toString() + sourceStr; - } else { - return sourceStr + stringBuilder.toString(); - } - } - - - /** - * 从头截取string字符串 - * - * @param str 被截取字符串 - * @param truncationIndex 0 -> 截取长度 - * @return - */ - public static String truncationString(String str, int truncationIndex) { - if (str == null) { - return SymbolConstant.BLANK; - } else if (truncationIndex < 1 - || str.length() <= truncationIndex) { - return str; - } - return str.substring(0, truncationIndex); - } - - /** - * 字符串驼峰转下划线 - * - * @param str 被转字符串 - * @return - */ - public static String humpToLine(String str) { - StringBuilder stringBuilder = new StringBuilder(); - char[] chars = str.toCharArray(); - for (char charactor : chars) { - if (Character.isUpperCase(charactor)) { - stringBuilder.append("_"); - charactor = Character.toLowerCase(charactor); - } - stringBuilder.append(charactor); - } - return stringBuilder.toString(); - } - - /** - * 字符串下划线转驼峰 - * - * @param str 被转字符串 - * @return - */ - public static String lineToHump(String str) { - str = str.toLowerCase(); - Matcher matcher = linePattern.matcher(str); - StringBuffer sb = new StringBuffer(); - while (matcher.find()) { - matcher.appendReplacement(sb, matcher.group(1).toUpperCase()); - } - matcher.appendTail(sb); - return sb.toString(); - } - - - /** - * 字符串截取前 - * - * @param str - * @return - */ - public static String substringBefore(String str, String separator) { - - if (!isEmpty(str) && separator != null) { - if (separator.isEmpty()) { - return ""; - } else { - int pos = str.indexOf(separator); - return pos == -1 ? str : str.substring(0, pos); - } - } else { - return str; - } - } - - /** - * 字符串截取后 - * - * @param str - * @return - */ - public static String substringAfter(String str, String separator) { - - if (isEmpty(str)) { - return str; - } else if (separator == null) { - return ""; - } else { - int pos = str.indexOf(separator); - return pos == -1 ? "" : str.substring(pos + separator.length()); - } - } - - /** - * 获取UUID字符串 - * - * @return - */ - public static String getUUID() { - return UUID.randomUUID().toString().replace(SymbolConstant.HYPHEN, SymbolConstant.BLANK); - } - - /** - * 生成4位随机[a-z]字符串 - * - * @return - */ - public static String getRandomString() { - return RandomStringUtils.randomAlphabetic(NumberConstant.NUMBER_4).toLowerCase(); - } - - /** - * 生成时间戳 + 4位随机数 - * - * @return - */ - public static String getTimestamp() { - SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss"); - String dateStr = df.format(System.currentTimeMillis()); - return dateStr + getRandomString(); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java deleted file mode 100644 index 6cb0344..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/TimeTransferUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; - -import static org.dubhe.base.MagicNumConstant.EIGHT; - -/** - * @description 时间格式转换工具类 - * @date 2020-05-20 - */ -public class TimeTransferUtil { - - private static final String UTC_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.sss'Z'"; - - /** - * Date转换为UTC时间 - * - * @param date - * @return utcTime - */ - public static String dateTransferToUtc(Date date){ - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - /**UTC时间与CST时间相差8小时**/ - calendar.set(Calendar.HOUR,calendar.get(Calendar.HOUR) - EIGHT); - SimpleDateFormat utcSimpleDateFormat = new SimpleDateFormat(UTC_FORMAT); - Date utcDate = calendar.getTime(); - return utcSimpleDateFormat.format(utcDate); - } -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java b/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java deleted file mode 100644 index 465fe04..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/UniqueKeyGenerator.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @description 唯一码生成器 （依赖时间轴） - * @date 2020-05-08 - */ -public class UniqueKeyGenerator { - - private UniqueKeyGenerator(){ - - } - - /** - * 累加上限 - */ - private static final int ACCUMULATOR_MAX = 1000; - - /** - * 累加器 format - */ - private static final String ACCUMULATOR_FORMAT = "%0" + (ACCUMULATOR_MAX+"").length() + "d"; - - /** - * 时钟 format - */ - private static final String DATE_FORMAT = "yyMMddHHmmssSSS"; - - /** - * 累加计数器仓库 - * - * key:prefix - * value:累加计数器 - */ - private static Map prefixAccumulatorStorage = new ConcurrentHashMap<>(); - - /** - * 时间点仓库 - * - * key:prefix - * value:DATE_FORMAT 时钟 - */ - private static Map prefixDateStorage = new ConcurrentHashMap<>(); - - private static String getCurDateStr(){ - DateFormat df = new SimpleDateFormat(DATE_FORMAT); - return df.format(new Date()); - } - - /** - * 数字前位自动补0 - * - * @param atomicInteger - * @return - */ - private static String getAccumulatorStr(AtomicInteger atomicInteger){ - return String.format(ACCUMULATOR_FORMAT,atomicInteger.intValue()); - } - - private static String generateKey(String prefix,String curDateStr,AtomicInteger accumulator){ - return prefix + curDateStr + getAccumulatorStr(accumulator); - } - - /** - * 生成唯一key - * - * @param prefix - * @return - */ - public static String generateKey(final String prefix){ - if (prefix == null){ - return null; - } - synchronized(prefix){ - String curDateStr = getCurDateStr(); - String prevDateStr = prefixDateStorage.get(prefix); - if (curDateStr.equals(prevDateStr)){ - AtomicInteger accumulator = prefixAccumulatorStorage.get(prefix); - if (accumulator.incrementAndGet() >= ACCUMULATOR_MAX){ - return generateKey(prefix); - }else { - updateStorage(prefix,curDateStr,accumulator); - return generateKey(prefix,curDateStr,accumulator); - } - }else { - AtomicInteger accumulator = new AtomicInteger(0); - updateStorage(prefix,curDateStr,accumulator); - return generateKey(prefix,curDateStr,accumulator); - } - } - } - - - /** - * 更新仓库 - * - * @param prefix - * @param curDateStr - * @param accumulator - */ - private static void updateStorage(String prefix, String curDateStr, AtomicInteger accumulator) { - prefixAccumulatorStorage.put(prefix,accumulator); - prefixDateStorage.put(prefix,curDateStr); - } - - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java b/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java deleted file mode 100644 index c6fa6f2..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/WrapperHelp.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.util.ObjectUtil; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import lombok.extern.slf4j.Slf4j; -import org.dubhe.annotation.Query; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; - -/** - * @description 构建Wrapper - * @date 2020-03-15 - */ -@Slf4j -public class WrapperHelp { - - public static QueryWrapper getWrapper(T query) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - if (query == null) { - return queryWrapper; - } - try { - List fields = getAllFields(query.getClass(), new ArrayList<>()); - for (Field field : fields) { - field.setAccessible(true); - Query q = field.getAnnotation(Query.class); - if (q != null) { - String propName = q.propName(); - String attributeName = isBlank(propName) ? field.getName() : propName; - Object val = field.get(query); - if (val == null) { - continue; - } - // 模糊多字段 - String blurry = q.blurry(); - if (ObjectUtil.isNotEmpty(blurry)) { - String[] blurrys = blurry.split(","); - queryWrapper.and(qw -> { - for (int i = 0; i < blurrys.length; i++) { - if (i == 0) { - qw.like(blurrys[i], val); - } else { - qw.or().like(blurrys[i], val); - } - } - }); - continue; - } - switch (q.type()) { - case EQ: - queryWrapper = queryWrapper.eq(attributeName, val); - break; - case NE: - queryWrapper = queryWrapper.ne(attributeName, val); - break; - case GE: - queryWrapper = queryWrapper.ge(attributeName, val); - break; - case GT: - queryWrapper = queryWrapper.gt(attributeName, val); - break; - case LT: - queryWrapper = queryWrapper.lt(attributeName, val); - break; - case LE: - queryWrapper = queryWrapper.le(attributeName, val); - break; - case BETWEEN: - List between = new ArrayList<>((List) val); - queryWrapper = queryWrapper.between(attributeName, between.get(0), between.get(1)); - break; - case NOT_BETWEEN: - List notBetween = new ArrayList<>((List) val); - queryWrapper = queryWrapper.notBetween(attributeName, notBetween.get(0), notBetween.get(1)); - break; - case LIKE: - queryWrapper = queryWrapper.like(attributeName, val); - break; - case NOT_LIKE: - queryWrapper = queryWrapper.notLike(attributeName, val); - break; - case LIkE_LEFT: - queryWrapper = queryWrapper.likeLeft(attributeName, val); - break; - case LIKE_RIGHT: - queryWrapper = queryWrapper.likeRight(attributeName, val); - break; - case IS_NULL: - queryWrapper = queryWrapper.isNull(attributeName); - break; - case IS_NOT_NULL: - queryWrapper = queryWrapper.isNotNull(attributeName); - break; - case IN: - queryWrapper = queryWrapper.in(attributeName, (Collection) val); - break; - case NOT_IN: - queryWrapper = queryWrapper.notIn(attributeName, (Collection) val); - break; - case INSQL: - queryWrapper = queryWrapper.inSql(attributeName, String.valueOf(val)); - break; - case NOT_INSQL: - queryWrapper = queryWrapper.notInSql(attributeName, String.valueOf(val)); - break; - case ORDER_BY: - queryWrapper = queryWrapper.last(" ORDER BY " + val); - break; - default: - break; - } - } - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - return queryWrapper; - } - - private static boolean isBlank(final CharSequence cs) { - int strLen; - if (cs == null || (strLen = cs.length()) == 0) { - return true; - } - for (int i = 0; i < strLen; i++) { - if (!Character.isWhitespace(cs.charAt(i))) { - return false; - } - } - return true; - } - - private static List getAllFields(Class clazz, List fields) { - if (clazz != null) { - fields.addAll(Arrays.asList(clazz.getDeclaredFields())); - getAllFields(clazz.getSuperclass(), fields); - } - return fields; - } - -} diff --git a/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java b/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java deleted file mode 100644 index 2a31df0..0000000 --- a/dubhe-server/common/src/main/java/org/dubhe/utils/ZipCommonUtil.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import net.lingala.zip4j.core.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import org.dubhe.enums.LogEnum; -import org.springframework.stereotype.Component; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Pattern; - -/** - * @description zip Util - * @date 2020-5-22 - */ -@Component -public class ZipCommonUtil { - public static boolean unzipLocal(String source) { - if (source == null) { - return false; - } - String[] sourceArr = source.split(Pattern.quote(File.separator)); - int length = sourceArr.length; - if (length <= 1) { - return false; - } - List destList = new ArrayList<>(Arrays.asList(sourceArr).subList(0, length - 1)); - String destination = String.join(File.separator, destList); - - try { - ZipFile zipFile = new ZipFile(source); - zipFile.extractAll(destination); - } catch (ZipException e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "解压失败: ", e); - return false; - } - return true; - } -} diff --git a/dubhe-server/common/src/main/resources/i18n/messages_en.properties b/dubhe-server/common/src/main/resources/i18n/messages_en.properties deleted file mode 100644 index a26af0d..0000000 --- a/dubhe-server/common/src/main/resources/i18n/messages_en.properties +++ /dev/null @@ -1,51 +0,0 @@ -#\u56fd\u9645\u5316\u4fe1\u606f -HTTPCODE_207=Your operation frequency is too fast, Please try again later! -HTTPCODE_200=Http request succeed -HTTPCODE_303=Login failed -HTTPCODE_400=Request parameter error -HTTPCODE_401=Unauthorized, please login -HTTPCODE_403=Forbidden -HTTPCODE_404=Page not found -HTTPCODE_408=Request timeout -HTTPCODE_409=Conflict -HTTPCODE_410=Gone -HTTPCODE_423=Locked -HTTPCODE_500=Server error -EMAIL.SET_HOST=Setting system property: mail.smtp.host={} -EMAIL.PRE_TALK=Ready to get email session object -EMAIL.ERROR_TALK=Error occurred during getting email session object -EMAIL.PRE_MIME=Ready to create MIME email object -EMAIL.ERROR_MIME=Create MIME email object failed -EMAIL.SET_AUTH=Setting smtp authorization :mail.smtp.auth={} -EMAIL.SET_SUBJECT=Setting email subject[{}]! -EMAIL.ERROR_SUBJECT=Error occurred while setting email subject -EMAIL.ERROR_BODY=Error occured while setting email body -EMAIL.ADD_ATTEND=Add attach [{}]! -EMAIL.SET_TO=Setting sendto : [{}]! -EMAIL.SET_COPYTO=Setting copyto [{}]! -EMAIL.SENDING=Email is sending... -EMAIL.SEND_SUCC=Email send succeed -EMAIL.SEND_ERR=Email send error -THIRDPARTY.LOGIN.NOTOKEN=Thirdparty token : %s get failed -IDCARD_ILLEGAL=Idcard is illegal -EMAIL_ILLEGAL=Email is illegal -MOBILE_ILLEGAL=Mobile is illegal -TASKID_IS_NULL=TaskId is null -LOGIN_FAIL=Incorrect username or password. -ACCOUNT_IS_NULL=Account is null -ACCOUNT_LENGTH=Account allowed length: {}-{}. -PASSWORD_IS_NULL=Password is null -USER_ID_IS_NULL=UserId is null -USER_IS_NULL=UserId[%1$s]error. -ID_IS_NULL=Id is null -TASKGROUP_IS_NULL=Taskgroup is null -TASKNAME_IS_NULL=Taskname is null -KEY_IS_NULL=Key is null -MENUID_IS_NULL=Menuid is null -NULL_POINTER_EXCEPTION_HAPPENS_IN_SERVICE_CALL=NULL POINTER EXCEPTION HAPPENS IN SERVICE CALL -SYS.EIXST_ROLE_CODE=ROLE CODE ALREADY EXISTS -DELETE_ROLE_ERROR=This role has user, delete failed -DEPARTMENT_ALREADY_EXIST=Department already exist -EXIST_SUBDEPARTMENT_OF_THIS_DEPARTMENT=Exist subdeptment of this department -EXIST_USER_OF_THIS_DEPARTMENT=Exist user of this department -HAS_SUB_LOCATIONS=Exist subLocation of this location diff --git a/dubhe-server/common/src/main/resources/i18n/messages_zh.properties b/dubhe-server/common/src/main/resources/i18n/messages_zh.properties deleted file mode 100644 index 0980220..0000000 --- a/dubhe-server/common/src/main/resources/i18n/messages_zh.properties +++ /dev/null @@ -1,51 +0,0 @@ -#\u56fd\u9645\u5316\u4fe1\u606f -HTTPCODE_207=\u60a8\u7684\u64cd\u4f5c\u9891\u7387\u8fc7\u5feb\uff0c\u8bf7\u7a0d\u540e\u518d\u8bd5! -HTTPCODE_200=\u8bf7\u6c42\u6210\u529f -HTTPCODE_303=\u767b\u5f55\u5931\u8d25 -HTTPCODE_400=\u8bf7\u6c42\u53c2\u6570\u51fa\u9519 -HTTPCODE_401=\u6ca1\u6709\u767b\u5f55 -HTTPCODE_403=\u6ca1\u6709\u6743\u9650 -HTTPCODE_404=\u627e\u4e0d\u5230\u9875\u9762 -HTTPCODE_408=\u8bf7\u6c42\u8d85\u65f6 -HTTPCODE_409=\u53d1\u751f\u51b2\u7a81 -HTTPCODE_410=\u5df2\u88ab\u5220\u9664 -HTTPCODE_423=\u5df2\u88ab\u9501\u5b9a -HTTPCODE_500=\u670d\u52a1\u5668\u51fa\u9519 -EMAIL.SET_HOST=\u8bbe\u7f6e\u7cfb\u7edf\u5c5e\u6027\uff1amail.smtp.host={} -EMAIL.PRE_TALK=\u51c6\u5907\u83b7\u53d6\u90ae\u4ef6\u4f1a\u8bdd\u5bf9\u8c61\uff01 -EMAIL.ERROR_TALK=\u83b7\u53d6\u90ae\u4ef6\u4f1a\u8bdd\u5bf9\u8c61\u65f6\u53d1\u751f\u9519\u8bef\uff01 -EMAIL.PRE_MIME=\u51c6\u5907\u521b\u5efaMIME\u90ae\u4ef6\u5bf9\u8c61\uff01 -EMAIL.ERROR_MIME=\u521b\u5efaMIME\u90ae\u4ef6\u5bf9\u8c61\u5931\u8d25\uff01 -EMAIL.SET_AUTH=\u8bbe\u7f6esmtp\u8eab\u4efd\u8ba4\u8bc1\uff1amail.smtp.auth={} -EMAIL.SET_SUBJECT=\u8bbe\u7f6e\u90ae\u4ef6\u4e3b\u9898[{}]\uff01 -EMAIL.ERROR_SUBJECT=\u8bbe\u7f6e\u90ae\u4ef6\u4e3b\u9898\u53d1\u751f\u9519\u8bef\uff01 -EMAIL.ERROR_BODY=\u8bbe\u7f6e\u90ae\u4ef6\u6b63\u6587\u65f6\u53d1\u751f\u9519\u8bef\uff01 -EMAIL.ADD_ATTEND=\u589e\u52a0\u90ae\u4ef6\u9644\u4ef6[{}]\uff01 -EMAIL.SET_TO=\u8bbe\u7f6e\u6536\u4fe1\u4eba[{}]\uff01 -EMAIL.SET_COPYTO=\u8bbe\u7f6e\u6284\u9001\u4eba[{}]\uff01 -EMAIL.SENDING=\u6b63\u5728\u53d1\u9001\u90ae\u4ef6.... -EMAIL.SEND_SUCC=\u53d1\u9001\u90ae\u4ef6\u6210\u529f\uff01 -EMAIL.SEND_ERR=\u90ae\u4ef6\u53d1\u9001\u5931\u8d25\uff01 -THIRDPARTY.LOGIN.NOTOKEN=\u672a\u83b7\u53d6\u5230%s\u767b\u5f55\u4ee4\u724c\uff01 -IDCARD_ILLEGAL=\u8eab\u4efd\u8bc1\u53f7\u7801\u9519\u8bef. -EMAIL_ILLEGAL=\u90ae\u7bb1\u683c\u5f0f\u9519\u8bef. -MOBILE_ILLEGAL=\u624b\u673a\u53f7\u683c\u5f0f\u9519\u8bef. -TASKID_IS_NULL=\u5b9a\u65f6\u4efb\u52a1Id\u4e0d\u80fd\u4e3a\u7a7a. -LOGIN_FAIL=\u7528\u6237\u540d\u6216\u5bc6\u7801\u9519\u8bef. -ACCOUNT_IS_NULL=\u5e10\u53f7\u4e0d\u80fd\u4e3a\u7a7a. -ACCOUNT_LENGTH=\u5e10\u53f7\u5141\u8bb8\u957f\u5ea6\u8303\u56f4{}-{}. -PASSWORD_IS_NULL=\u5bc6\u7801\u4e0d\u80fd\u4e3a\u7a7a. -USER_ID_IS_NULL=\u7528\u6237ID\u4e0d\u80fd\u4e3a\u7a7a. -USER_IS_NULL=\u7528\u6237Id[%1$s]\u9519\u8bef. -ID_IS_NULL=Id\u4e0d\u80fd\u4e3a\u7a7a. -TASKGROUP_IS_NULL=\u4efb\u52a1\u7ec4\u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a. -TASKNAME_IS_NULL=\u4efb\u52a1\u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a. -KEY_IS_NULL=\u952e\u503c\u4e3a\u7a7a -MENUID_IS_NULL=\u6b63\u5728\u64cd\u4f5c\u83dc\u5355id\u4e0d\u80fd\u4e3a\u7a7a -DELETE_ROLE_ERROR=\u8be5\u89d2\u8272\u5df2\u88ab\u7528\u6237\u5173\u8054\u4e0d\u652f\u6301\u5220\u9664 -NULL_POINTER_EXCEPTION_HAPPENS_IN_SERVICE_CALL=\u670d\u52a1\u8c03\u7528\u53d1\u751f\u7a7a\u6307\u9488\u5f02\u5e38 -SYS.EIXST_ROLE_CODE=\u89d2\u8272\u7f16\u7801\u91cd\u590d -DEPARTMENT_ALREADY_EXIST=\u90e8\u95e8\u5df2\u5b58\u5728 -EXIST_SUBDEPARTMENT_OF_THIS_DEPARTMENT=\u8be5\u90e8\u95e8\u4e0b\u5b58\u5728\u5b50\u90e8\u95e8 -EXIST_USER_OF_THIS_DEPARTMENT=\u8be5\u90e8\u95e8\u4e0b\u5b58\u5728\u7528\u6237 -HAS_SUB_LOCATIONS=\u8be5\u5730\u70b9\u4e0b\u542b\u6709\u5b50\u8282\u70b9 diff --git a/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java deleted file mode 100644 index c6f5c9f..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/EncryptUtilsTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Test; - -import static org.dubhe.utils.EncryptUtils.desDecrypt; -import static org.dubhe.utils.EncryptUtils.desEncrypt; -import static org.junit.Assert.assertEquals; -/** - * @description 对称加密|解密 - * @date 2020-03-25 - */ -public class EncryptUtilsTest { - - /** - * 对称加密 - */ - @Test - public void testDesEncrypt() { - try { - assertEquals("7772841DC6099402", desEncrypt("123456")); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 对称解密 - */ - @Test - public void testDesDecrypt() { - try { - assertEquals("123456", desDecrypt("7772841DC6099402")); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java deleted file mode 100644 index 61119be..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/FileUtilTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Test; -import org.springframework.mock.web.MockMultipartFile; - -import static org.dubhe.utils.FileUtil.*; -import static org.junit.Assert.assertEquals; -/** - * @description 测试类 - * @date 2020-03-25 - */ -public class FileUtilTest { - - @Test - public void testToFile() { - long retval = toFile(new MockMultipartFile("foo", (byte[]) null)).getTotalSpace(); - assertEquals(500695072768L, retval); - } - - @Test - public void testGetExtensionName() { - assertEquals("foo", getExtensionName("foo")); - assertEquals("exe", getExtensionName("bar.exe")); - } - - @Test - public void testGetFileNameNoEx() { - assertEquals("foo", getFileNameNoEx("foo")); - assertEquals("bar", getFileNameNoEx("bar.txt")); - } - - @Test - public void testGetSize() { - assertEquals("1000B ", getSize(1000)); - assertEquals("1.00KB ", getSize(1024)); - assertEquals("1.00MB ", getSize(1048576)); - assertEquals("1.00GB ", getSize(1073741824)); - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java deleted file mode 100644 index 40be6b7..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/HttpUtilsTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Assert; -import org.junit.Test; - -import static org.dubhe.utils.HttpUtils.isSuccess; - -/** - * @description HttpUtil - * @date 2020-04-30 - */ -public class HttpUtilsTest { - - @Test - public void testIsSuccess() { - Assert.assertTrue(isSuccess("200")); - Assert.assertFalse(isSuccess("2020")); - Assert.assertFalse(isSuccess("401")); - Assert.assertFalse(isSuccess(null)); - Assert.assertFalse(isSuccess("")); - Assert.assertTrue(isSuccess(200)); - } - - -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java b/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java deleted file mode 100644 index 5264e93..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/StringUtilsTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.dubhe.utils.StringUtils; -import org.junit.Test; -import org.springframework.mock.web.MockHttpServletRequest; - -import java.text.SimpleDateFormat; -import java.util.Date; - -import static org.dubhe.utils.StringUtils.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -/** - * @description 测试类 - * @date 2020-03-25 - */ -public class StringUtilsTest { - - @Test - public void testToCamelCase() { - assertNull(toCamelCase(null)); - } - - @Test - public void testToCapitalizeCamelCase() { - assertNull(StringUtils.toCapitalizeCamelCase(null)); - assertEquals("HelloWorld", toCapitalizeCamelCase("hello_world")); - } - - @Test - public void testGetWeekDay() { - SimpleDateFormat simpleDateformat = new SimpleDateFormat("E"); - assertEquals(simpleDateformat.format(new Date()), getWeekDay()); - } - - @Test - public void testGetIp() { - assertEquals("127.0.0.1", getIp(new MockHttpServletRequest())); - } - - @Test - public void truncationString() { - assertEquals("", StringUtils.truncationString(null,6)); - assertEquals("012345", StringUtils.truncationString("012345",0)); - assertEquals("0", StringUtils.truncationString("012345",1)); - assertEquals("012345", StringUtils.truncationString("012345",6)); - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java deleted file mode 100644 index b308dcc..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/AesUtilTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.junit.Assert; -import org.junit.Test; - -/** - * @desc - * @date 2020.06.01 - */ -public class AesUtilTest { - - @Test - public void encryptAndDecrypt(){ - String data = "20200601163000"; - String key = "123412asdsad"; - String val = AesUtil.encrypt(data,key); - System.out.println(val); - Assert.assertEquals(data, AesUtil.decrypt(val,key)); - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java deleted file mode 100644 index b667dd5..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/NotebookBizExceptionTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.alibaba.fastjson.JSON; -import lombok.extern.slf4j.Slf4j; -import org.dubhe.exception.NotebookBizException; -import org.junit.Test; - -/** - * @desc - * @date 2020.06.16 - */ -@Slf4j -public class NotebookBizExceptionTest { - - @Test - public void toStringTest() { - try { - throw new NotebookBizException("NotebookBizException toString Test"); - }catch (NotebookBizException e){ - log.error("{}", e); - } - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java deleted file mode 100644 index a7369e2..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/ReadFileTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @description - * @create: 2020-05-07 - */ -public class ReadFileTest { - - @Test - public void testRead(){ - String path = "/Users/lizhe/hello2.log"; - List ans = Arrays.asList("lin1", "l2", "l3", "l4"); - List ans2 = ans.subList(1, 3); - assertEquals(ans, ReadFile.read(path,-2, 10)); - assertEquals(ans2, ReadFile.read(path, 1, 2)); - } -} diff --git a/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java b/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java deleted file mode 100644 index 9ff3c93..0000000 --- a/dubhe-server/common/src/test/java/org/dubhe/utils/ZipUtilTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.junit.Test; -/** - * @description - * @date 2020-05-25 - */ -public class ZipUtilTest { - @Test - public void testUnzip(){ - ZipCommonUtil.unzipLocal("Z:\\dubhe-dev\\undefined\\a1.zip"); - } -} - diff --git a/dubhe-server/deploy-docker.sh b/dubhe-server/deploy-docker.sh deleted file mode 100644 index 6ae01a9..0000000 --- a/dubhe-server/deploy-docker.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -#环境，用以区分部署的命名空间，日志路径 -ENV=$1 - -SOURCE_CODE_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd ) -#harbor 地址 -HARBOR_URL=harbor.dubhe.ai -#harbor 用户名 -HARBOR_USERNAME=admin -#harbor 密码 -HARBOR_PWD=123456 -#nfs 地址 -NFS_ADDRESS=127.0.0.1 -#nfs 服务器 root权限用户名 -NFS_HOST_USER=root -#nfs 服务器 root权限用户密码 -NFS_HOST_PWD=123456 -#nfs 共享目录 -NFS_PATH=/nfs -#容器日志路径 -CONTAINER_LOG_PATH=/logs -#宿主机日志路径 -HOST_LOG_PATH=/logs/dubhe-${ENV} -#admin模块端口 -ADMIN_PORT=8000 -#serving-gateway模块端口 -GATEWAY_PORT=8081 - -#挂载nfs -mount_nfs() { - mkdir ${NFS_PATH} - mount -t nfs ${NFS_ADDRESS}:${NFS_PATH} ${NFS_PATH} -} - -update_dockerfile() { - cd ${SOURCE_CODE_PATH} && sed -i "s#nfs-host-pwd#${NFS_HOST_PWD}#g;s#nfs-host-user#${NFS_HOST_USER}#g;s#nfs-host-ip#${NFS_ADDRESS}#g" Dockerfile -} -mvn_build() { - cd ${SOURCE_CODE_PATH} && mvn clean compile package - cd ${SOURCE_CODE_PATH}/dubhe-serving-gateway && mvn clean compile package -} -#删除镜像 -delete_old_image() { - docker rmi -f ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} -} -#构建镜像 -build_image() { - cd ${SOURCE_CODE_PATH} && docker build -t ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} . -} - -#运行admin模块 -run_admin() { - docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${ADMIN_PORT}:${ADMIN_PORT} ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-admin-1.0-exec.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null -} -#运行task模块 -run_task() { - docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-task-1.0.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null -} -#运行serving-gateway模块 -run_gateway() { - docker run -d -v ${HOST_LOG_PATH}:${CONTAINER_LOG_PATH} -v ${NFS_PATH}:${NFS_PATH} -v /var/run/docker.sock:/var/run/docker.sock -p ${GATEWAY_PORT}:${GATEWAY_PORT} ${HARBOR_URL}/dubhe/dubhe-spring:${ENV} java -jar /dubhe/dubhe-serving-gateway.jar --spring.profiles.active=${ENV} 2>&1 > /dev/null -} - -mount_nfs -update_dockerfile -mvn_build -delete_old_image -build_image -run_admin -run_task -run_gateway diff --git a/dubhe-server/deploy.sh b/dubhe-server/deploy.sh deleted file mode 100644 index 89d6d9b..0000000 --- a/dubhe-server/deploy.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -PROG_NAME=$0 -ACTION=$1 -ENV=$2 -APP_HOME=$3 - -APP_NAME=dubhe-${ENV} - -APP_HOME=$APP_HOME/${APP_NAME} # 从package.tgz中解压出来的jar包放到这个目录下 -JAR_NAME=${APP_HOME}/dubhe-admin/target/dubhe-admin-1.0-exec.jar # jar包的名字 -JAVA_OUT=/dev/null - -# 创建出相关目录 -mkdir -p ${APP_HOME} -mkdir -p ${APP_HOME}/logs - -usage() { - echo "Usage: $PROG_NAME {start|stop|restart} {dev|test|prod}" - exit 2 -} - -start_application() { - echo "starting java process" - echo "nohup java -Dfile.encoding=utf-8 -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 &" - nohup java -Dfile.encoding=utf-8 -jar ${JAR_NAME} > ${JAVA_OUT} --spring.profiles.active=${ENV} 2>&1 & - echo "started java process" -} - -stop_application() { - checkjavapid=`ps -ef | grep java | grep ${APP_NAME} | grep -v grep |grep -v 'deploy.sh'| awk '{print$2}'` - - if [ -z $checkjavapid ];then - echo -e "\rno java process "$checkjavapid - return - fi - - echo "stop java process" - times=60 - for e in $(seq 60) - do - sleep 1 - COSTTIME=$(($times - $e )) - checkjavapid=`ps -ef | grep java | grep ${APP_NAME} | grep -v grep |grep -v 'deploy.sh'| awk '{print$2}'` - if [ "$checkjavapid" != "" ];then - echo "kill "$checkjavapid - kill -9 $checkjavapid - echo -e "\r -- stopping java lasts `expr $COSTTIME` seconds." - else - echo -e "\rjava process has exited" - break; - fi - done - echo "" -} - -case "$ACTION" in - start) - start_application - ;; - stop) - stop_application - ;; - restart) - stop_application - start_application - ;; - *) - usage - ;; -esac diff --git a/dubhe-server/dubhe-admin/pom.xml b/dubhe-server/dubhe-admin/pom.xml deleted file mode 100644 index 8a4c76e..0000000 --- a/dubhe-server/dubhe-admin/pom.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - dubheplatform - zhejianglab - 1.0 - - 4.0.0 - - dubhe-admin - 平台管理 - - 0.10.6 - dubhe - - - - zhejianglab - common - 1.0 - - - zhejianglab - dubhe-system - 1.0 - - - zhejianglab - common - - - - - zhejianglab - dubhe-model - 1.0 - - - zhejianglab - common - - - - - zhejianglab - dubhe-k8s - 1.0 - - - zhejianglab - common - - - - - zhejianglab - dubhe-dcm - 1.0 - - - zhejianglab - common - - - - - zhejianglab - dubhe-serving - 1.0 - - - zhejianglab - common - - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - false - true - exec - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java deleted file mode 100644 index 58e3188..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/AppRun.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.dubhe.utils.SpringContextHolder; -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; -import org.springframework.boot.web.servlet.server.ServletWebServerFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.scheduling.annotation.EnableAsync; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @description 访问首页提示 - * @date 2020-03-05 - */ -@EnableAsync -@RestController -@SpringBootApplication -@EnableTransactionManagement -@EnableScheduling -@MapperScan("org.dubhe.**.dao") -public class AppRun { - - public static void main(String[] args) { - SpringApplication.run(AppRun.class, args); - } - - @Bean - public SpringContextHolder springContextHolder() { - return new SpringContextHolder(); - } - - @Bean - public ServletWebServerFactory webServerFactory() { - TomcatServletWebServerFactory fa = new TomcatServletWebServerFactory(); - fa.addConnectorCustomizers(connector -> connector.setProperty("relaxedQueryChars", "[]{}")); - return fa; - } - - @Bean - public BCryptPasswordEncoder passwordEncoder() { - return new BCryptPasswordEncoder(); - } - - /** - * 访问首页提示 - * - * @return / - */ - @GetMapping("/") - public String index() { - return "Backend service started successfully"; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java deleted file mode 100644 index eae4676..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/HarborImagePushAsync.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.async; - -import cn.hutool.core.util.StrUtil; -import org.dubhe.base.ResponseCode; -import org.dubhe.config.TrainHarborConfig; -import org.dubhe.dao.PtImageMapper; -import org.dubhe.domain.entity.PtImage; -import org.dubhe.enums.ImageStateEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.harbor.api.HarborApi; -import org.dubhe.utils.IOUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -import java.io.BufferedReader; -import java.io.InputStreamReader; - -/** - * @description 异步推送镜像到harbor - * @date 2020-07-10 - */ -@Component -public class HarborImagePushAsync { - - @Autowired - private TrainHarborConfig trainHarborConfig; - - @Autowired - private HarborApi harborApi; - - @Autowired - private PtImageMapper ptImageMapper; - - /** - * 组装shell脚本执行命令 - * - * @param imagePath - * @param imageNameandTag - **/ - @Async - public void execShell(String imagePath, String imageNameandTag, PtImage ptImage) { - try { - String imageResource = trainHarborConfig.getAddress() + StrUtil.SLASH + ptImage.getProjectName() - + StrUtil.SLASH + imageNameandTag; - String cmdStr = "docker login --username=" + trainHarborConfig.getUsername() + " " + trainHarborConfig.getAddress() + " --password=" + trainHarborConfig.getPassword() + " ; docker " + - "load < " + imagePath + " |awk '{print $3}' |xargs -I str docker tag str " + imageResource + " ; docker push " + imageResource + "; docker rmi " + imageResource; - String[] cmd = {"/bin/bash", "-c", cmdStr}; - LogUtil.info(LogEnum.BIZ_TRAIN, "镜像上传执行脚本参数:{}", cmd); - - Process process = Runtime.getRuntime().exec(cmd); - if (checkImagePushIsOk(ptImage, process)) { - updateImageStatus(ptImage, ImageStateEnum.SUCCESS.getCode()); - } else { - updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); - } - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "上传镜像异常:{}", e); - updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); - throw new BusinessException("上传镜像异常!"); - - } - } - - /** - * 更新镜像上传状态 - * - * @param ptImage - * @param status - * @return java.lang.Integer - **/ - public Integer updateImageStatus(PtImage ptImage, Integer status) { - ptImage.setImageStatus(status); - ptImageMapper.updateById(ptImage); - return ResponseCode.SUCCESS; - } - - - /** - * 校验镜像是否上传成功 - * - * @param ptImage 镜像信息 - * @param process process对象 - * @return 是否上传成功 - */ - public boolean checkImagePushIsOk(PtImage ptImage, Process process) { - //读取标准输出流 - BufferedReader brOut = new BufferedReader(new InputStreamReader(process.getInputStream())); - //读取标准错误流 - BufferedReader brErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - String line; - StringBuilder outMessage = new StringBuilder(); - StringBuilder errMessage = new StringBuilder(); - boolean isPushOk = true; - try { - while ((line = brOut.readLine()) != null) { - outMessage.append(line); - } - if (StringUtils.isNotEmpty(outMessage)) { - LogUtil.info(LogEnum.BIZ_TRAIN, "shell上传镜像输出信息:{}", outMessage.toString()); - } - while ((line = brErr.readLine()) != null) { - errMessage.append(line); - } - if (StringUtils.isNotEmpty(errMessage)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "shell上传镜像异常信息:{}", errMessage.toString()); - } - Integer status = process.waitFor(); - LogUtil.info(LogEnum.BIZ_TRAIN, "上传镜像状态:{}", status); - if (status == null) { - if (!harborApi.isExistImage(ptImage.getImageUrl())) { - isPushOk = false; - } - } else if (status != 0) { - isPushOk = false; - } - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "上传镜像异常:{}", e); - return false; - } finally { - IOUtil.close(brErr, brOut); - } - return isPushOk; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java deleted file mode 100644 index 3bd6783..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/StopTrainJobAsync.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.async; - -import org.dubhe.config.TrainJobConfig; -import org.dubhe.dao.PtTrainJobMapper; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.TrainJobStatusEnum; -import org.dubhe.enums.TrainTypeEnum; -import org.dubhe.k8s.api.DistributeTrainApi; -import org.dubhe.k8s.api.PodApi; -import org.dubhe.k8s.api.TrainJobApi; -import org.dubhe.k8s.domain.resource.BizPod; -import org.dubhe.utils.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.function.Consumer; - -/** - * @description 停止训练任务异步处理 - * @date 2020-08-13 - */ -@Component -public class StopTrainJobAsync { - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private PodApi podApi; - - @Autowired - private TrainJobApi trainJobApi; - - @Autowired - private TrainJobConfig trainJobConfig; - - @Autowired - private DistributeTrainApi distributeTrainApi; - - @Autowired - private PtTrainJobMapper ptTrainJobMapper; - - /** - * 停止任务 - * - * @param currentUser 用户 - * @param jobList 任务集合 - */ - @Async("trainExecutor") - public void stopJobs(UserDTO currentUser, List jobList) { - String namespace = k8sNameTool.generateNamespace(currentUser.getId()); - jobList.forEach(job -> { - BizPod bizPod = podApi.getWithResourceName(namespace, job.getJobName()); - if (!bizPod.isSuccess()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stops training Job return code:{},message:{}", currentUser.getUsername(), Integer.valueOf(bizPod.getCode()), bizPod.getMessage()); - } - boolean bool = TrainTypeEnum.isDistributeTrain(job.getTrainType()) ? - distributeTrainApi.deleteByResourceName(namespace, job.getJobName()).isSuccess() : - trainJobApi.delete(namespace, job.getJobName()); - if (!bool) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stops training Job and K8S fails in the stop process, namespace为{}, resourceName为{}", - currentUser.getUsername(), namespace, job.getJobName()); - } - //更新训练状态 - job.setRuntime(calculateRuntime(bizPod)) - .setTrainStatus(TrainJobStatusEnum.STOP.getStatus()); - ptTrainJobMapper.updateById(job); - - }); - } - - - /** - * 计算job训练时长 - * - * @param bizPod pod信息 - * @return String 训练时长 - */ - private String calculateRuntime(BizPod bizPod) { - return calculateRuntime(bizPod, (x) -> { - }); - } - - - /** - * 计算job训练时长 - * - * @param bizPod - * @param consumer pod已经完成状态的回调函数 - * @return res 返回训练时长 - */ - private String calculateRuntime(BizPod bizPod, Consumer consumer) { - Long completedTime; - if (StringUtils.isBlank(bizPod.getStartTime())) { - return TrainUtil.INIT_RUNTIME; - } - Long startTime = transformTime(bizPod.getStartTime()); - boolean hasCompleted = StringUtils.isNotBlank(bizPod.getCompletedTime()); - completedTime = hasCompleted ? transformTime(bizPod.getCompletedTime()) : LocalDateTime.now().toEpochSecond(ZoneOffset.of(trainJobConfig.getPlusEight())); - Long time = completedTime - startTime; - String res = DubheDateUtil.convert2Str(time); - if (hasCompleted) { - consumer.accept(res); - } - return res; - } - - - /** - * 时间转换 - * - * @param time 时间 - * @return Long 时间戳 - */ - private Long transformTime(String time) { - LocalDateTime localDateTime = LocalDateTime.parse(time, DateTimeFormatter.ISO_OFFSET_DATE_TIME); - //没有根据时区做处理, 默认当前为东八区 - localDateTime = localDateTime.plusHours(Long.valueOf(trainJobConfig.getEight())); - return localDateTime.toEpochSecond(ZoneOffset.of(trainJobConfig.getPlusEight())); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java deleted file mode 100644 index 843ad9a..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainAlgorithmUploadAsync.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.async; - -import org.dubhe.config.NfsConfig; -import org.dubhe.dao.PtTrainAlgorithmMapper; -import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainAlgorithm; -import org.dubhe.enums.AlgorithmStatusEnum; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.NoteBookService; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.LocalFileUtil; -import org.dubhe.utils.LogUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -/** - * @description 异步上传算法 - * @date 2020-08-10 - */ -@Component -public class TrainAlgorithmUploadAsync { - - @Autowired - private LocalFileUtil localFileUtil; - - @Autowired - private NfsConfig nfsConfig; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private NoteBookService noteBookService; - - @Autowired - private PtTrainAlgorithmMapper trainAlgorithmMapper; - - /** - * 异步任务创建算法 - * - * @param user 当前登录用户信息 - * @param ptTrainAlgorithm 算法信息 - * @param trainAlgorithmCreateDTO 创建算法条件 - */ - @Async("trainExecutor") - public void createTrainAlgorithm(UserDTO user, PtTrainAlgorithm ptTrainAlgorithm, PtTrainAlgorithmCreateDTO trainAlgorithmCreateDTO) { - String path = nfsConfig.getBucket() + trainAlgorithmCreateDTO.getCodeDir(); - //校验创建算法来源(true:由fork创建算法，false：其它创建算法方式),若为true则拷贝预置算法文件至新路径 - if (trainAlgorithmCreateDTO.getFork()) { - //生成算法相对路径 - String algorithmPath = k8sNameTool.getNfsPath(BizNfsEnum.ALGORITHM, user.getId()); - //拷贝预置算法文件夹 - boolean copyResult = localFileUtil.copyPath(path, nfsConfig.getBucket() + algorithmPath); - if (!copyResult) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} copied the preset algorithm path {} successfully", user.getUsername(), path); - updateTrainAlgorithm(ptTrainAlgorithm, trainAlgorithmCreateDTO, false); - throw new BusinessException("内部错误"); - } - - ptTrainAlgorithm.setCodeDir(algorithmPath); - - //修改算法上传状态 - updateTrainAlgorithm(ptTrainAlgorithm, trainAlgorithmCreateDTO, true); - - } else { - updateTrainAlgorithm(ptTrainAlgorithm, trainAlgorithmCreateDTO, true); - } - } - - - /** - * 更新上传算法状态 - * - * @param ptTrainAlgorithm 算法信息 - * @param trainAlgorithmCreateDTO 创建算法的条件 - * @param flag 创建算法是否成功(true:成功，false:失败) - */ - public void updateTrainAlgorithm(PtTrainAlgorithm ptTrainAlgorithm, PtTrainAlgorithmCreateDTO trainAlgorithmCreateDTO, boolean flag) { - - LogUtil.info(LogEnum.BIZ_TRAIN, "async update algorithmPath by algorithmId:{} and update noteBook by noteBookId:{}", ptTrainAlgorithm.getId(), trainAlgorithmCreateDTO.getNoteBookId()); - if (flag) { - ptTrainAlgorithm.setAlgorithmStatus(AlgorithmStatusEnum.SUCCESS.getCode()); - //更新fork算法新路径 - trainAlgorithmMapper.updateById(ptTrainAlgorithm); - //保存算法根据notbookId更新算法id - if (trainAlgorithmCreateDTO.getNoteBookId() != null) { - LogUtil.info(LogEnum.BIZ_TRAIN, "Save algorithm Update algorithm ID :{} according to notBookId:{}", trainAlgorithmCreateDTO.getNoteBookId(), ptTrainAlgorithm.getId()); - noteBookService.updateTrainIdByNoteBookId(trainAlgorithmCreateDTO.getNoteBookId(), ptTrainAlgorithm.getId()); - } - } else { - ptTrainAlgorithm.setAlgorithmStatus(AlgorithmStatusEnum.FAIL.getCode()); - trainAlgorithmMapper.updateById(ptTrainAlgorithm); - } - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java deleted file mode 100644 index c7952e2..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TrainJobAsync.java +++ /dev/null @@ -1,511 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.async; - -import cn.hutool.core.util.StrUtil; -import com.alibaba.fastjson.JSONObject; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.config.TrainJobConfig; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.PtTrainJobMapper; -import org.dubhe.domain.dto.BaseTrainJobDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.domain.vo.PtImageAndAlgorithmVO; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.ModelResourceEnum; -import org.dubhe.enums.ResourcesPoolTypeEnum; -import org.dubhe.enums.TrainJobStatusEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.DistributeTrainApi; -import org.dubhe.k8s.api.NamespaceApi; -import org.dubhe.k8s.api.TrainJobApi; -import org.dubhe.k8s.domain.bo.DistributeTrainBO; -import org.dubhe.k8s.domain.bo.PtJupyterJobBO; -import org.dubhe.k8s.domain.resource.BizDistributeTrain; -import org.dubhe.k8s.domain.resource.BizNamespace; -import org.dubhe.k8s.domain.vo.PtJupyterJobVO; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.LocalFileUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NfsUtil; -import org.dubhe.utils.StringUtils; -import org.dubhe.utils.TrainUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -/** - * @description 提交训练任务 - * @date 2020-07-17 - */ -@Component -public class TrainJobAsync { - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private NamespaceApi namespaceApi; - - @Autowired - private TrainJobConfig trainJobConfig; - - @Autowired - private NfsUtil nfsUtil; - - @Autowired - private LocalFileUtil localFileUtil; - - @Autowired - private PtTrainJobMapper ptTrainJobMapper; - - @Autowired - private TrainJobApi trainJobApi; - - @Autowired - private DistributeTrainApi distributeTrainApi; - - public static final String DATASET_VOLUME_MOUNTS = "/dataset"; - - public static final String WORKSPACE_VOLUME_MOUNTS = "/workspace"; - - public static final String MODEL_VOLUME_MOUNTS = "/model"; - - - /** - * 提交分布式训练 - * - * @param baseTrainJobDTO 训练任务信息 - * @param currentUser 用户 - * @param ptImageAndAlgorithmVO 镜像和算法信息 - * @param ptTrainJob 训练任务实体信息 - */ - public void doDistributedJob(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob) { - try { - //判断是否存在相应的namespace,如果没有则创建 - String namespace = getNamespace(currentUser); - // 构建DistributeTrainBO - DistributeTrainBO bo = buildDistributeTrainBO(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob, namespace); - if (null == bo) { - LogUtil.error(LogEnum.BIZ_TRAIN, "user{}create TrainJob，Encapsulating ptjupyterjobbo object is empty，the received parameters namespace：{}", currentUser.getId(), namespace); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, "", false); - return; - } - // 调度K8s - BizDistributeTrain bizDistributeTrain = distributeTrainApi.create(bo); - if (bizDistributeTrain.isSuccess()) { - // 调度成功 - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, bizDistributeTrain.getName(), true); - } else { - // 调度失败 - LogUtil.error(LogEnum.BIZ_TRAIN, "distributeTrainApi.create FAILED! {}", bizDistributeTrain); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, bizDistributeTrain.getName(), false); - } - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "doDistributedJob ERROR！{} ", e); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, "", false); - } - } - - /** - * 构造分布式训练DistributeTrainBO(炼知模型暂不支持) - * - * @param baseTrainJobDTO 训练任务信息 - * @param currentUser 用户 - * @param ptImageAndAlgorithmVO 镜像和算法信息 - * @param ptTrainJob 训练任务实体信息 - * @param namespace 命名空间 - * @return DistributeTrainBO - */ - private DistributeTrainBO buildDistributeTrainBO(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob, String namespace) { - //绝对路径 - String basePath = nfsUtil.getNfsConfig().getBucket() + trainJobConfig.getManage() + StrUtil.SLASH - + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); - //相对路径 - String relativePath = StrUtil.SLASH + trainJobConfig.getManage() + StrUtil.SLASH - + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); - String[] codeDirArray = ptImageAndAlgorithmVO.getCodeDir().split(StrUtil.SLASH); - String workspaceDir = codeDirArray[codeDirArray.length - 1]; - // 算法路径待拷贝的地址 - String sourcePath = nfsUtil.getNfsConfig().getBucket() + ptImageAndAlgorithmVO.getCodeDir().substring(1); - String trainDir = basePath.substring(1) + StrUtil.SLASH + workspaceDir; - - if (!localFileUtil.copyPath(sourcePath, trainDir)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "buildDistributeTrainBO copyPath failed ! sourcePath:{},basePath:{},trainDir:{}", sourcePath, basePath, trainDir); - return null; - } - // 参数前缀 - String paramPrefix = trainJobConfig.getPythonFormat(); - // 初始化固定头命令，获取分布式节点IP - StringBuilder sb = new StringBuilder("export NODE_IPS=`cat /home/hostfile.json |jq -r \".[]|.ip\"|paste -d \",\" -s` "); - // 切换到算法路径下 - sb.append(" && cd ").append(trainJobConfig.getDockerTrainPath()).append(StrUtil.SLASH).append(workspaceDir).append(" && "); - // 拼接用户自定义python启动命令 - sb.append(ptImageAndAlgorithmVO.getRunCommand()); - // 拼接python固定参数 节点IP - sb.append(paramPrefix).append(trainJobConfig.getNodeIps()).append("=\"$NODE_IPS\" "); - // 拼接python固定参数 节点数量 - sb.append(paramPrefix).append(trainJobConfig.getNodeNum()).append(SymbolConstant.FLAG_EQUAL).append(ptTrainJob.getResourcesPoolNode()).append(StrUtil.SPACE); - if (ptImageAndAlgorithmVO.getIsTrainOut()) { - // 拼接 out - nfsUtil.createDir(basePath + StrUtil.SLASH + trainJobConfig.getOutPath()); - baseTrainJobDTO.setOutPath(relativePath + StrUtil.SLASH + trainJobConfig.getOutPath()); - sb.append(paramPrefix).append(trainJobConfig.getDockerOutPath()); - } - if (ptImageAndAlgorithmVO.getIsTrainLog()) { - // 拼接 输出日志 - nfsUtil.createDir(basePath + StrUtil.SLASH + trainJobConfig.getLogPath()); - baseTrainJobDTO.setLogPath(relativePath + StrUtil.SLASH + trainJobConfig.getLogPath()); - sb.append(paramPrefix).append(trainJobConfig.getDockerLogPath()); - } - if (ptImageAndAlgorithmVO.getIsVisualizedLog()) { - // 拼接 输出可视化日志 - nfsUtil.createDir(basePath + StrUtil.SLASH + trainJobConfig.getVisualizedLogPath()); - baseTrainJobDTO.setVisualizedLogPath(relativePath + StrUtil.SLASH + trainJobConfig.getVisualizedLogPath()); - sb.append(paramPrefix).append(trainJobConfig.getDockerVisualizedLogPath()); - } - // 拼接python固定参数 数据集 - sb.append(paramPrefix).append(trainJobConfig.getDockerDataset()); - - // 模型路径挂载及其参数拼接 - DistributeTrainBO distributeTrainBO = new DistributeTrainBO(); - buildBoAboutModel(baseTrainJobDTO, distributeTrainBO, sb); - - JSONObject runParams = baseTrainJobDTO.getRunParams(); - if (null != runParams && !runParams.isEmpty()) { - // 拼接用户自定义参数 - runParams.entrySet().forEach(entry -> - sb.append(paramPrefix).append(entry.getKey()).append(SymbolConstant.FLAG_EQUAL).append(entry.getValue()).append(StrUtil.SPACE) - ); - } - // 在用户自定以参数拼接晚后拼接固定参数，防止被用户自定义参数覆盖 - if (ResourcesPoolTypeEnum.isGpuCode(baseTrainJobDTO.getPtTrainJobSpecs().getResourcesPoolType())) { - // 需要GPU - sb.append(paramPrefix).append(trainJobConfig.getGpuNumPerNode()).append(SymbolConstant.FLAG_EQUAL).append(baseTrainJobDTO.getGpuNumPerNode()).append(StrUtil.SPACE); - } - String mainCommand = sb.toString(); - // 拼接辅助日志打印 - String wholeCommand = " echo 'Distribute training mission begins... " - + mainCommand - + " ' && " - + mainCommand - + " && echo 'Distribute training mission is over' "; - distributeTrainBO - .setNamespace(namespace) - .setName(baseTrainJobDTO.getJobName()) - .setSize(ptTrainJob.getResourcesPoolNode()) - .setImage(ptImageAndAlgorithmVO.getImageName()) - .setMasterCmd(wholeCommand) - .setMemNum(baseTrainJobDTO.getMenNum()) - .setCpuNum(baseTrainJobDTO.getCpuNum()) - .putNfsMounts(DATASET_VOLUME_MOUNTS,k8sNameTool.getAbsoluteNfsPath(baseTrainJobDTO.getDataSourcePath())) - .putNfsMounts(WORKSPACE_VOLUME_MOUNTS,localFileUtil.formatPath(nfsUtil.getNfsConfig().getRootDir() + basePath)) - .putNfsMounts(MODEL_VOLUME_MOUNTS,k8sNameTool.getAbsoluteNfsPath(relativePath + StrUtil.SLASH + trainJobConfig.getOutPath())) - .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.ALGORITHM)); - //延时启动，单位为分钟 - if (baseTrainJobDTO.getDelayCreateTime() != null && baseTrainJobDTO.getDelayCreateTime() > 0) { - distributeTrainBO.setDelayCreateTime(baseTrainJobDTO.getDelayCreateTime() * MagicNumConstant.SIXTY); - } - //定时停止，单位为分钟 - if (baseTrainJobDTO.getDelayDeleteTime() != null && baseTrainJobDTO.getDelayDeleteTime() > 0) { - distributeTrainBO.setDelayDeleteTime(baseTrainJobDTO.getDelayDeleteTime() * MagicNumConstant.SIXTY); - } - if (ResourcesPoolTypeEnum.isGpuCode(baseTrainJobDTO.getPtTrainJobSpecs().getResourcesPoolType())) { - // 需要GPU - distributeTrainBO.setGpuNum(baseTrainJobDTO.getGpuNumPerNode()); - } - // 主从一致 - distributeTrainBO.setSlaveCmd(distributeTrainBO.getMasterCmd()); - return distributeTrainBO; - } - - - /** - * 提交job - * - * @param baseTrainJobDTO 训练任务信息 - * @param currentUser 用户 - * @param ptImageAndAlgorithmVO 镜像和算法信息 - */ - public void doJob(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob) { - PtJupyterJobBO jobBo = null; - String k8sJobName = ""; - try { - //判断是否存在相应的namespace,如果没有则创建 - String namespace = getNamespace(currentUser); - - //封装PtJupyterJobBO对象,调用创建训练任务接口 - jobBo = pkgPtJupyterJobBo(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, namespace); - if (null == jobBo) { - LogUtil.error(LogEnum.BIZ_TRAIN, "user {} create TrainJob，Encapsulating ptjupyterjobbo object is empty，the received parameters namespace：{}", currentUser.getId(), namespace); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, false); - } - PtJupyterJobVO ptJupyterJobResult = trainJobApi.create(jobBo); - if (!ptJupyterJobResult.isSuccess()) { - String message = null == ptJupyterJobResult.getMessage() ? "未知的错误" : ptJupyterJobResult.getMessage(); - LogUtil.error(LogEnum.BIZ_TRAIN, "user {} create TrainJob, K8s creation failed, the received parameters are {}, the wrong information is{}", currentUser.getUsername(), jobBo, message); - ptTrainJob.setTrainMsg(message); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, false); - } - k8sJobName = ptJupyterJobResult.getName(); - //更新训练任务状态 - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, true); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "user {} create TrainJob, K8s creation failed, the received parameters are {}, the wrong information is{}", currentUser.getUsername(), - jobBo, e); - ptTrainJob.setTrainMsg("内部错误"); - updateTrainStatus(currentUser, ptTrainJob, baseTrainJobDTO, k8sJobName, false); - } - } - - - /** - * 获取namespace - * - * @param currentUser 用户 - * @return String 命名空间 - */ - private String getNamespace(UserDTO currentUser) { - String namespaceStr = k8sNameTool.generateNamespace(currentUser.getId()); - BizNamespace bizNamespace = namespaceApi.get(namespaceStr); - if (null == bizNamespace) { - BizNamespace namespace = namespaceApi.create(namespaceStr, null); - if (null == namespace || !namespace.isSuccess()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to create namespace during training job..."); - throw new BusinessException("内部错误"); - } - } - return namespaceStr; - } - - /** - * 封装出创建job所需的BO - * - * @param baseTrainJobDTO 训练任务信息 - * @param ptImageAndAlgorithmVO 镜像和算法信息 - * @param namespace 命名空间 - * @return PtJupyterJobBO jupyter任务BO - */ - private PtJupyterJobBO pkgPtJupyterJobBo(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, - PtImageAndAlgorithmVO ptImageAndAlgorithmVO, String namespace) { - - //绝对路径 - String commonPath = nfsUtil.getNfsConfig().getBucket() + trainJobConfig.getManage() + StrUtil.SLASH - + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); - //相对路径 - String relativeCommonPath = StrUtil.SLASH + trainJobConfig.getManage() + StrUtil.SLASH - + currentUser.getId() + StrUtil.SLASH + baseTrainJobDTO.getJobName(); - String[] codeDirArray = ptImageAndAlgorithmVO.getCodeDir().split(StrUtil.SLASH); - String workspaceDir = codeDirArray[codeDirArray.length - 1]; - // 算法路径待拷贝的地址 - String sourcePath = nfsUtil.getNfsConfig().getBucket() + ptImageAndAlgorithmVO.getCodeDir().substring(1); - String trainDir = commonPath.substring(1) + StrUtil.SLASH + workspaceDir; - LogUtil.info(LogEnum.BIZ_TRAIN, "Algorithm path copy::sourcePath:{},commonPath:{},trainDir:{}", sourcePath, commonPath, trainDir); - boolean bool = localFileUtil.copyPath(sourcePath.substring(1), trainDir); - if (!bool) { - LogUtil.error(LogEnum.BIZ_TRAIN, "During the process of user {} creating training Job and encapsulating k8s creating job interface parameters, it failed to copy algorithm directory {} to the specified directory {}", currentUser.getUsername(), sourcePath.substring(1), - trainDir); - return null; - } - - List list = new ArrayList<>(); - PtJupyterJobBO jobBo = new PtJupyterJobBO(); - JSONObject runParams = baseTrainJobDTO.getRunParams(); - - StringBuilder sb = new StringBuilder(); - sb.append(ptImageAndAlgorithmVO.getRunCommand()); - // 拼接out,log和dataset - String pattern = trainJobConfig.getPythonFormat(); - if (ptImageAndAlgorithmVO.getIsTrainOut()) { - nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConfig.getOutPath()); - baseTrainJobDTO.setOutPath(relativeCommonPath + StrUtil.SLASH + trainJobConfig.getOutPath()); - sb.append(pattern).append(trainJobConfig.getDockerOutPath()); - } - if (ptImageAndAlgorithmVO.getIsTrainLog()) { - nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConfig.getLogPath()); - baseTrainJobDTO.setLogPath(relativeCommonPath + StrUtil.SLASH + trainJobConfig.getLogPath()); - sb.append(pattern).append(trainJobConfig.getDockerLogPath()); - } - if (ptImageAndAlgorithmVO.getIsVisualizedLog()) { - nfsUtil.createDir(commonPath + StrUtil.SLASH + trainJobConfig.getVisualizedLogPath()); - baseTrainJobDTO.setVisualizedLogPath(relativeCommonPath + StrUtil.SLASH + trainJobConfig.getVisualizedLogPath()); - sb.append(pattern).append(trainJobConfig.getDockerVisualizedLogPath()); - } - sb.append(pattern).append(trainJobConfig.getDockerDataset()); - - String valDataSourcePath = baseTrainJobDTO.getValDataSourcePath(); - if (StringUtils.isNotBlank(valDataSourcePath)) { - sb.append(pattern).append(trainJobConfig.getLoadValDatasetKey()).append(SymbolConstant.FLAG_EQUAL).append(trainJobConfig.getDockerValDatasetPath()); - } - //模型路径挂载及其参数拼接 - buildBoAboutModel(baseTrainJobDTO, jobBo, sb); - - if (null != runParams && !runParams.isEmpty()) { - runParams.forEach((k, v) -> - sb.append(pattern).append(k).append(SymbolConstant.FLAG_EQUAL).append(v).append(StrUtil.SPACE) - ); - } - // 在用户自定以参数拼接晚后拼接固定参数，防止被用户自定义参数覆盖 - if (ResourcesPoolTypeEnum.isGpuCode(baseTrainJobDTO.getPtTrainJobSpecs().getResourcesPoolType())) { - // 需要GPU - sb.append(pattern).append(trainJobConfig.getGpuNumPerNode()).append(SymbolConstant.FLAG_EQUAL).append(baseTrainJobDTO.getGpuNumPerNode()).append(StrUtil.SPACE); - } - String executeCmd = sb.toString(); - list.add("-c"); - - String workPath = trainJobConfig.getDockerTrainPath() + StrUtil.SLASH + workspaceDir; - String command; - Integer modelResource = baseTrainJobDTO.getModelResource(); - if(null != modelResource && modelResource.intValue() == ModelResourceEnum.ATLAS.getType().intValue()) { - command = "&& " + trainJobConfig.getAtlasAnaconda() + - " && cd " + workPath + - " && " + trainJobConfig.getAtlasPythonioencoding() + executeCmd; - } else { - command = " && cd " + workPath + " && " + executeCmd; - } - command = "echo 'training mission begins... " + executeCmd + "\r\n '" + command + " && echo 'the training mission is over' "; - - list.add(command); - - jobBo.setNamespace(namespace) - .setName(baseTrainJobDTO.getJobName()) - .setImage(ptImageAndAlgorithmVO.getImageName()) - .putNfsMounts(trainJobConfig.getDockerDatasetPath(), nfsUtil.getNfsConfig().getRootDir() + nfsUtil.getNfsConfig().getBucket().substring(1) + baseTrainJobDTO.getDataSourcePath()) - .setCmdLines(list) - .putNfsMounts(trainJobConfig.getDockerTrainPath(), nfsUtil.getNfsConfig().getRootDir() + commonPath.substring(1)) - .putNfsMounts(trainJobConfig.getDockerValDatasetPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(valDataSourcePath))) - .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.ALGORITHM)); - //延时启动，单位为分钟 - if (baseTrainJobDTO.getDelayCreateTime() != null && baseTrainJobDTO.getDelayCreateTime() > 0) { - jobBo.setDelayCreateTime(baseTrainJobDTO.getDelayCreateTime() * MagicNumConstant.SIXTY); - } - //自动停止，单位为分钟 - if (baseTrainJobDTO.getDelayDeleteTime() != null && baseTrainJobDTO.getDelayDeleteTime() > 0) { - jobBo.setDelayDeleteTime(baseTrainJobDTO.getDelayDeleteTime() * MagicNumConstant.SIXTY); - } - jobBo.setCpuNum(baseTrainJobDTO.getCpuNum()).setMemNum(baseTrainJobDTO.getMenNum()); - if (ResourcesPoolTypeEnum.isGpuCode(baseTrainJobDTO.getPtTrainJobSpecs().getResourcesPoolType())) { - jobBo.setUseGpu(true).setGpuNum(baseTrainJobDTO.getGpuNumPerNode()); - } else { - jobBo.setUseGpu(false); - } - return jobBo; - } - - /** - * 模型路径挂载及其参数拼接 - * - * @param baseTrainJobDTO 训练任务基本信息 - * @param jobBo 训练任务实体 - * @param sb 训练命令参数 - */ - private void buildBoAboutModel(BaseTrainJobDTO baseTrainJobDTO, Object jobBo, StringBuilder sb) { - if(null == baseTrainJobDTO.getModelResource()) { - return; - } - String modelLoadPathDir = baseTrainJobDTO.getModelLoadPathDir(); - //非炼知模型 - if (StringUtils.isNotBlank(modelLoadPathDir)) { - //将模型路径model_load_dir路径 - sb.append(trainJobConfig.getPythonFormat()).append(trainJobConfig.getLoadKey()).append(SymbolConstant.FLAG_EQUAL).append(trainJobConfig.getDockerModelPath()); - if(jobBo instanceof PtJupyterJobBO) { - PtJupyterJobBO ptJupyterJobBO = (PtJupyterJobBO)jobBo; - ptJupyterJobBO.putNfsMounts(trainJobConfig.getDockerModelPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelLoadPathDir))); - } else if(jobBo instanceof DistributeTrainBO) { - DistributeTrainBO distributeTrainBO = (DistributeTrainBO)jobBo; - distributeTrainBO.putNfsMounts(trainJobConfig.getDockerModelPath(), nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelLoadPathDir))); - } - return; - } - //炼知模型中的教师模型 - appendAtlasModelPath(baseTrainJobDTO.getTeacherModelPathList(), jobBo, sb, true); - //炼知模型中的学生模型 - appendAtlasModelPath(baseTrainJobDTO.getStudentModelPathList(), jobBo, sb, false); - } - - /** - * 炼知模型路径挂载及其参数拼接 - * - * @param modelPathList 模型路径集合 - * @param jobBo 训练任务实体 - * @param sb 训练命令参数 - * @param isTeacher 是否教师模型 - */ - private void appendAtlasModelPath(List modelPathList, Object jobBo, StringBuilder sb, boolean isTeacher) { - if(null == modelPathList || modelPathList.isEmpty()) { - return; - } - StringBuilder appendModelPath = new StringBuilder(); - String preModelKey; - String preModelPath; - if(isTeacher){ - preModelKey = trainJobConfig.getDockerTeacherModelKey(); - preModelPath = trainJobConfig.getDockerTeacherModelPath(); - } else { - preModelKey = trainJobConfig.getDockerStudentModelKey(); - preModelPath = trainJobConfig.getDockerStudentModelPath(); - } - modelPathList.stream() - .forEach(modelPath -> { - String[] urlArray = modelPath.split(SymbolConstant.SLASH); - String dockerModelPath = urlArray[urlArray.length - TrainUtil.NUMBER_ONE]; - String mountPath = preModelPath + SymbolConstant.SLASH + dockerModelPath; - appendModelPath.append(mountPath).append(SymbolConstant.COMMA); - if(jobBo instanceof PtJupyterJobBO) { - PtJupyterJobBO ptJupyterJobBO = (PtJupyterJobBO)jobBo; - ptJupyterJobBO.putNfsMounts(mountPath, nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelPath))); - } else if(jobBo instanceof DistributeTrainBO) { - DistributeTrainBO distributeTrainBO = (DistributeTrainBO)jobBo; - distributeTrainBO.putNfsMounts(mountPath, nfsUtil.formatPath(nfsUtil.getAbsolutePath(modelPath))); - } - }); - String resultPath = SymbolConstant.MARK + - appendModelPath.toString().substring(TrainUtil.NUMBER_ZERO, appendModelPath.toString().length() - TrainUtil.NUMBER_ONE) + - SymbolConstant.MARK; - - sb.append(trainJobConfig.getPythonFormat()).append(preModelKey).append(SymbolConstant.FLAG_EQUAL).append(resultPath); - } - - /** - * 训练任务异步处理更新训练状态 - * - * @param user 用户 - * @param ptTrainJob 训练任务 - * @param baseTrainJobDTO 训练任务信息 - * @param k8sJobName k8s创建的job名称，或者分布式训练名称 - * @param flag 创建训练任务是否异常(true：正常，false：失败) - **/ - private void updateTrainStatus(UserDTO user, PtTrainJob ptTrainJob, BaseTrainJobDTO baseTrainJobDTO, String k8sJobName, boolean flag) { - - ptTrainJob.setK8sJobName(k8sJobName) - .setOutPath(baseTrainJobDTO.getOutPath()) - .setLogPath(baseTrainJobDTO.getLogPath()) - .setVisualizedLogPath(baseTrainJobDTO.getVisualizedLogPath()); - LogUtil.info(LogEnum.BIZ_TRAIN, "user {} training tasks are processed asynchronously to update training status，receiving parameters:{}", user.getId(), ptTrainJob); - if (flag) { - ptTrainJobMapper.updateById(ptTrainJob); - } else { - ptTrainJob.setTrainStatus(TrainJobStatusEnum.CREATE_FAILED.getStatus()); - //训练任务创建失败 - ptTrainJobMapper.updateById(ptTrainJob); - } - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java deleted file mode 100644 index b7e4fa5..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/async/TransactionAsyncManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.async; - -import org.dubhe.aspect.LogAspect; -import org.dubhe.base.DataContext; -import org.dubhe.domain.dto.BaseTrainJobDTO; -import org.dubhe.domain.dto.CommonPermissionDataDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.domain.vo.PtImageAndAlgorithmVO; -import org.dubhe.enums.TrainTypeEnum; -import org.slf4j.MDC; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.transaction.support.TransactionSynchronizationAdapter; -import org.springframework.transaction.support.TransactionSynchronizationManager; - -import javax.annotation.Resource; -import java.util.concurrent.Executor; - -/** - * @description 事务提交后触发异步操作 - * @date 2020-07-17 - */ -@Component -public class TransactionAsyncManager { - - @Autowired - private TrainJobAsync trainJobAsync; - - @Resource(name = "trainExecutor") - private Executor trainExecutor; - - public void execute(BaseTrainJobDTO baseTrainJobDTO, UserDTO currentUser, PtImageAndAlgorithmVO ptImageAndAlgorithmVO, PtTrainJob ptTrainJob) { - String traceId = MDC.get(LogAspect.TRACE_ID); - CommonPermissionDataDTO commonPermissionDataDTO = DataContext.get(); - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() { - @Override - public void afterCommit() { - - trainExecutor.execute(() -> { - MDC.put(LogAspect.TRACE_ID, traceId); - DataContext.set(commonPermissionDataDTO); - - if (TrainTypeEnum.isDistributeTrain(ptTrainJob.getTrainType())) { - // 分布式训练 - trainJobAsync.doDistributedJob(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); - } else { - // 普通训练 - trainJobAsync.doJob(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); - } - MDC.remove(LogAspect.TRACE_ID); - DataContext.remove(); - }); - } - }); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java deleted file mode 100644 index 9d14108..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/GlobalFilter.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import com.alibaba.fastjson.JSON; -import org.dubhe.constant.StringConstant; -import org.dubhe.constatnts.UserConstant; -import org.dubhe.dto.GlobalRequestRecordDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.StringUtils; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Component; - -import javax.servlet.*; -import javax.servlet.annotation.WebFilter; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; - -import static org.dubhe.constant.StringConstant.K8S_CALLBACK_URI; - -/** - * @description 全局请求拦截器 用于日志收集 - * @date 2020-08-13 - */ -@Order(1) -@Component -@WebFilter(filterName = "GlobalFilter", urlPatterns = "/**") -public class GlobalFilter implements Filter { - @Override - public void init(FilterConfig filterConfig) throws ServletException { - - } - - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain chain) throws IOException, ServletException { - - long start = System.currentTimeMillis(); - HttpServletRequest request = ((HttpServletRequest) servletRequest); - HttpServletResponse response = ((HttpServletResponse) servletResponse); - - GlobalRequestRecordDTO dto = new GlobalRequestRecordDTO(); - - try { - if (StringUtils.isNotBlank(request.getContentType()) && request.getContentType().contains(StringConstant.MULTIPART)) { - chain.doFilter(request, response); - } else { - checkScheduleRequest(request); - checkK8sCallback(request); - - RequestBodyWrapper requestBodyWrapper = new RequestBodyWrapper(request); - ResponseBodyWrapper responseBodyWrapper = new ResponseBodyWrapper(response); - dto.setRequestBody(requestBodyWrapper.getBodyString()); - chain.doFilter(requestBodyWrapper, responseBodyWrapper); - - if (StringConstant.JSON_REQUEST.equals(responseBodyWrapper.getContentType())) { - final String responseBody = responseBodyWrapper.getResponseBody(); - dto.setResponseBody(responseBody); - } else { - responseBodyWrapper.flush(); - } - } - } catch (Exception e) { - LogUtil.error(LogEnum.GLOBAL_REQ, "Global request record error : {}", e); - throw e; - } finally { - buildGlobalRequestDTO(dto, request, response); - dto.setTimeCost(System.currentTimeMillis() - start); - LogUtil.info(LogEnum.GLOBAL_REQ, "Global request record: {}", dto); - LogUtil.cleanTrace(); - } - } - - /** - * 构建全局请求对象 - * - * @param dto - * @param request - * @param response - */ - private void buildGlobalRequestDTO(GlobalRequestRecordDTO dto, HttpServletRequest request, HttpServletResponse response) { - dto.setClientHost(request.getRemoteHost()); - dto.setParams(JSON.toJSONString(request.getParameterMap())); - dto.setMethod(request.getMethod()); - dto.setUri(request.getRequestURI()); - //身份认证信息 - String token = request.getHeader(UserConstant.USER_TOKEN_KEY); - dto.setAuthorization(token); - if (token != null) { - String userName = JwtUtils.getUserName(token); - dto.setUsername(userName); - } - dto.setContentType(response.getContentType()); - dto.setStatus(response.getStatus()); - } - - /** - * 检查是否是前端的定时请求 - * - * @param request 请求信息 - * @return 是否是前端的定时请求 - */ - private boolean checkScheduleRequest(HttpServletRequest request) { - - if (StringConstant.REQUEST_METHOD_GET.equals(request.getMethod()) - && StringUtils.isNotBlank(request.getParameter(LogUtil.SCHEDULE_LEVEL))) { - LogUtil.startScheduleTrace(); - return true; - } - - return false; - } - - /** - * 校验请求是否为k8s回调 - * @param request 请求信息 - * @return 是否为k8s回调 - */ - private boolean checkK8sCallback(HttpServletRequest request) { - if (request.getRequestURI() != null && request.getRequestURI().contains(K8S_CALLBACK_URI)) { - LogUtil.startK8sCallbackTrace(); - return true; - } - return false; - } - - @Override - public void destroy() { - - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java deleted file mode 100644 index 954f620..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/RequestBodyWrapper.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.LogUtil; - -import javax.servlet.ReadListener; -import javax.servlet.ServletInputStream; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -/** - * @description 用于获取请求body参数的包装类 - * @date 2020-08-20 - */ -public class RequestBodyWrapper extends HttpServletRequestWrapper { - - private ByteArrayInputStream byteArrayInputStream; - - private String bodyString; - - public RequestBodyWrapper(HttpServletRequest request) - throws IOException { - super(request); - try (BufferedReader reader = request.getReader()) { - StringBuilder sb = new StringBuilder(); - String line = null; - while ((line = reader.readLine()) != null) { - sb.append(line); - } - if (sb.length() > 0) { - bodyString = sb.toString(); - } else { - bodyString = ""; - } - byteArrayInputStream = new ByteArrayInputStream(bodyString.getBytes()); - } catch (Exception e) { - LogUtil.error(LogEnum.GLOBAL_REQ, "request get reader error : {}", e); - throw e; - } - - } - - /** - * 获取请求体的json数据 - * - * @return - */ - public String getBodyString() { - return bodyString; - } - - @Override - public BufferedReader getReader() throws IOException { - return new BufferedReader(new InputStreamReader(getInputStream())); - } - - @Override - public ServletInputStream getInputStream() throws IOException { - return new ServletInputStream() { - @Override - public boolean isFinished() { - return false; - } - - @Override - public boolean isReady() { - return false; - } - - @Override - public void setReadListener(ReadListener readListener) { - } - - @Override - public int read() throws IOException { - return byteArrayInputStream.read(); - } - }; - } -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java deleted file mode 100644 index 6dc8eba..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/ResponseBodyWrapper.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.dubhe.enums.LogEnum; -import org.dubhe.utils.LogUtil; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; - -import javax.servlet.ServletOutputStream; -import javax.servlet.WriteListener; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpServletResponseWrapper; - -/** - * @description 用于获取response 的 json 返回值 - * @date 2020-08-19 - */ -public class ResponseBodyWrapper extends HttpServletResponseWrapper { - - private ByteArrayOutputStream byteArrayOutputStream = null; - - private ServletOutputStream servletOutputStream = null; - - private PrintWriter printWriter = null; - - private HttpServletResponse response; - - public ResponseBodyWrapper(HttpServletResponse response) throws IOException { - super(response); - this.response = response; - byteArrayOutputStream = new ByteArrayOutputStream(); - printWriter = new PrintWriter(new OutputStreamWriter(byteArrayOutputStream, "UTF-8")); - servletOutputStream = new ServletOutputStream() { - - @Override - public void write(int b) throws IOException { - byteArrayOutputStream.write(b); - } - - @Override - public boolean isReady() { - return false; - } - - @Override - public void setWriteListener(WriteListener writeListener) { - } - }; - } - - @Override - public ServletOutputStream getOutputStream() throws IOException { - return servletOutputStream; - } - - @Override - public PrintWriter getWriter() throws IOException { - return printWriter; - } - - @Override - public void flushBuffer() throws IOException { - if (servletOutputStream != null) { - servletOutputStream.flush(); - } - if (printWriter != null) { - printWriter.flush(); - } - } - - @Override - public void reset() { - byteArrayOutputStream.reset(); - } - - /** - * 获取json返回值 - * @return - * @throws IOException - */ - public String getResponseBody() throws IOException { - //清空response的流，之后再添加进去 - flushBuffer(); - byte[] bytes = byteArrayOutputStream.toByteArray(); - try { - return new String(bytes, "UTF-8"); - } catch (UnsupportedEncodingException e) { - LogUtil.error(LogEnum.GLOBAL_REQ, e); - } finally { - response.getOutputStream().write(bytes); - } - return ""; - } - - /** - * 清掉缓冲 - * @throws IOException - */ - public void flush() throws IOException { - //清空response的流，之后再添加进去 - flushBuffer(); - byte[] bytes = byteArrayOutputStream.toByteArray(); - response.getOutputStream().write(bytes); - } - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java deleted file mode 100644 index 679a027..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/config/TimestampConverter.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.config; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.core.convert.converter.Converter; -import org.springframework.stereotype.Component; - -import java.sql.Timestamp; - -/** - * @description 转换时间戳类型 - * @date 2020-05-22 - */ -@Component -public class TimestampConverter implements Converter { - - /** - * Convert string to Timestamp type - * @param timeStr - * @return - */ - @Override - public Timestamp convert(String timeStr) { - - Timestamp timestamp = null; - if (StringUtils.isNotBlank(timeStr)) { - long time = Long.valueOf(timeStr); - timestamp = new Timestamp(time); - } - - return timestamp; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java deleted file mode 100644 index 3afe75b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptBuiltInMapper.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Param; -import org.dubhe.domain.entity.ModelOptBuiltIn; -import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; -import org.dubhe.domain.vo.ModelOptDatasetQueryVO; -import org.dubhe.domain.vo.ModelOptModelQueryVO; - -import java.util.List; - -/** - * @description 内置模型 - * @date 2020-05-22 - */ -public interface ModelOptBuiltInMapper extends BaseMapper { - /** - * 获取模型优化数据集 - * @param type 算法类型 - * @return 模型优化数据集列表 - */ - List getDataset(@Param("type") Integer type, @Param("model") String model, @Param("algorithm") String algorithm); - - /** - * 获取模型优化教师模型 - * @param type - * @return - */ - List getModel(@Param("type") Integer type, @Param("dataset") String dataset, @Param("algorithm") String algorithm); - - /** - * 获取模型优化算法 - * @param type - * @return - */ - List getAlgorithm(@Param("type") Integer type, @Param("model") String model, @Param("dataset") String dataset); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java deleted file mode 100644 index 9537712..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptDatasetMapper.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Select; -import org.dubhe.domain.entity.ModelOptDataset; - -import java.util.List; - -/** - * @description 模型优化数据集 - * @date 2021-01-06 - */ -public interface ModelOptDatasetMapper extends BaseMapper { - - @Select("select id, name, path from model_opt_dataset where deleted = 0 order by id desc") - List getAllDataset(); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java deleted file mode 100644 index 2ec4961..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskInstanceMapper.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Select; -import org.apache.ibatis.annotations.Update; -import org.dubhe.domain.entity.ModelOptTaskInstance; - -import java.util.List; - -/** - * @description 模型优化实例 - * @date 2020-05-22 - */ -public interface ModelOptTaskInstanceMapper extends BaseMapper { - - /** - * 将实例状态标记为执行失败 - * - * @param instId 实例id - * @return 更新行数 - */ - @Update("update model_opt_task_instance set status = '3' where id = #{instId} and status in ('-1','0')") - Integer markInstanceExecFailed(Long instId); - - /** - * 获取5分钟以前状态仍为等待中的实例 - * - * @return 实例列表 - */ - @Select("select * from model_opt_task_instance where status = '-1' and create_time < date_sub(now(), interval 5 minute)") - List selectWaitingFor5MinutesInstances(); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java deleted file mode 100644 index 66c5648..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/ModelOptTaskMapper.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.dubhe.domain.entity.ModelOptTask; - -/** - * @description 模型优化任务 - * @date 2020-05-22 - */ -public interface ModelOptTaskMapper extends BaseMapper { -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java deleted file mode 100644 index 5c1d1a6..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookMapper.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.NoteBook; - -import java.util.List; - -/** - * @description notebook mapper - * @date 2020-04-28 - */ -@DataPermission(ignoresMethod = {"insert","findByNamespaceAndResourceName","selectRunNotUrlList"}) -public interface NoteBookMapper extends BaseMapper { - - /** - * 根据名称查询 - * - * @param name - * @param status - * @return NoteBook - */ - @Select("select * from notebook where notebook_name = #{name} and status != #{status} and deleted = 0 limit 1") - NoteBook findByNameAndStatus(@Param("name") String name, @Param("status") Integer status); - - /** - * 查询正在运行的notebook数量 - * - * @param status - * @return int - */ - @Select("select count(1) from notebook where status = #{status} and deleted = 0") - int selectRunNoteBookNum( @Param("status") Integer status); - - /** - * 根据namespace + resourceName查询 - * - * @param namespace - * @param resourceName - * @param status - * @return NoteBook - */ - @Select("select * from notebook where k8s_namespace = #{namespace} and k8s_resource_name = #{resourceName} and status != #{status} and deleted = 0 limit 1") - NoteBook findByNamespaceAndResourceName(@Param("namespace") String namespace, @Param("resourceName") String resourceName, @Param("status") Integer status); - - /** - * 查询已经运行并且没有URL的notebook - * - * @param page - * @param status - * @return List - */ - @Select("select * from notebook where deleted = 0 and status = #{status} and (url is null or url = '')") - List selectRunNotUrlList(@Param("page") Page page, @Param("status") Integer status); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java deleted file mode 100644 index 280f68b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/NoteBookModelMapper.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Select; -import org.dubhe.domain.entity.NoteBookModel; - -import java.util.List; - -/** - * @description : notebook 模板 Mapper - * @date 2020-06-01 - */ -public interface NoteBookModelMapper extends BaseMapper { - - /** - * 根据名称查询 - * - * @return NoteBookModel - */ - @Select("select * from notebook_model where deleted = 0 order by model_type") - List selectAllNoteBookModel(); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java deleted file mode 100644 index 0136d8e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtImageMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Select; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtImage; - -import java.util.List; - -/** - * @description 镜像 Mapper 接口 - * @date 2020-04-27 - */ -@DataPermission(ignoresMethod = {"insert"}) -public interface PtImageMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java deleted file mode 100644 index 094b95e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtJobParamMapper.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import org.dubhe.domain.entity.PtJobParam; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * @description job运行参数及结果 - * @date 2020-04-27 - */ -public interface PtJobParamMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java deleted file mode 100644 index b716125..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtMeasureMapper.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtMeasure; - -/** - * @description 度量 Mapper 接口 - * @date 2020-11-16 - */ -@DataPermission(ignoresMethod = {"insert"}) -public interface PtMeasureMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java deleted file mode 100644 index d22ce96..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmMapper.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtTrainAlgorithm; - -import java.util.List; - -/** - * @description 训练算法Mapper - * @date 2020-04-27 - */ -@DataPermission(ignoresMethod = {"insert"}) -public interface PtTrainAlgorithmMapper extends BaseMapper { - - /** - * 根据算法id查询算法信息 - * @param id 算法id - * @return PtTrainAlgorithm 算法信息 - */ - @Select("select * from pt_train_algorithm where id= #{id}") - PtTrainAlgorithm selectAllById(@Param("id") Long id); - - /** - * 根据算法id集合查询对应的算法信息 - * @param ids 算法集合id - * @return List 算法信息集合 - */ - @Select({ - "" - }) - List selectAllBatchIds(@Param("ids") List ids); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java deleted file mode 100644 index fa7d656..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainAlgorithmUsageMapper.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtTrainAlgorithmUsage; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * - * @description 用户辅助信息Mapper 接口 - * @date 2020-06-23 - */ -@DataPermission(ignoresMethod = "insert") -public interface PtTrainAlgorithmUsageMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java deleted file mode 100644 index 290db66..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobMapper.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.domain.vo.PtTrainVO; - -/** - * @description 训练作业job Mapper 接口 - * @date 2020-04-27 - */ -@DataPermission(ignoresMethod = {"insert","selectCountByStatus","getPageTrain"}) -public interface PtTrainJobMapper extends BaseMapper { - - /** - * 获取训练列表，并进行分页。 - * - * @param page 页 - * @param createUserId 用户id - * @param trainStatus 训练状态 - * @param trainName 训练名称 - * @param sort 排序字段 - * @param order 排序方式 - * - * @return PtTrainVO - */ - Page getPageTrain(Page page, @Param("createUserId") Long createUserId, - @Param("trainStatus") Integer trainStatus, @Param("trainName") String trainName, @Param("sort") String sort, - @Param("order") String order); - - /** - * 根据状态进行统计数量 - * - * @param userId 当前用户id - * @param param sql片段 - * @return 统计的数量 - */ - @Select("select count(1) from pt_train_job t1 inner join pt_train t2 on t1.train_id = t2.id where t1.create_user_id= #{userId} and t1.train_status in ${param} and t1.deleted= 0 and t2.deleted = 0 ") - Integer selectCountByStatus(@Param("userId") Long userId, @Param("param") String param); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java deleted file mode 100644 index a5dd5ec..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainJobSpecsMapper.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.dubhe.domain.entity.PtTrainJobSpecs; - -/** - * @description 训练作业规格 Mapper 接口 - * @date 2020-05-06 - */ -public interface PtTrainJobSpecsMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java deleted file mode 100644 index 7e043d4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtTrain; - -/** - * @description 训练作业主 Mapper 接口 - * @date 2020-04-27 - */ -@DataPermission(ignoresMethod = {"insert"}) -public interface PtTrainMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java deleted file mode 100644 index 2a9efd1..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/dao/PtTrainParamMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.dao; - -import org.dubhe.annotation.DataPermission; -import org.dubhe.domain.entity.PtTrainParam; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * @description 任务参数 Mapper 接口 - * @date 2020-04-27 - */ -@DataPermission(ignoresMethod = "insert") -public interface PtTrainParamMapper extends BaseMapper { - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java deleted file mode 100644 index 354c8e9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainJobDTO.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.domain.entity.PtTrainJobSpecs; - -import java.io.Serializable; -import java.util.List; - -/** - * @description 创建训练任务的数据包 - * @date 2020-07-15 - */ -@Data -@Accessors(chain = true) -public class BaseTrainJobDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - private JSONObject runParams; - private String jobName; - private String dataSourcePath; - private PtTrainJobSpecs ptTrainJobSpecs; - private String outPath; - private String logPath; - private String visualizedLogPath; - private Integer delayCreateTime; - private Integer delayDeleteTime; - private String modelPath; - private List teacherModelPathList; - private List studentModelPathList; - private Long modelId; - private Long modelBranchId; - private Integer modelResource; - private String teacherModelIds; - private String studentModelIds; - - /** - * @return 每个节点的GPU数量 - */ - public Integer getGpuNumPerNode(){ - return getPtTrainJobSpecs().getSpecsInfo().getInteger("gpuNum"); - } - - /** - * @return cpu数量 - */ - public Integer getCpuNum(){ - return getPtTrainJobSpecs().getSpecsInfo().getInteger("cpuNum"); - } - - /** - * @return memNum - */ - public Integer getMenNum(){ - return getPtTrainJobSpecs().getSpecsInfo().getInteger("memNum"); - } - /** - * "验证数据来源名称" - */ - private String valDataSourceName; - - /** - * 验证数据来源路径 - */ - private String valDataSourcePath; - /** - * 模型路径 - */ - private String modelLoadPathDir; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java deleted file mode 100644 index a3fb87e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/BaseTrainParamDTO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; - -/** - * @description 创建训练模板的数据包 - * @date 2020-07-15 - */ -@Data -@Accessors(chain = true) -public class BaseTrainParamDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - private Long modelId; - private Long modelBranchId; - private Integer modelResource; - private String teacherModelIds; - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java deleted file mode 100644 index 1a5e6e8..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptDatasetCreateDTO.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotEmpty; -import java.io.Serializable; - -/** - * @description 模型优化模块我的数据集创建参数 - * @date 2021-01-06 - */ -@Data -public class ModelOptDatasetCreateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "名称", required = true) - @NotEmpty(message = "数据集名称不能为空") - private String name; - - @ApiModelProperty(value = "路径", required = true) - @NotEmpty(message = "数据集路径不能为空") - private String path; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java deleted file mode 100644 index 53caa33..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptLogQueryDTO.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; - -/** - * @description 模型优化日志查询参数 - * @date 2020-05-29 - */ -@Data -public class ModelOptLogQueryDTO { - - @ApiModelProperty("模型优化实例ID") - @NotNull(message = "模型优化实例ID") - private Long instId; - - @ApiModelProperty(value = "起始行") - @Min(value = 0, message = "startLine不能小于0") - private Integer startLine; - - @ApiModelProperty(value = "行数") - @Min(value = 1, message = "lines不能小于1") - @Max(value = 1000, message = "lines不能大于1000") - private Integer lines; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java deleted file mode 100644 index 8e7e9cd..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskCreateDTO.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import javax.validation.constraints.Size; -import java.io.Serializable; - -/** - * @description 创建模型优化任务 - * @date 2020-05-22 - */ -@Data -@Accessors(chain = true) -public class ModelOptTaskCreateDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "任务名称", required = true) - @NotEmpty(message = "任务名称不能为空") - @Size(max = 32, message = "任务名称长度不能超过32") - @Pattern(regexp = TrainUtil.REGEXP, message = "任务名称仅支持字母、数字、汉字、英文横杠和下划线") - private String name; - - @ApiModelProperty(value = "任务描述") - @Size(max = 500, message = "任务描述长度不能超过500") - private String description; - - @ApiModelProperty(value = "是否内置", required = true) - @NotNull(message = "请选择是否内置") - private Boolean isBuiltIn; - - @ApiModelProperty(value = "数据集id") - private Long datasetId; - - @ApiModelProperty(value = "数据集名称", required = true) - @NotEmpty(message = "数据集不能为空") - private String datasetName; - - @ApiModelProperty(value = "数据集路径", required = true) - @NotEmpty(message = "数据集路径为空") - private String datasetPath; - - @ApiModelProperty(value = "模型id") - private Long modelId; - - @ApiModelProperty(value = "模型名称") - private String modelName; - - @ApiModelProperty(value = "模型路径") - private String modelAddress; - - @ApiModelProperty(value = "优化算法类别") - private Integer algorithmType; - - @ApiModelProperty(value = "优化算法id") - private Long algorithmId; - - @ApiModelProperty(value = "是否编辑") - private Boolean editAlgorithm ; - - @ApiModelProperty(value = "优化算法", required = true) - @NotEmpty(message = "请选择优化算法") - private String algorithmName; - - @ApiModelProperty(value = "算法路径", required = true) - @NotEmpty(message = "算法路径不能为空") - private String algorithmPath; - - @ApiModelProperty(value = "运行命令") - private String command; - - @ApiModelProperty(value = "运行参数") - private JSONObject params; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java deleted file mode 100644 index 18de210..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskDeleteDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 删除模型优化任务参数 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskDeleteDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java deleted file mode 100644 index 4145e2d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceCancelDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 取消模型优化任务参数 - * @date 2020-06-29 - */ -@Data -public class ModelOptTaskInstanceCancelDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "实例id不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java deleted file mode 100644 index 3ff1574..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDeleteDTO.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 删除任务实例对象 - * @date 2021-01-07 - */ -@Data -public class ModelOptTaskInstanceDeleteDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "实例ID") - @NotNull(message = "实例ID不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java deleted file mode 100644 index 9f6bee3..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceDetailDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 查看任务实例详情参数 - * @date 2020-06-29 - */ -@Data -public class ModelOptTaskInstanceDetailDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "实例id不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java deleted file mode 100644 index e1abeed..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceQueryDTO.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.annotation.Query; -import org.dubhe.base.PageQueryBase; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.List; - -/** - * @description 模型优化任务实例查询条件 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskInstanceQueryDTO extends PageQueryBase implements Serializable { - @ApiModelProperty("任务ID") - @Query(propName = "task_id") - private Long taskId; - /** - * 任务状态 - */ - @ApiModelProperty("任务状态") - @Query - private String status; - /** - * 优化算法 - */ - @ApiModelProperty("优化算法") - @Query(propName = "algorithm_name") - private String algorithmName; - /** - * 提交时间 - */ - @ApiModelProperty("提交时间") - @Query(propName = "create_time", type = Query.Type.BETWEEN) - private List createTime; - /** - * 完成时间 - */ - @ApiModelProperty("完成时间") - @Query(propName = "end_time", type = Query.Type.BETWEEN) - private List endTime; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java deleted file mode 100644 index ebac902..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskInstanceResubmitDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 重新提交任务实例参数 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskInstanceResubmitDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "实例id不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java deleted file mode 100644 index 82cda7a..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskQueryDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.annotation.Query; -import org.dubhe.base.PageQueryBase; - -import java.io.Serializable; -import java.sql.Timestamp; -import java.util.List; - -/** - * @description 模型优化任务查询条件 - * @date 2020-05-22 - */ -@Data -@Accessors(chain = true) -public class ModelOptTaskQueryDTO extends PageQueryBase implements Serializable { - @ApiModelProperty("任务id") - @Query(propName = "id") - private Long id; - - @Query(type = Query.Type.LIKE) - @ApiModelProperty("任务名称或id") - private String name; - - @ApiModelProperty("更新时间") - @Query(type = Query.Type.BETWEEN, propName = "update_time") - private List updateTime; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java deleted file mode 100644 index 0417029..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskRunParamDTO.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; -import lombok.experimental.Accessors; - -/** - * @description 模型优化创建资源运行的参数类 - * @date 2020-05-25 - */ -@Data -@Accessors(chain = true) -public class ModelOptTaskRunParamDTO { - - /** - * 实例id - **/ - private Long instId; - /** - * 使用的镜像 - **/ - private String imageUrl; - /** - * 输入目录 - **/ - private String inputDir; - /** - * 输出目录 - **/ - private String outputDir; - /** - * 算法目录 - **/ - private String algorithmDir; - /** - * 日志目录 - **/ - private String logDir; - /** - * 数据集路径 - **/ - private String datasetDir; - /** - * 压缩前结果json目录 - **/ - private String resultJsonBeforeDir; - /** - * 压缩后结果json目录 - **/ - private String resultJsonAfterDir; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java deleted file mode 100644 index 2ff55da..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskSubmitDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 模型优化任务提交参数 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskSubmitDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Long id; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java deleted file mode 100644 index cffc132..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelOptTaskUpdateDTO.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import javax.validation.constraints.Size; -import java.io.Serializable; - -/** - * @description 修改模型优化任务 - * @date 2020-05-22 - */ -@Data -@Accessors(chain = true) -public class ModelOptTaskUpdateDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Long id; - - @ApiModelProperty(value = "任务名称", required = true) - @NotEmpty(message = "任务名称不能为空") - @Size(max = 32, message = "任务名称长度不能超过32") - @Pattern(regexp = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$", message = "任务名称仅支持字母、数字、汉字、英文横杠和下划线") - private String name; - - @ApiModelProperty(value = "任务描述") - @Size(max = 500, message = "任务描述长度不能超过500") - private String description; - - @ApiModelProperty(value = "是否内置", required = true) - @NotNull(message = "请选择是否内置") - private Boolean isBuiltIn; - - @ApiModelProperty(value = "数据集id") - private Long datasetId; - - @ApiModelProperty(value = "数据集名称", required = true) - @NotEmpty(message = "数据集不能为空") - private String datasetName; - - @ApiModelProperty(value = "数据集路径", required = true) - @NotEmpty(message = "数据集路径为空") - private String datasetPath; - - @ApiModelProperty(value = "模型id") - private Long modelId; - - @ApiModelProperty(value = "模型名称") - private String modelName; - - @ApiModelProperty(value = "模型路径") - private String modelAddress; - - @ApiModelProperty(value = "优化算法类型") - private Integer algorithmType; - - @ApiModelProperty(value = "优化算法id") - private Long algorithmId; - - @ApiModelProperty(value = "优化算法", required = true) - @NotEmpty(message = "请选择优化算法") - private String algorithmName; - - @ApiModelProperty(value = "算法路径", required = true) - @NotEmpty(message = "算法路径不能为空") - private String algorithmPath; - - @ApiModelProperty(value = "是否编辑") - private Boolean editAlgorithm ; - - @ApiModelProperty(value = "运行命令") - private String command; - - @ApiModelProperty(value = "运行参数") - private JSONObject params; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java deleted file mode 100644 index c03e128..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/ModelQueryDTO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -/** - * @description model 查询dto - * @date 2020-10-09 - */ -@Data -@ApiModel("模型查询") -public class ModelQueryDTO { - @ApiModelProperty(value = "模型类型") - private Integer modelResource; - @ApiModelProperty(value = "模型名称") - private String name; - @ApiModelProperty(value = "模型版本") - private String version; - @ApiModelProperty(value = "模型id") - private Integer id; - @ApiModelProperty(value = "模型路径") - private String modelPath; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java deleted file mode 100644 index 0f38743..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookCreateDTO.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.utils.NotebookUtil; - -import javax.validation.constraints.*; - -/** - * @description 创建notebook请求体 - * @date 2020-06-28 - */ -@Data -public class NoteBookCreateDTO { - - @NotBlank - @Pattern(regexp = NotebookUtil.K8S_NOTEBOOK_REGEX, message = "Notebook 名称仅支持字母、数字、汉字、英文横杠和下划线") - @Size(max = 30, message = "名称内容超长") - @ApiModelProperty(value = "notebook 名称") - private String noteBookName; - - @Size(max = 255, message = "描述内容超长") - @ApiModelProperty(value = "notebook 描述") - private String description; - - @ApiModelProperty(hidden = true) - private String url; - - @ApiModelProperty(hidden = true) - private Integer totalRunMin; - - @NotNull - @Min(value = NotebookUtil.CPU_MIN_NUMBER, message = "最少需要一个CPU") - @Max(value = NotebookUtil.CPU_MAX_NUMBER, message = "CPU数量超过限制，不可大于8") - @ApiModelProperty(value = "cpu数量") - private Integer cpuNum; - - @Min(value = NotebookUtil.GPU_MIN_NUMBER, message = "最少需要一个GPU") - @Max(value = NotebookUtil.GPU_MAX_NUMBER, message = "GPU数量超过限制，不可大于4") - @ApiModelProperty(value = "gpu数量") - private Integer gpuNum; - - @NotNull - @Min(value = NotebookUtil.MEMORY_MIN_NUMBER, message = "最少需要1G内存") - @Max(value = NotebookUtil.MEMORY_MAX_NUMBER, message = "内存超出限制，不可大于8") - @ApiModelProperty(value = "内存大小") - private Integer memNum; - - @NotNull - @Min(value = NotebookUtil.DISK_MEMORY_MIN_NUMBER, message = "至少需要1G硬盘内存") - @Max(value = NotebookUtil.DISK_MEMORY_MAX_NUMBER, message = "硬盘内存超出限制，不可大于1024") - @ApiModelProperty(value = "硬盘内存大小") - private Integer diskMemNum; - - @ApiModelProperty("镜像名称") - @NotNull(message = "请选择镜像") - private String k8sImageName; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java deleted file mode 100644 index 05457b5..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookListQueryDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.annotation.Query; - -import java.io.Serializable; - -/** - * @description 查询请求体 - * @date 2020-06-28 - */ -@Data -public class NoteBookListQueryDTO implements Serializable { - - @ApiModelProperty("0运行中，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用）") - private Integer status; - - @Query(propName = "notebook_name", type = Query.Type.LIKE) - @ApiModelProperty("notebook名称") - private String noteBookName; - - @Query(propName = "origin_user_id", type = Query.Type.EQ) - @ApiModelProperty(value = "所属用户ID", hidden = true) - private Long userId; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java deleted file mode 100644 index 2543fb8..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookQueryDTO.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; -import org.dubhe.annotation.Query; -import org.dubhe.enums.NoteBookStatusEnum; -import org.dubhe.utils.K8sCallBackTool; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.List; - -/** - * @description 查询notebook请求体 - * @date 2020-04-28 - */ -@Data -public class NoteBookQueryDTO implements Serializable { - - @Query(propName = "status", type = Query.Type.NE) - private Integer neStatus; - - @Query(propName = "status", type = Query.Type.IN) - private List statusList; - - @Query(propName = "k8s_pvc_path", type = Query.Type.EQ) - private String k8sPvcPath; - - @Query(propName = "origin_user_id", type = Query.Type.EQ) - private Long userId; - - @Query(propName = "last_operation_timeout", type = Query.Type.LT) - private Long lastOperationTimeout; - - - private static final NoteBookQueryDTO QUERY_CRITERIA; - - public NoteBookQueryDTO() { - } - - public NoteBookQueryDTO(Integer neStatus, String k8sPvcPath, Long userId) { - this.neStatus = neStatus; - this.k8sPvcPath = k8sPvcPath; - this.userId = userId; - } - - static { - QUERY_CRITERIA = new NoteBookQueryDTO(); - QUERY_CRITERIA.setNeStatus(NoteBookStatusEnum.DELETE.getCode()); - } - - /** - * 获取需要刷新状态的查询条件 - * - * @return - */ - public static NoteBookQueryDTO getToRefreshCriteria() { - NoteBookQueryDTO refreshCriteria = new NoteBookQueryDTO(); - refreshCriteria.setStatusList(Arrays.asList( - NoteBookStatusEnum.STARTING.getCode() - , NoteBookStatusEnum.STOPPING.getCode() - , NoteBookStatusEnum.DELETING.getCode())); - refreshCriteria.setLastOperationTimeout(K8sCallBackTool.getCurrentSecondLong()); - return refreshCriteria; - } - - /** - * 获取查询列表需要的查询条件 - * - * @return - */ - public static NoteBookQueryDTO getListCriteria() { - return QUERY_CRITERIA; - } - - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java deleted file mode 100644 index 228a01e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/NoteBookStatusDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description note book status - * @date 2020-05-14 - */ -@Data -public class NoteBookStatusDTO implements Serializable { - - @ApiModelProperty("状态编码") - private Integer statusCode; - - @ApiModelProperty("状态名称") - private String statusName; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java deleted file mode 100644 index 36fe478..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetDTO.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 数据集 - * @date 2020-03-17 - */ -@Data -public class PtDatasetDTO implements Serializable { - private Long id; - private String name; - private String remark; - private String type; - private TeamSmallDTO team; - /** - * 创建用户 - */ - private UserSmallDTO createUser; - private Timestamp createTime; - private Timestamp updateTime; - private Boolean deleted; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java deleted file mode 100644 index 397579c..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDatasetSmallDTO.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 数据集 - * @date 2020-03-17 - */ -@Data -public class PtDatasetSmallDTO implements Serializable { - private Long id; - private String name; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java deleted file mode 100644 index 3309b18..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsDTO.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 开发环境 - * @date 2020-03-17 - */ -@Data -public class PtDevEnvsDTO implements Serializable { - - private Long id; - private String name; - private String remark; - private String type; - private Integer podNum; - private Integer gpuNum; - private Integer memNum; - private Integer cpuNum; - private Integer duration; - private Timestamp startTime; - private Timestamp closeTime; - private PtDatasetSmallDTO dataset; - private PtImageSmallDTO image; - private PtStorageSmallDTO storage; - private TeamSmallDTO team; - private UserSmallDTO createUser; - private Timestamp createTime; - private Timestamp updateTime; - private Boolean deleted; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java deleted file mode 100644 index 3d7f1bf..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtDevEnvsQueryCriteria.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -/** - * @description 开发环境查询条件 - * @date 2020-03-17 - */ -@Data -public class PtDevEnvsQueryCriteria { -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java deleted file mode 100644 index 3856980..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDTO.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.domain.entity.Team; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description： 镜像 - * @date 2020-03-17 - */ -@Data -public class PtImageDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("镜像ID") - private Long id; - - @ApiModelProperty("镜像名称") - private String name; - - @ApiModelProperty("镜像描述") - private String remark; - - @ApiModelProperty("类型") - private String type; - - @ApiModelProperty("标签") - private String label; - - @ApiModelProperty("地址") - private String uri; - - @ApiModelProperty("团队") - private Team team; - - @ApiModelProperty("创建人") - private UserSmallDTO createUser; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("更新时间") - private Timestamp updateTime; - - @ApiModelProperty("删除(0正常，1已删除)") - private Boolean deleted; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java deleted file mode 100644 index 6faa8d8..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageDeleteDTO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.List; - -/** - * @description 训练镜像删除DTO - * @date 2020-08-13 - */ -@Data -@Accessors(chain = true) -public class PtImageDeleteDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "镜像id不能为空") - private List ids; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java deleted file mode 100644 index a413e01..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryDTO.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.annotation.FlagValidator; -import org.dubhe.base.PageQueryBase; - -import java.io.Serializable; - -/** - * @description 镜像查询 - * @date 2020-05-09 - */ -@Data -@Accessors(chain = true) -public class PtImageQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - - @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") - private Integer imageResource; - - @ApiModelProperty(value = "镜像状态，0为制作中，1位制作成功，2位制作失败") - private Integer imageStatus; - - @ApiModelProperty(value = "镜像名称或id") - private String imageNameOrId; - - @ApiModelProperty(value = "镜像项目类型(0:notebook , 1:train)") - @FlagValidator(value = {"0", "1"}, message = "镜像项目类型不支持") - private Integer projectType; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java deleted file mode 100644 index 6e46601..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageQueryUrlDTO.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.annotation.FlagValidator; - -/** - * @description 查询镜像路径 - * @date 2020-12-14 - */ -@Data -@Accessors(chain = true) -@EqualsAndHashCode -public class PtImageQueryUrlDTO { - - @ApiModelProperty(value = "镜像来源(0为我的镜像, 1为预置镜像)") - private Integer imageResource; - - @ApiModelProperty(value = "镜像名称") - private String imageName; - - @ApiModelProperty(value = "镜像标签") - private String iamgeTag; - - @ApiModelProperty(value = "镜像项目类型(0:notebook , 1:train)") - @FlagValidator(value = {"0", "1"}, message = "镜像项目类型不支持") - private Integer projectType; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java deleted file mode 100644 index 7aaa3d9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageSmallDTO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description： 镜像 - * @date 2020-03-17 - */ -@Data -public class PtImageSmallDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("镜像ID") - private Long id; - - @ApiModelProperty("镜像名称") - private String name; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java deleted file mode 100644 index 047168e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUpdateDTO.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.List; - -/** - * @description 训练镜像信息修改DTO - * @date 2020-08-13 - */ -@Data -@Accessors(chain = true) -public class PtImageUpdateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "镜像id不能为空") - private List ids; - - @ApiModelProperty("镜像描述") - @Length(max = TrainUtil.NUMBER_ONE_THOUSAND_AND_TWENTY_FOUR, message = "镜像描述-输入长度不能超过1024个字符") - private String remark; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java deleted file mode 100644 index f7ccb0f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtImageUploadDTO.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - - -/** - * @description 上传镜像 - * @date 2020-06-11 - */ -@Data -@Accessors(chain = true) -public class PtImageUploadDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "镜像项目类型", required = true) - @NotNull(message = "镜像项目类型不能为空") - private Integer projectType; - - @ApiModelProperty(value = "镜像文件路径", required = true) - @NotBlank(message = "镜像文件路径不能为空") - private String imagePath; - - @ApiModelProperty(value = "镜像名称", required = true) - @NotBlank(message = "源镜像名称不能为空") - @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_SIXTY_FOUR, message = "镜像名称长度在1-64个字符") - @Pattern(regexp = TrainUtil.REGEXP_NAME, message = "镜像名称支持字母、数字、英文横杠和下划线") - private String imageName; - - @ApiModelProperty(value = "镜像版本号", required = true) - @NotBlank(message = "镜像版本号不能为空") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "镜像版本号长度在1-32个字符") - @Pattern(regexp = TrainUtil.REGEXP_TAG, message = "镜像版本号支持字母、数字、英文横杠、英文.号和下划线") - private String imageTag; - - @ApiModelProperty("镜像描述") - @Length(max = TrainUtil.NUMBER_ONE_THOUSAND_AND_TWENTY_FOUR, message = "镜像描述-输入长度不能超过1024个字符") - private String remark; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java deleted file mode 100644 index 83d1e57..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDTO.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotBlank; -import java.io.Serializable; - -/** - * @description 度量管理入参DTO - * @date 2020-11-16 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class PtMeasureDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "度量名称", required = true) - @NotBlank(message = "度量名称不能为空") - private String name; - - @ApiModelProperty("度量描述(选填)") - private String description; - - @ApiModelProperty(value = "度量文件路径", required = true) - @NotBlank(message = "度量文件路径不能为空") - private String url; - - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java deleted file mode 100644 index 843891c..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureDeleteDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.Set; - -/** - * @description 删除度量信息入参DTO - * @date 2020-11-16 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class PtMeasureDeleteDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "参数id,参数id不能小于1", required = true) - @NotNull(message = "度量id不能为空") - private Set ids; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java deleted file mode 100644 index 0ba9e35..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureQueryDTO.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.base.PageQueryBase; - -import java.io.Serializable; - -/** - * @description 度量管理查询接口入参DTO - * @date 2020-11-16 - */ -@Data -public class PtMeasureQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "度量名称", required = true) - private String nameOrId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java deleted file mode 100644 index 6b41b2e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtMeasureUpdateDTO.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 修改度量信息入参DTO - * @date 2020-11-16 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class PtMeasureUpdateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "参数id,参数id不能小于1", required = true) - @NotNull(message = "度量id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "度量参数id不能小于1") - private Long id; - - @ApiModelProperty(value = "度量名称", required = true) - @NotBlank(message = "度量名称不能为空") - private String name; - - @ApiModelProperty("度量描述(选填)") - private String description; - - @ApiModelProperty(value = "度量文件路径", required = true) - @NotBlank(message = "度量文件路径不能为空") - private String url; - - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java deleted file mode 100644 index faf6f78..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtModelAlgorithmCreateDTO.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - -/** - * @description 模型优化上传算法入参 - * @date 2021-01-06 - */ -@Data -@Accessors(chain = true) -public class PtModelAlgorithmCreateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符", required = true) - @NotBlank(message = "算法名称不能为空") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") - private String name; - - @ApiModelProperty(value = "代码目录（路径规则：/algorithm-manage/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/用户上传的算法具体文件(zip文件）名称或从notebook跳转时为/notebook/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/）", required = true) - @NotBlank(message = "代码目录不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "代码目录-输入长度不能超过128个字符") - private String path; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java deleted file mode 100644 index d6f4b66..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtStorageSmallDTO.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 存储 - * @date 2020-03-17 - */ -@Data -public class PtStorageSmallDTO implements Serializable { - private Long id; - private String name; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java deleted file mode 100644 index d9f6eea..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmCreateDTO.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - -/** - * @desciption 创建算法条件 - * @date 2020-04-29 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmCreateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符", required = true) - @NotBlank(message = "算法名称不能为空") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") - private String algorithmName; - - @ApiModelProperty("算法描述，输入长度不能超过256个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "算法描述-输入长度不能超过256个字符") - private String description; - - @ApiModelProperty(value = "镜像版本") - private String imageTag; - - @ApiModelProperty(value = "镜像名称") - private String imageName; - - @ApiModelProperty(value = "创建算法来源(true:由fork创建算法，false：其它创建算法方式)") - private Boolean fork; - - @ApiModelProperty(value = "代码目录（路径规则：/algorithm-manage/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/用户上传的算法具体文件(zip文件）名称或从notebook跳转时为/notebook/{userId}/{YYYYMMDDhhmmssSSS+四位随机数}/）", required = true) - @NotBlank(message = "代码目录不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "代码目录-输入长度不能超过128个字符") - private String codeDir; - - @ApiModelProperty(value = "运行命令") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty("算法用途，输入长度不能超过128个字符") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "算法用途-输入长度不能超过128个字符") - private String algorithmUsage; - - @ApiModelProperty("是否输出训练结果，不填则默认为true") - private Boolean isTrainOut; - - @ApiModelProperty("是否输出训练日志，不填则默认为true") - private Boolean isTrainLog; - - @ApiModelProperty("是否输出可视化日志，不填则默认为false") - private Boolean isVisualizedLog; - - @ApiModelProperty("noteBookId") - private Long noteBookId; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java deleted file mode 100644 index 8768150..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmDeleteDTO.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.Set; - -/** - * @description 算法删除 - * @date 2020-07-02 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmDeleteDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Set ids; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java deleted file mode 100644 index 8f00024..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmQueryDTO.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.annotation.Query; -import org.dubhe.base.PageQueryBase; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import java.io.Serializable; - -/** - * @description 查询算法条件 - * @date 2020-04-29 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "算法来源(1为我的算法, 2为预置算法)") - @Min(value = TrainUtil.NUMBER_ONE, message = "算法来源错误") - @Max(value = TrainUtil.NUMBER_TWO, message = "算法来源错误") - @Query(propName = "algorithm_source", type = Query.Type.EQ) - private Integer algorithmSource; - - @ApiModelProperty(value = "算法名称或者id") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称或者id有误") - private String algorithmName; - - @ApiModelProperty(value = "算法用途") - private String algorithmUsage; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java deleted file mode 100644 index 06785dc..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUpdateDTO.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - -/** - * @description 修改算法 - * @date 2020-06-19 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmUpdateDTO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为null") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long id; - - @ApiModelProperty(value = "算法名称，输入长度不能超过32个字符") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "算法名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "算法名称支持字母、数字、汉字、英文横杠和下划线") - private String algorithmName; - - @ApiModelProperty("算法描述，输入长度不能超过256个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "算法描述-输入长度不能超过256个字符") - private String description; - - @ApiModelProperty("算法用途，输入长度不能超过128个字符") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "算法用途-输入长度不能超过128个字符") - private String algorithmUsage; - - @ApiModelProperty("是否输出训练日志") - private Boolean isTrainLog; - - @ApiModelProperty("是否输出可视化日志") - private Boolean isVisualizedLog; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java deleted file mode 100644 index aff4a97..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageCreateDTO.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - -/** - * @description 创建算法用途条件 - * @date 2020-06-23 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmUsageCreateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "名称输入长度不能超过32个字符", required = true) - @NotBlank(message = "名称不能为空") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "名称支持字母、数字、汉字、英文横杠和下划线") - private String auxInfo; - - @ApiModelProperty(value = "类型", hidden = true) - private String type; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java deleted file mode 100644 index 3d96b15..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageDeleteDTO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 算法用途删除 - * @date 2020/7/2 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmUsageDeleteDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Long[] ids; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java deleted file mode 100644 index f85cd22..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageQueryDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotNull; - -import org.dubhe.base.PageQueryBase; - -/** - * @description 查询算法用途条件 - * @date 2020-06-23 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmUsageQueryDTO extends PageQueryBase { - - @ApiModelProperty(value = "类型", hidden = true) - private String type; - - @ApiModelProperty(value = "是否包含默认值 0：不包含 1包含", required = true) - @NotNull - private Boolean isContainDefault; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java deleted file mode 100644 index 2b8b19a..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainAlgorithmUsageUpdateDTO.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 算法用途修改 - * @date 2020-06-23 - */ -@Data -@Accessors(chain = true) -public class PtTrainAlgorithmUsageUpdateDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为null") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long id; - - @ApiModelProperty("描述, 长度不能超过20个字符") - @Length(max = TrainUtil.NUMBER_TWENTY, message = "名称长度不能超过20个字符") - private String auxInfo; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java deleted file mode 100644 index d586e04..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainDataSourceStatusQueryDTO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; -import java.util.List; - -/** - * @description 查询数据集对应训练状态查询条件 - * @date 2020-05-21 - */ -@Data -@Accessors(chain = true) -public class PtTrainDataSourceStatusQueryDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("数据集路径") - private List dataSourcePath; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java deleted file mode 100644 index f7c85a4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobCreateDTO.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseImageDTO; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; - -/** - * @description 创建训练任务 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainJobCreateDTO extends BaseImageDTO { - - @ApiModelProperty(value = "训练作业名, 长度在1-32个字符", required = true) - @NotNull(message = "训练作业名不能为空") - @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_THIRTY_TWO, message = "训练作业名长度在1-32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "训练作业名称支持字母、数字、汉字、英文横杠和下划线") - private String trainName; - - @ApiModelProperty("描述, 长度不能超过255个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "描述长度不能超过255个字符") - private String description; - - @ApiModelProperty(value = "算法来源id", required = true) - @NotNull(message = "algorithmId不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "algorithmId必须不小于1") - private Long algorithmId; - - @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) - @NotBlank(message = "运行命令不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty(value = "数据来源名称, 长度在1-127个字符", required = true) - @NotNull(message = "数据来源名称不能为空") - @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据来源名称长度在1-127个字符") - private String dataSourceName; - - @ApiModelProperty(value = "数据来源路径, 长度在1-127个字符", required = true) - @NotNull(message = "数据来源路径不能为空") - @Length(min = TrainUtil.NUMBER_ONE, max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据来源路径长度在1-127个字符") - private String dataSourcePath; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") - @NotNull(message = "类型(0为CPU，1为GPU)不能为空") - private Integer resourcesPoolType; - - @ApiModelProperty(value = "规格名称", required = true) - @NotNull(message = "规格名称不能为空") - private String trainJobSpecsName; - - @ApiModelProperty(value = "规格信息", required = true) - @NotNull(message = "规格信息不能为空") - private String trainJobSpecsInfo; - - @ApiModelProperty("true代表保存作业参数") - private Boolean saveParams; - - @ApiModelProperty("作业参数名称，当saveParams为true的时候传递, 输入长度不能超过32个字符") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "作业参数名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "作业参数名称支持字母、数字、汉字、英文横杠和下划线") - private String trainParamName; - - @ApiModelProperty("作业参数描述，当saveParams为true的时候传递, 长度不能超过255个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "作业参数描述-输入长度不能超过255个字符") - private String trainParamDesc; - - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") - @NotNull(message = "训练类型(0为普通训练，1为分布式训练)不能为空") - private Integer trainType; - - @ApiModelProperty(value = "节点个数", required = true) - @Min(value = TrainUtil.NUMBER_ONE, message = "节点个数至少1个") - @NotNull(message = "节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty(value = "训练延时启动时长，单位为小时") - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练延时启动时长不能小于0小时") - @Max(value = TrainUtil.NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT, message = "训练延时启动时长不能大于168小时即时长不能超过一周（7*24小时）") - private Integer delayCreateTime; - - @ApiModelProperty(value = "训练自动停止时长，单位为小时") - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练自动停止时长不能小于0小时") - @Max(value = TrainUtil.NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT, message = "训练自动停止时长不能大于168小时即时长不能超过一周（7*24小时）") - private Integer delayDeleteTime; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; - - @ApiModelProperty(value = "训练信息(失败信息)") - private String trainMsg; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + - "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") - @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") - @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") - private Integer modelResource; - - @ApiModelProperty(value = "模型id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型id必须大于1") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") - private Long modelBranchId; - - @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") - private String teacherModelIds; - - @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java deleted file mode 100644 index 5ce65f6..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDTO.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description： 训练任务 - * @date 2020-03-17 - */ -@Data -public class PtTrainJobDTO implements Serializable { - private Long id; - private String name; - private String remark; - private PtDatasetSmallDTO dataset; - private PtImageSmallDTO image; - private PtStorageSmallDTO storage; - private TeamSmallDTO team; - private String selectorLabel; - private Integer podNum; - private Integer gpuNum; - private Integer memNum; - private Integer cpuNum; - private Integer level; - private Timestamp startTime; - private Timestamp closeTime; - private String cmd; - private String status; - private UserSmallDTO confirmUser; - private UserSmallDTO createUser; - private Timestamp createTime; - private Timestamp updateTime; - private Boolean deleted; - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java deleted file mode 100644 index 371060b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDeleteDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 删除训练任务 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobDeleteDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "trainId,如果只传递trainId,代表删除该trainId下的所有job", required = true) - @NotNull(message = "trainId不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "trainId数值不合法") - private Long trainId; - - @ApiModelProperty(value = "jobId") - @Min(value = TrainUtil.NUMBER_ONE, message = "jobId数值不合法") - private Long id; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java deleted file mode 100644 index fba7474..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobDetailQueryDTO.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 根据jobId查询训练任务详情查询条件 - * @date 2020-06-12 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobDetailQueryDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为null") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long id; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java deleted file mode 100644 index a494745..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobResumeDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 恢复训练 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobResumeDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "id数值不合法") - private Long id; - - @ApiModelProperty(value = "path", required = true) - @NotNull(message = "path不能为空") - private String path; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java deleted file mode 100644 index d041954..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobSpecsQueryDTO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description： 训练作业规格查询 - * @date 2020-05-06 - */ -@Data -public class PtTrainJobSpecsQueryDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") - private Integer resourcesPoolType; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java deleted file mode 100644 index 4b52592..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobStopDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 停止训练 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobStopDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "trainId,如果只传递trainId,代表删除该trainId下的所有job", required = true) - @NotNull(message = "trainId不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "trainId数值不合法") - private Long trainId; - - @ApiModelProperty(value = "jobId") - @Min(value = TrainUtil.NUMBER_ONE, message = "jobId数值不合法") - private Long id; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java deleted file mode 100644 index f7570dc..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobUpdateDTO.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseImageDTO; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; - -/** - * @description 训练任务修改 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainJobUpdateDTO extends BaseImageDTO { - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为null") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long id; - - @ApiModelProperty("描述, 长度不能超过255个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "描述长度不能超过255个字符") - private String description; - - @ApiModelProperty("算法id") - @NotNull(message = "algorithmId不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "algorithmId必须大于1") - private Long algorithmId; - - @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) - @NotBlank(message = "运行命令不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源路径不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") - private String dataSourcePath; - - @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源名称不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") - private String dataSourceName; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") - @NotNull(message = "类型(0为CPU，1为GPU)不能为空") - private Integer resourcesPoolType; - - @ApiModelProperty(value = "规格名称", required = true) - @NotNull(message = "规格名称不能为空") - private String trainJobSpecsName; - - @ApiModelProperty(value = "规格信息", required = true) - @NotNull(message = "规格信息不能为空") - private String trainJobSpecsInfo; - - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") - @NotNull(message = "训练类型(0为普通训练，1为分布式训练)不能为空") - private Integer trainType; - - @ApiModelProperty(value = "节点个数", required = true) - @Min(value = TrainUtil.NUMBER_ONE, message = "节点个数在1~8之间") - @Max(value = TrainUtil.NUMBER_EIGHT, message = "节点个数在1~8之间") - @NotNull(message = "节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty(value = "训练延时启动时长，单位为小时") - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练延时启动时长不能小于0小时") - @Max(value = TrainUtil.NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT, message = "训练延时启动时长不能大于168小时即时长不能超过一周（7*24小时）") - private Integer delayCreateTime; - - @ApiModelProperty(value = "训练自动停止时长，单位为小时") - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练自动停止时长必须不能小于0小时") - @Max(value = TrainUtil.NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT, message = "训练自动停止时长不能大于168小时即时长不能超过一周（7*24小时）") - private Integer delayDeleteTime; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + - "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") - @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") - @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") - private Integer modelResource; - - @ApiModelProperty(value = "模型id") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") - private Long modelBranchId; - - @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") - private String teacherModelIds; - - @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java deleted file mode 100644 index d3e93b1..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainJobVersionQueryDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.base.PageQueryBase; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 训练任务版本查询 - * @date 2020-06-30 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobVersionQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "训练作业id", required = true) - @NotNull(message = "训练作业id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "训练作业id必须不小于1") - private Long trainId; - - @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知,6为删除，7为创建失败") - private Integer trainStatus; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java deleted file mode 100644 index 776a75d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainLogQueryDTO.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.TrainUtil; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 训练日志 查询DTO - * @date 2020-05-08 - */ -@Data -@Accessors(chain = true) -public class PtTrainLogQueryDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练job的id") - @NotNull(message = "训练job的id不能为空") - private Integer jobId; - - @ApiModelProperty(value = "起始行") - @Min(value = TrainUtil.NUMBER_ZERO, message = "startLine不能小于0") - private Integer startLine; - - @ApiModelProperty(value = "行数") - @Min(value = TrainUtil.NUMBER_ONE, message = "lines不能小于1") - @Max(value = TrainUtil.NUMBER_ONE_THOUSAND, message = "lines不能大于1000") - private Integer lines; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java deleted file mode 100644 index 56e1b31..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainModelDTO.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; -import java.io.Serializable; - -/** - * @description 训练使用模型 - * @date 2020-11-23 - */ -@Data -@Accessors(chain = true) -public class PtTrainModelDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("非炼知模型id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型id必须大于1") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") - private Long modelBranchId; - - @ApiModelProperty("模型是否为预置模型（0默认模型，1预置模型, 2炼知模型" + - "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") - @NotNull(message = "模型类型不能为空") - @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") - @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") - private Integer modelResource; - - @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") - private String teacherModelIds; - - @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java deleted file mode 100644 index 0169e00..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamCreateDTO.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseImageDTO; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; - -/** - * @description 任务参数创建条件 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainParamCreateDTO extends BaseImageDTO { - - @ApiModelProperty(value = "任务参数名称,输入长度不能超过32个字符", required = true) - @NotBlank(message = "任务参数名称不能为空") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "任务参数名称-输入长度不能超过32个字符") - @Pattern(regexp = TrainUtil.REGEXP, message = "任务参数名称支持字母、数字、汉字、英文横杠和下划线") - private String paramName; - - @ApiModelProperty("描述，输入长度不能超过256个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "任务参数描述-输入长度不能超过256个字符") - private String description; - - @ApiModelProperty(value = "算法id,算法id不能小于1", required = true) - @NotNull(message = "算法id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "算法id不能小于1") - private Long algorithmId; - - @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) - @NotBlank(message = "运行命令不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源路径不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") - private String dataSourcePath; - - @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源名称不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") - private String dataSourceName; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") - @NotNull(message = "类型(0为CPU，1为GPU)不能为空") - private Integer resourcesPoolType; - - @ApiModelProperty(value = "规格名称", required = true) - @NotNull(message = "规格名称不能为空") - private String trainJobSpecsName; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") - @NotNull(message = "训练类型(0为普通训练，1为分布式训练)不能为空") - private Integer trainType; - - @ApiModelProperty(value = "节点个数", required = true) - @Min(value = TrainUtil.NUMBER_ONE, message = "节点个数在1~8之间") - @Max(value = TrainUtil.NUMBER_EIGHT, message = "节点个数在1~8之间") - @NotNull(message = "节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + - "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") - @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") - @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") - private Integer modelResource; - - @ApiModelProperty(value = "模型id") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") - private Long modelBranchId; - - @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") - private String teacherModelIds; - - @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") - private String studentModelIds; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java deleted file mode 100644 index edb82e7..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamDeleteDTO.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; -import java.util.Set; - -/** - * @description 训练模板删除 - * @date 2020-07-02 - */ -@Data -@Accessors(chain = true) -public class PtTrainParamDeleteDTO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "id", required = true) - @NotNull(message = "id不能为空") - private Set ids; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java deleted file mode 100644 index 4a3cf22..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamQueryDTO.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.base.PageQueryBase; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import java.io.Serializable; - -/** - * @description 任务参数查询条件 - * @date 2020-4-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainParamQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("任务参数名称,输入长度不能超过128个字符") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "任务参数名称-输入长度不能超过128个字符") - private String paramName; - - @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") - private Integer resourcesPoolType; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java deleted file mode 100644 index 87a56ca..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainParamUpdateDTO.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseImageDTO; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Pattern; - -/** - * @description 任务参数修改条件 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -public class PtTrainParamUpdateDTO extends BaseImageDTO { - - @ApiModelProperty(value = "任务参数id,任务参数id不能小于1", required = true) - @NotNull(message = "任务参数id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "任务参数id不能小于1") - private Long id; - - @ApiModelProperty(value = "任务参数名称,输入长度不能超过32个字符", required = true) - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "任务参数名称-输入长度不能超过32个字符") - @NotBlank(message = "任务参数名称不能为空") - @Pattern(regexp = TrainUtil.REGEXP, message = "任务参数名称支持字母、数字、汉字、英文横杠和下划线") - private String paramName; - - @ApiModelProperty("描述,输入长度不能超过256个字符") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "任务参数描述-输入长度不能超过256个字符") - private String description; - - @ApiModelProperty(value = "算法id,算法id不能小于1", required = true) - @NotNull(message = "算法id不能为空") - @Min(value = TrainUtil.NUMBER_ONE, message = "算法id不能小于1") - private Long algorithmId; - - @ApiModelProperty(value = "运行命令,输入长度不能超过128个字符", required = true) - @NotBlank(message = "运行命令不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT, message = "运行命令-输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty(value = "数据集来源路径,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源路径不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源路径-输入长度不能超过127个字符") - private String dataSourcePath; - - @ApiModelProperty(value = "数据集来源名称,输入长度不能超过127个字符", required = true) - @NotBlank(message = "数据集来源名称不能为空") - @Length(max = TrainUtil.NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN, message = "数据集来源名称-输入长度不能超过127个字符") - private String dataSourceName; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty(value = "类型(0为CPU，1为GPU)", required = true) - @Min(value = TrainUtil.NUMBER_ZERO, message = "类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "类型错误") - @NotNull(message = "类型(0为CPU，1为GPU)不能为空") - private Integer resourcesPoolType; - - @ApiModelProperty(value = "规格名称", required = true) - @NotNull(message = "规格名称不能为空") - private String trainJobSpecsName; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty(value = "训练类型 0：普通训练，1：分布式训练") - @Min(value = TrainUtil.NUMBER_ZERO, message = "训练类型错误") - @Max(value = TrainUtil.NUMBER_ONE, message = "训练类型错误") - private Integer trainType; - - @ApiModelProperty(value = "节点个数") - @Min(value = TrainUtil.NUMBER_ONE, message = "节点个数在1~8之间") - @Max(value = TrainUtil.NUMBER_EIGHT, message = "节点个数在1~8之间") - private Integer resourcesPoolNode; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)" + - "当值为0和1的时候，需要传递ModelId, 当值为2的时候传递teacherModelIds和studentModelIds") - @Min(value = PtModelUtil.NUMBER_ZERO, message = "模型来源错误") - @Max(value = PtModelUtil.NUMBER_TWO, message = "模型来源错误") - private Integer modelResource; - - @ApiModelProperty(value = "模型id") - @Min(value = TrainUtil.NUMBER_ONE, message = "id必须大于1") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - @Min(value = TrainUtil.NUMBER_ONE, message = "模型版本对应的id必须大于1") - private Long modelBranchId; - - @ApiModelProperty(value = "教师模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "教师模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "教师模型ids参数格式不正确") - private String teacherModelIds; - - @ApiModelProperty(value = "学生模型ids,多个id之前用','隔开") - @Length(max = TrainUtil.NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE, message = "学生模型长度不能超过255个字符") - @Pattern(regexp = "^([1-9][0-9]*,)*[1-9][0-9]*$", message = "学生模型ids参数格式不正确") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java deleted file mode 100644 index f1016ad..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/PtTrainQueryDTO.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.base.PageQueryBase; -import org.dubhe.utils.TrainUtil; -import org.hibernate.validator.constraints.Length; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import java.io.Serializable; - -/** - * @description 查询训练 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainQueryDTO extends PageQueryBase implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练作业名称或者id") - @Length(max = TrainUtil.NUMBER_THIRTY_TWO, message = "训练作业名称或者id有误") - private String trainName; - - @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") - @Min(value = TrainUtil.NUMBER_ZERO, message = "trainStatus错误") - @Max(value = TrainUtil.NUMBER_SEVEN, message = "trainStatus错误") - private Integer trainStatus; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java deleted file mode 100644 index a966041..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/dto/SourceNoteBookDTO.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.dto; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -/** - * @description 第三方创建NoteBook请求对象 - * - * @date 2020.05.15 - */ -@Data -public class SourceNoteBookDTO implements Serializable { - - @ApiModelProperty("第三方源主键") - @NotNull - private Long sourceId; - - @ApiModelProperty("第三方源资源路径") - @NotBlank - private String sourceFilePath; - - @ApiModelProperty(value = "当前登录人", hidden = true) - private Long curUserId; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java deleted file mode 100644 index 0aa61d6..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptBuiltIn.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import org.dubhe.base.BaseEntity; - -/** - * @description 内置模型 - * @date 2020-05-22 - */ -@Data -@TableName("model_opt_built_in") -public class ModelOptBuiltIn extends BaseEntity { - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - /** - * 算法类型：0-剪枝，1-蒸馏，2-量化 - */ - @TableField("type") - private Integer type; - /** - * 算法名称 - */ - @TableField("algorithm") - private String algorithm; - /** - * 算法路径 - */ - @TableField("algorithm_path") - private String algorithmPath; - /** - * 数据集名称 - */ - @TableField("dataset") - private String dataset; - /** - * 数据集路径 - */ - @TableField("dataset_path") - private String datasetPath; - /** - * 模型名称 - */ - @TableField("model") - private String model; - /** - * 模型路径 - */ - @TableField(value = "model_address") - private String modelAddress; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java deleted file mode 100644 index d42d737..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptDataset.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; -import org.dubhe.base.BaseEntity; - -/** - * @description 模型优化数据集 - * @date 2021-01-06 - */ -@Data -@TableName("model_opt_dataset") -public class ModelOptDataset extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 数据集名称 - */ - @TableField(value = "name") - private String name; - - /** - * 数据集路径 - */ - @TableField(value = "path") - private String path; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java deleted file mode 100644 index e426469..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTask.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.bean.copier.CopyOptions; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import org.dubhe.base.BaseEntity; - -/** - * @description 模型优化任务 - * @date 2020-05-22 - */ -@Data -@TableName(value = "model_opt_task", autoResultMap = true) -public class ModelOptTask extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - /** - * 任务名称 - */ - @TableField(value = "name") - private String name; - /** - * 任务描述 - */ - @TableField(value = "description") - private String description; - /** - * 是否内置 - */ - @TableField(value = "is_built_in") - private Boolean isBuiltIn; - /** - * 模型名称 - */ - @TableField(value = "model_name") - private String modelName; - /** - * 模型id - */ - @TableField(value = "model_id") - private Long modelId; - /** - * 模型路径 - */ - @TableField(value = "model_address") - private String modelAddress; - /** - * 算法选择类型 - */ - @TableField(value = "algorithm_type") - private Integer algorithmType; - /** - * 优化算法id - */ - @TableField(value = "algorithm_id") - private Long algorithmId; - /** - * 优化算法 - */ - @TableField(value = "algorithm_name") - private String algorithmName; - /** - * 算法路径 - */ - @TableField(value = "algorithm_path") - private String algorithmPath; - /** - * 数据集id - */ - @TableField(value = "dataset_id") - private Long datasetId; - /** - * 数据集名称 - */ - @TableField(value = "dataset_name") - private String datasetName; - /** - * 数据集路径 - */ - @TableField(value = "dataset_path") - private String datasetPath; - /** - * 运行命令 - */ - @TableField(value = "command") - private String command; - /** - * 运行参数 - */ - @TableField(value = "params", typeHandler = FastjsonTypeHandler.class) - private JSONObject params; - - public void copy(ModelOptTask source) { - BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java deleted file mode 100644 index 239a9d9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelOptTaskInstance.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.bean.copier.CopyOptions; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -import java.sql.Timestamp; - -/** - * @description 模型优化任务实例 - * @date 2020-05-22 - */ -@Data -@TableName(value = "model_opt_task_instance", autoResultMap = true) -@Accessors(chain = true) -public class ModelOptTaskInstance extends BaseEntity { - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - /** - * 任务id - */ - @TableField(value = "task_id") - private Long taskId; - /** - * 任务名称 - */ - @TableField(value = "task_name") - private String taskName; - /** - * 是否内置 - */ - @TableField(value = "is_built_in") - private Boolean isBuiltIn; - /** - * 模型id - */ - @TableField(value = "model_id") - private Long modelId; - /** - * 模型名称 - */ - @TableField(value = "model_name") - private String modelName; - /** - * 模型路径 - */ - @TableField(value = "model_address") - private String modelAddress; - - /** - * 算法选择类型 - */ - @TableField(value = "algorithm_type") - private Integer algorithmType; - /** - * 优化算法id - */ - @TableField(value = "algorithm_id") - private Long algorithmId; - /** - * 优化算法 - */ - @TableField(value = "algorithm_name") - private String algorithmName; - /** - * 算法路径 - */ - @TableField(value = "algorithm_path") - private String algorithmPath; - /** - * 提交时间 - */ - @TableField(value = "start_time") - private Timestamp startTime; - /** - * 完成时间 - */ - @TableField(value = "end_time") - private Timestamp endTime; - /** - * 输出模型路径 - */ - @TableField(value = "output_model_dir") - private String outputModelDir; - /** - * 日志路径 - */ - @TableField(value = "log_path") - private String logPath; - /** - * 任务状态 - */ - @TableField(value = "status") - private String status; - /** - * 数据集id - */ - @TableField(value = "dataset_id") - private Long datasetId; - /** - * 数据集名称 - */ - @TableField(value = "dataset_name") - private String datasetName; - /** - * 数据集路径 - */ - @TableField(value = "dataset_path") - private String datasetPath; - /** - * 运行命令 - */ - @TableField(value = "command") - private String command; - /** - * 运行参数 - */ - @TableField(value = "params", typeHandler = FastjsonTypeHandler.class) - private JSONObject params; - /** - * 优化前性能参数 - */ - @TableField(value = "opt_result_before") - private String optResultBefore; - /** - * 优化后性能参数 - */ - @TableField(value = "opt_result_after") - private String optResultAfter; - - /** - * 模型优化前性能参数json文件路径 - */ - @TableField(value = "opt_result_json_path_before") - private String optResultJsonPathBefore; - /** - * 模型优化后性能参数json文件路径 - */ - @TableField(value = "opt_result_json_path_after") - private String optResultJsonPathAfter; - - public void copy(ModelOptTaskInstance source) { - BeanUtil.copyProperties(source, this, CopyOptions.create().setIgnoreNullValue(true)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java deleted file mode 100644 index ea52708..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQuery.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.entity; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -/** - * @description model详情 - * @date 2020-10-09 - */ -@Data -@TableName("pt_model_info") -public class ModelQuery { - @TableField(value = "model_resource") - @ApiModelProperty(value = "模型类型") - private Integer modelResource; - @TableField(value = "name") - @ApiModelProperty(value = "模型名称") - private String name; - @TableField(value = "model_version") - @ApiModelProperty(value = "模型版本") - private String modelVersion; - @TableField(value = "id") - @ApiModelProperty(value = "模型id") - private Integer id; - @TableField(value = "url") - @ApiModelProperty(value = "模型路径") - private String url; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java deleted file mode 100644 index dadbb1e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/ModelQueryBrance.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.entity; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -/** - * @description model 模型版本 - * @date 2020-10-09 - */ -@Data -@TableName("pt_model_branch") -public class ModelQueryBrance { - @TableField(value = "model_resource") - @ApiModelProperty(value = "模型类型") - private Integer modelResource; - @TableField(value = "name") - @ApiModelProperty(value = "模型名称") - private String name; - @TableField(value = "version") - @ApiModelProperty(value = "模型版本") - private String version; - @TableField(value = "id") - @ApiModelProperty(value = "模型id") - private Integer id; - @TableField(value = "url") - @ApiModelProperty(value = "模型路径") - private String url; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java deleted file mode 100644 index b32d5d9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBook.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import com.fasterxml.jackson.annotation.JsonIgnore; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.base.BaseEntity; - -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Size; -import java.util.Date; - -/** - * @description note book 实体 - * @date 2020-04-28 - */ -@Data -@TableName("notebook") -public class NoteBook extends BaseEntity { - - @TableId(value = "id", type = IdType.AUTO) - @ApiModelProperty(hidden = true) - private Long id; - - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - @ApiModelProperty(hidden = true) - private Long userId; - - @TableField(value = "name") - @JsonIgnore - private String name; - - @TableField(value = "notebook_name") - private String noteBookName; - - @TableField(value = "description") - private String description; - - public final static String COLUMN_URL = "url"; - @TableField(value = COLUMN_URL) - @ApiModelProperty(hidden = true) - private String url; - - @TableField(value = "total_run_min") - @ApiModelProperty(hidden = true) - private Integer totalRunMin; - - @TableField(value = "cpu_num") - @ApiModelProperty(value = "cpu数量") - private Integer cpuNum; - - @TableField(value = "gpu_num") - @ApiModelProperty(value = "gpu数量") - private Integer gpuNum; - - @TableField(value = "mem_num") - @ApiModelProperty(value = "内存大小") - private Integer memNum; - - @TableField(value = "disk_mem_num") - @ApiModelProperty(value = "硬盘内存大小") - private Integer diskMemNum; - - - public final static String COLUMN_STATUS = "status"; - /** - * 0运行中，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用） - */ - @TableField(value = COLUMN_STATUS) - @ApiModelProperty(hidden = true) - private Integer status; - - @TableField(value = "last_start_time") - @ApiModelProperty(hidden = true) - private Date lastStartTime; - - @TableField(value = "last_operation_timeout") - @ApiModelProperty(hidden = true) - private Long lastOperationTimeout; - - /** - * 0 - notebook 创建 1- 其它系统创建 - */ - @TableField(value = "create_resource") - @ApiModelProperty(hidden = true) - private Integer createResource; - - @TableField(value = "k8s_status_code") - @ApiModelProperty(hidden = true) - private String k8sStatusCode; - - @TableField(value = "k8s_status_info") - @ApiModelProperty(hidden = true) - private String k8sStatusInfo; - - @TableField(value = "k8s_namespace") - @ApiModelProperty(hidden = true) - private String k8sNamespace; - - @TableField(value = "k8s_resource_name") - @ApiModelProperty(hidden = true) - private String k8sResourceName; - - @TableField(value = "k8s_image_name") - @ApiModelProperty("镜像名称") - @NotNull(message = "请选择镜像") - private String k8sImageName; - - @TableField(value = "k8s_mount_path") - @ApiModelProperty(hidden = true) - private String k8sMountPath; - - @TableField(value = "k8s_pvc_path") - @ApiModelProperty(hidden = true) - private String k8sPvcPath; - - @TableField(value = "data_source_name") - @ApiModelProperty(hidden = true) - @Size(max = 255, message = "数据集名称超长") - private String dataSourceName; - - @TableField(value = "data_source_path") - @Size(max = 255, message = "数据集路径超长") - @ApiModelProperty(hidden = true) - private String dataSourcePath; - - @TableField(value = "algorithm_id") - @ApiModelProperty(hidden = true) - private Long algorithmId; - - @Override - public String toString() { - return "NoteBook{" + - "id=" + id + - ", userId=" + userId + - ", name='" + name + '\'' + - ", noteBookName='" + noteBookName + '\'' + - ", description='" + description + '\'' + - ", url='" + url + '\'' + - ", totalRunMin=" + totalRunMin + - ", cpuNum=" + cpuNum + - ", gpuNum=" + gpuNum + - ", memNum=" + memNum + - ", diskMemNum=" + diskMemNum + - ", status=" + status + - ", lastStartTime=" + lastStartTime + - ", lastOperationTimeout=" + lastOperationTimeout + - ", createResource=" + createResource + - ", k8sStatusCode='" + k8sStatusCode + '\'' + - ", k8sStatusInfo='" + k8sStatusInfo + '\'' + - ", k8sNamespace='" + k8sNamespace + '\'' + - ", k8sResourceName='" + k8sResourceName + '\'' + - ", k8sImageName='" + k8sImageName + '\'' + - ", k8sMountPath='" + k8sMountPath + '\'' + - ", k8sPvcPath='" + k8sPvcPath + '\'' + - ", dataSourceName='" + dataSourceName + '\'' + - ", dataSourcePath='" + dataSourcePath + '\'' + - ", algorithmId=" + algorithmId + - '}'; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java deleted file mode 100644 index 46a6f08..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/NoteBookModel.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.base.BaseEntity; - -/** - * @description notebook 模板 - * @date 2020-06-01 - */ -@Data -@TableName("notebook_model") -public class NoteBookModel extends BaseEntity { - - @TableField(value = "model_type") - private String modelType; - - @TableField(value = "cpu_num") - @ApiModelProperty("cpu数量") - private Integer cpuNum; - - @TableField(value = "gpu_num") - @ApiModelProperty("gpu数量") - private Integer gpuNum; - - @TableField(value = "mem_num") - @ApiModelProperty("内存大小") - private Integer memNum; - - @TableField(value = "spec") - @ApiModelProperty("GPU规格") - private String spec; - - @TableField(value = "disk_mem_num") - @ApiModelProperty("硬盘内存大小") - private Integer diskMemNum; - - @TableField(value = "default_status") - @ApiModelProperty("默认值状态 0 - 非默认 1- 默认") - private Integer defaultStatus; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java deleted file mode 100644 index 460aa1f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtImage.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -/** - * @description 镜像 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName(value = "pt_image", autoResultMap = true) -public class PtImage extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 镜像名称 - */ - @TableField(value = "image_name") - private String imageName; - - /** - * 镜像地址 - */ - @TableField(value = "image_url") - private String imageUrl; - - /** - * 镜像版本 - */ - @TableField(value = "image_tag") - private String imageTag; - - /** - * 镜像描述 - */ - @TableField(value = "remark") - private String remark; - - /** - * projectName - */ - @TableField(value = "project_name") - private String projectName; - - /** - * 镜像来源 - */ - @TableField(value = "image_resource") - private Integer imageResource; - - - /** - * 镜像状态 - */ - @TableField(value = "image_status") - private Integer imageStatus; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id", fill = FieldFill.INSERT) - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java deleted file mode 100644 index 769b730..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtJobParam.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -import java.sql.Timestamp; - -/** - * @description job运行参数及结果 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName(value = "pt_job_param", autoResultMap = true) -public class PtJobParam extends BaseEntity { - - /** - * 主键id - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 训练作业jobId - */ - @TableField(value = "train_job_id") - private Long trainJobId; - - /** - * 算法来源id - */ - @TableField(value = "algorithm_id") - private Long algorithmId; - - /** - * 运行命令 - */ - @TableField(value = "run_command") - private String runCommand; - - /** - * 镜像名称 - */ - @TableField(value = "image_name") - private String imageName; - - /** - * 运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数) - */ - @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) - private JSONObject runParams; - - /** - * F1值 - */ - @TableField(value = "param_f1") - private String paramF1; - - /** - * 召回率 - */ - @TableField(value = "param_callback") - private String paramCallback; - - /** - * 精确率 - */ - @TableField(value = "param_precise") - private String paramPrecise; - - /** - * 准确率 - */ - @TableField(value = "param_accuracy") - private String paramAccuracy; - - /** - *训练延时启动时间 - */ - @TableField(value = "delay_create_time") - private Timestamp delayCreateTime; - - /** - *训练自动停止时间 - */ - @TableField(value = "delay_delete_time") - private Timestamp delayDeleteTime; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java deleted file mode 100644 index 192ed80..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtMeasure.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.*; -import org.dubhe.base.BaseEntity; - -/** - * @description 度量管理实体类 - * @date 2020-11-16 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Builder -@TableName("pt_measure") -@NoArgsConstructor -@AllArgsConstructor -public class PtMeasure extends BaseEntity { - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 度量名称 - */ - @TableField("name") - private String name; - - /** - * 度量文件路径 - */ - @TableField("url") - private String url; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id", fill = FieldFill.INSERT) - private Long originUserId; - - /** - * 度量描述信息 - */ - @TableField("description") - private String description; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java deleted file mode 100644 index ee675ac..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrain.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -/** - * @description 训练作业主 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName("pt_train") -public class PtTrain extends BaseEntity { - - /** - * 主键id - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 训练作业名 - */ - @TableField(value = "train_name") - private String trainName; - - /** - * 训练作业名 - */ - @TableField(value = "train_key") - private String trainKey; - - /** - * 训练作业有效job版本数量 - */ - @TableField(value = "version_num") - private Integer versionNum; - - /** - * 训练作业job总版本数量 - */ - @TableField(value = "total_num") - private Integer totalNum; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java deleted file mode 100644 index 82bf3eb..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithm.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.*; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -import javax.validation.constraints.NotNull; - -/** - * @description 算法 - * @date 2020-04-29 - */ - -@EqualsAndHashCode(callSuper = true) -@Data -@TableName(value = "pt_train_algorithm", autoResultMap = true) -@Accessors(chain = true) -public class PtTrainAlgorithm extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - @NotNull(groups = {Update.class}) - private Long id; - - /** - * 算法名称 - */ - @TableField(value = "algorithm_name") - private String algorithmName; - - /** - * 算法描述 - */ - @TableField(value = "description") - private String description; - - /** - * 算法来源(1为我的算法，2为预置算法) - */ - @TableField(value = "algorithm_source") - private Integer algorithmSource; - - /** - * 环境镜像名称 - */ - @TableField(value = "image_name") - private String imageName; - - /** - * 代码目录 - */ - @TableField(value = "code_dir") - private String codeDir; - - /** - * 运行命令 - */ - @TableField(value = "run_command") - private String runCommand; - - /** - * 运行参数 - */ - @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) - private JSONObject runParams; - - /** - * 算法用途 - */ - @TableField(value = "algorithm_usage") - private String algorithmUsage; - - /** - * 算法精度 - */ - @TableField(value = "accuracy") - private String accuracy; - - /** - * P4推理速度（ms） - */ - @TableField(value = "p4_inference_speed") - private Integer p4InferenceSpeed; - - /** - * 输出结果（1是，0否） - */ - @TableField(value = "is_train_out") - private Boolean isTrainOut; - - /** - * 输出日志（1是，0否） - */ - @TableField(value = "is_train_log") - private Boolean isTrainLog; - - /** - * 可视化日志（1是，0否） - */ - @TableField(value = "is_visualized_log") - private Boolean isVisualizedLog; - - /** - * 算法状态 - */ - @TableField(value = "algorithm_status") - private Integer algorithmStatus; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java deleted file mode 100644 index f7cc715..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainAlgorithmUsage.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -import javax.validation.constraints.NotNull; - -/** - * @description 算法 - * @date 2020-06-23 - */ - -@EqualsAndHashCode(callSuper = true) -@Data -@TableName("pt_auxiliary_info") -@Accessors(chain = true) -public class PtTrainAlgorithmUsage extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - @NotNull(groups = {Update.class}) - private Long id; - - /** - * 用户id - */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - private Long originUserId; - - /** - * 类型 - */ - @TableField(value = "type") - private String type; - - /** - * 算法来源(1为我的算法，2为预置算法) - */ - @TableField(value = "aux_info") - private String auxInfo; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java deleted file mode 100644 index 6a1b6a9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJob.java +++ /dev/null @@ -1,205 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -/** - * @description 训练作业job - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName(value = "pt_train_job", autoResultMap = true) -public class PtTrainJob extends BaseEntity { - - /** - * 主键id - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 训练作业id - */ - @TableField(value = "train_id") - private Long trainId; - - /** - * job版本 - */ - @TableField(value = "train_version") - private String trainVersion; - - /** - * job父版本 - */ - @TableField(value = "parent_train_version") - private String parentTrainVersion; - - /** - * 任务名称 - */ - @TableField(value = "job_name") - private String jobName; - - /** - * 描述 - */ - @TableField(value = "description") - private String description; - - /** - * 数据来源路径 - */ - @TableField(value = "data_source_path") - private String dataSourcePath; - - - /** - * 数据来源名称 - */ - @TableField(value = "data_source_name") - private String dataSourceName; - - /** - * 训练时长 - */ - @TableField(value = "runtime") - private String runtime; - - /** - * 训练输出位置 - */ - @TableField(value = "out_path") - private String outPath; - - /** - * 作业日志路径 - */ - @TableField(value = "log_path") - private String logPath; - - /** - * 规格名称 - */ - @TableField(value = "train_job_specs_name") - private String trainJobSpecsName; - - /** - * 类型(0为CPU，1为GPU) - */ - @TableField(value = "resources_pool_type") - private Integer resourcesPoolType; - - /** - * 规格 - */ - @TableField(value = "resources_pool_specs") - private String resourcesPoolSpecs; - - /** - * 节点个数 - */ - @TableField(value = "resources_pool_node") - private Integer resourcesPoolNode; - - /** - * 训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败 - */ - @TableField(value = "train_status") - private Integer trainStatus; - - /** - * 可视化日志路径 - */ - @TableField(value = "visualized_log_path") - private String visualizedLogPath; - - /** - * k8s返回的job名称 - */ - @TableField(value = "k8s_job_name") - private String k8sJobName; - - /** - * 训练类型,0:普通训练，1：分布式训练 - */ - @TableField(value = "train_type") - private Integer trainType; - - /** - * 验证数据集来源名称 - */ - @TableField(value = "val_data_source_name") - private String valDataSourceName; - - /** - * 验证数据集来源路径 - */ - @TableField(value = "val_data_source_path") - private String valDataSourcePath; - - /** - * 是否验证数据集 - */ - @TableField(value = "val_type") - private Integer valType; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - private Long originUserId; - - /** - * 模型来源 - */ - @TableField(value = "model_resource") - private Integer modelResource; - - /** - * 模型id - */ - @TableField(value = "model_id") - private Long modelId; - - /** - * 模型对应版本id - */ - @TableField(value = "model_branch_id") - private Long modelBranchId; - - /** - * 训练信息(失败信息) - */ - @TableField(value = "train_msg") - private String trainMsg; - - @TableField(value = "teacher_model_ids") - private String teacherModelIds; - - @TableField(value = "student_model_ids") - private String studentModelIds; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java deleted file mode 100644 index c282bb9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainJobSpecs.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -/** - * @decription 训练作业规格 - * @date 2020-05-06 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName(value = "pt_train_job_specs", autoResultMap = true) -public class PtTrainJobSpecs extends BaseEntity { - - /** - * 主键id - */ - @TableId(value = "id", type = IdType.AUTO) - private Integer id; - - /** - * 规格名称 - */ - @TableField(value = "specs_name") - private String specsName; - - /** - * 规格信息 - */ - @TableField(value = "specs_info", typeHandler = FastjsonTypeHandler.class) - private JSONObject specsInfo; - - /** - * 规格类型 - */ - @TableField(value = "resources_pool_type") - private Integer resourcesPoolType; - -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java deleted file mode 100644 index 9024159..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/entity/PtTrainParam.java +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.entity; - -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.annotation.*; -import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.experimental.Accessors; -import org.dubhe.base.BaseEntity; - -/** - * @descrption 任务参数 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -@Accessors(chain = true) -@TableName(value = "pt_train_param", autoResultMap = true) -public class PtTrainParam extends BaseEntity { - - /** - * 主键 - */ - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - /** - * 任务参数名称 - */ - @TableField(value = "param_name") - private String paramName; - - /** - * 描述 - */ - @TableField(value = "description") - private String description; - - /** - * 算法id - */ - @TableField(value = "algorithm_id") - private Long algorithmId; - - /** - * 运行命令 - */ - @TableField(value = "run_command") - private String runCommand; - - /** - * 镜像名称 - */ - @TableField(value = "image_name") - private String imageName; - - /** - * 数据集来源名称 - */ - @TableField(value = "data_source_name") - private String dataSourceName; - - /** - * 数据集来源路径 - */ - @TableField(value = "data_source_path") - private String dataSourcePath; - - /** - * 输出路径 - */ - @TableField(value = "out_path") - private String outPath; - - /** - * 运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数) - */ - @TableField(value = "run_params", typeHandler = FastjsonTypeHandler.class) - private JSONObject runParams; - - /** - * 算法来源(1为我的算法，2为预置算法) - */ - @TableField(value = "algorithm_source") - private Integer algorithmSource; - - /** - * 日志输出路径 - */ - @TableField(value = "log_path") - private String logPath; - - /** - * 规格名称 - */ - @TableField(value = "train_job_specs_name") - private String trainJobSpecsName; - - /** - * 类型(0为CPU，1为GPU) - */ - @TableField(value = "resources_pool_type") - private Integer resourcesPoolType; - - /** - * 节点个数 - */ - @TableField(value = "resources_pool_node") - private Integer resourcesPoolNode; - - /** - * 验证数据集来源名称 - */ - @TableField(value = "val_data_source_name") - private String valDataSourceName; - - /** - * 验证数据集来源路径 - */ - @TableField(value = "val_data_source_path") - private String valDataSourcePath; - - /** - * 是否验证数据集 - */ - @TableField(value = "val_type") - private Integer valType; - - /** - * 模型id - */ - @TableField(value = "model_id") - private Long modelId; - - /** - * 模型版本对应id - */ - @TableField(value = "model_branch_id") - private Long modelBranchId; - - /** - * 模型来源 - */ - @TableField(value = "model_resource") - private Integer modelResource; - - /** - * 训练类型,0:普通训练，1：分布式训练 - */ - @TableField(value = "train_type") - private Integer trainType; - - /** - * 资源拥有者ID - */ - @TableField(value = "origin_user_id",fill = FieldFill.INSERT) - private Long originUserId; - - @TableField(value = "teacher_model_ids") - private String teacherModelIds; - - @TableField(value = "student_model_ids") - private String studentModelIds; - -} - diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java deleted file mode 100644 index 4ab46a2..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptAlgorithmQueryVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化算法返回 - * @date 2021-01-05 - */ -@Data -public class ModelOptAlgorithmQueryVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("算法类型") - private Integer type; - - @ApiModelProperty("算法名称") - private String algorithm; - - @ApiModelProperty("算法路径") - private String algorithmPath; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java deleted file mode 100644 index 9ec1a69..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptCreateVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化任务创建返回 - * @date 2021-01-07 - */ -@Data -@AllArgsConstructor -public class ModelOptCreateVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("算法id") - private Long algorithmId; - - @ApiModelProperty("算法路径") - private String algorithmPath; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java deleted file mode 100644 index 8f5fb53..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetQueryVO.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化数据集查询返回 - * @date 2021-01-05 - */ -@Data -public class ModelOptDatasetQueryVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("数据集名称") - private String dataset; - @ApiModelProperty("数据集路径") - private String datasetPath; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java deleted file mode 100644 index 50b358e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptDatasetVO.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.dubhe.domain.entity.ModelOptDataset; - -import java.io.Serializable; - -/** - * @description 模型优化模块我的数据集查询 - * @date 2021-01-06 - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class ModelOptDatasetVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("主键") - private Long id; - - @ApiModelProperty("名称") - private String name; - - @ApiModelProperty("路径") - private String path; - - public ModelOptDatasetVO(ModelOptDataset modelOptDataset) { - this.id = modelOptDataset.getId(); - this.name = modelOptDataset.getName(); - this.path = modelOptDataset.getPath(); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java deleted file mode 100644 index e9db090..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptModelQueryVO.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化内置模型查询 - * @date 2021-01-06 - */ -@Data -public class ModelOptModelQueryVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("模型名称") - private String model; - - @ApiModelProperty("模型路径") - private String modelPath; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java deleted file mode 100644 index b4b085b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptResultQueryVO.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化结果对象 - * @date 2020-05-22 - */ -@Data -public class ModelOptResultQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - /** - * 参数名称 - */ - private String name; - - /** - * 结果对比 - */ - private String positive; - - /** - * 优化后值 - */ - private Float after; - - /** - * 优化前值 - */ - private Float before; - - /** - * 参数单位 - */ - private String unit; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java deleted file mode 100644 index d4c22fa..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskInstanceQueryVO.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import lombok.Data; -import org.dubhe.base.BaseVO; - -import java.sql.Timestamp; -import java.util.List; - -/** - * @description 模型优化任务实例 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskInstanceQueryVO extends BaseVO { - /** - * 主键 - */ - private Long id; - /** - * 任务id - */ - private Long taskId; - /** - * 任务名称 - */ - private String taskName; - - /** - * 是否内置 - */ - private Boolean isBuiltIn; - - /** - * 模型id - */ - private Long modelId; - /** - * 模型名称 - */ - private String modelName; - /** - * 模型路径 - */ - private String modelAddress; - /** - * 算法选择类型 - */ - private Integer algorithmType; - /** - * 优化算法 - */ - private String algorithmName; - /** - * 优化算法id - */ - private Long algorithmId; - /** - * 优化算法路径 - */ - private String algorithmPath; - /** - * 提交时间 - */ - private Timestamp startTime; - /** - * 完成时间 - */ - private Timestamp endTime; - /** - * 日志路径 - */ - private String logPath; - /** - * 任务状态 - */ - private String status; - /** - * 数据集id - */ - private Long datasetId; - /** - * 数据集名称 - */ - private String datasetName; - /** - * 数据集路径 - */ - private String datasetPath; - /** - * 运行命令 - */ - private String command; - /** - * 运行参数 - */ - private JSONObject params; - /** - * 容器名称 - */ - private String podName; - /** - * 模型输出路径 - */ - private String outputModelDir; - /** - * 优化结果 - */ - private List optResult; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java deleted file mode 100644 index c3ba79d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptTaskQueryVO.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import org.dubhe.base.BaseVO; - -/** - * @description 查询模型优化任务返回结果 - * @date 2020-05-22 - */ -@Data -public class ModelOptTaskQueryVO extends BaseVO { - - @ApiModelProperty("ID") - private Long id; - - @ApiModelProperty("任务名称") - private String name; - - @ApiModelProperty("任务描述") - private String description; - - @ApiModelProperty(value = "数据集id") - private Long datasetId; - - @ApiModelProperty("数据集名称") - private String datasetName; - - @ApiModelProperty("数据集路径") - private String datasetPath; - - @ApiModelProperty("是否内置") - private Boolean isBuiltIn; - - @ApiModelProperty("模型id") - private Long modelId; - - @ApiModelProperty("模型名称") - private String modelName; - - @ApiModelProperty("模型路径") - private String modelAddress; - - @ApiModelProperty("优化算法类型") - private Integer algorithmType; - - @ApiModelProperty("优化算法") - private String algorithmName; - - @ApiModelProperty("优化算法id") - private Long algorithmId; - - @ApiModelProperty("优化算法路径") - private String algorithmPath; - - @ApiModelProperty(value = "运行命令") - private String command; - - @ApiModelProperty(value = "运行参数") - private JSONObject params; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java deleted file mode 100644 index 7ab2e56..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelOptUpdateVO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 模型优化任务修改返回 - * @date 2021-01-08 - */ -@Data -@AllArgsConstructor -public class ModelOptUpdateVO implements Serializable { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("算法id") - private Long algorithmId; - - @ApiModelProperty("算法路径") - private String algorithmPath; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java deleted file mode 100644 index c69b72c..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/ModelVO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * @description 模型返回结果 - * @date 2021-01-14 - */ -@Data -@AllArgsConstructor -public class ModelVO { - - @ApiModelProperty("模型名称") - private String name; - - @ApiModelProperty("模型版本") - private String modelVersion; - - @ApiModelProperty("模型地址") - private String modelUrl; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java deleted file mode 100644 index d67ae3b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/NoteBookVO.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import cn.hutool.core.date.DatePattern; -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonIgnore; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.util.Date; - -/** - * @description 返回前端请求体 - * @create 2020/4/28 - */ -@Data -@ApiModel("NoteBookDTO 响应") -public class NoteBookVO implements Serializable { - - @ApiModelProperty("ID") - private Long id; - - @ApiModelProperty("所属用户") - private Long userId; - - @ApiModelProperty("NoteBook 名称") - @JsonIgnore - private String name; - - @ApiModelProperty("NoteBook 名称") - private String noteBookName; - - @ApiModelProperty("备注描述") - private String description; - - @ApiModelProperty("可访问jupyter地址") - private String url; - - @JsonIgnore - private Integer totalRunMin; - - @ApiModelProperty("CPU数量") - private Integer cpuNum; - - @ApiModelProperty("GPU数量") - private Integer gpuNum; - - @ApiModelProperty("内存大小（G）") - private Integer memNum; - - @ApiModelProperty("硬盘内存大小（G）") - private Integer diskMemNum; - - @ApiModelProperty("0运行，1停止, 2删除, 3启动中，4停止中，5删除中，6运行异常（暂未启用）") - private Integer status; - - @ApiModelProperty("k8s响应状态码") - private String k8sStatusCode; - - @ApiModelProperty("k8s响应状态信息") - private String k8sStatusInfo; - - @JsonIgnore - private String k8sNamespace; - - @JsonIgnore - private String k8sResourceName; - - private String k8sImageName; - - @ApiModelProperty("k8s中pvc存储路径") - private String k8sPvcPath; - - @JsonFormat(pattern = DatePattern.NORM_DATETIME_MS_PATTERN) - private Date createTime; - - @JsonIgnore - private Long createUserId; - - @JsonFormat(pattern = DatePattern.NORM_DATETIME_MS_PATTERN) - private Date updateTime; - - @JsonIgnore - private Long updateUserId; - - @ApiModelProperty("数据集名称") - private String dataSourceName; - - @ApiModelProperty("数据集路径") - private String dataSourcePath; - - @ApiModelProperty("算法ID") - private Long algorithmId; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java deleted file mode 100644 index 55e94da..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtDoJobResultVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import lombok.Data; -import lombok.experimental.Accessors; - -/** - * @description doJob返回封装 - * @date 2020-07-03 - */ -@Data -@Accessors(chain = true) -public class PtDoJobResultVO { - /** - * 返回是否成功 - */ - private Boolean isSuccess; - - /** - * 返回k8s返回的JobName - */ - private String k8sJobName; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java deleted file mode 100644 index 0800505..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageAndAlgorithmVO.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; - -/** - * @description 镜像 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtImageAndAlgorithmVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("镜像名称") - private String imageName; - - @ApiModelProperty("镜像地址") - private String imageUrl; - - @ApiModelProperty("代码目录") - private String codeDir; - - @ApiModelProperty("运行命令") - private String runCommand; - - @ApiModelProperty("输出目录") - private Boolean isTrainOut; - - @ApiModelProperty("输出日志") - private Boolean isTrainLog; - - @ApiModelProperty("输出可视化日志") - private Boolean isVisualizedLog; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java deleted file mode 100644 index 31c3ed7..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtImageQueryVO.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 返回镜像查询结果 - * @date 2020-04-27 - */ -@Data -public class PtImageQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - @ApiModelProperty("镜像ID") - private Long id; - - @ApiModelProperty("镜像项目名") - private String projectName; - - @ApiModelProperty("镜像名称") - private String imageName; - - @ApiModelProperty("镜像版本") - private String imageTag; - - @ApiModelProperty("镜像状态(0:上传中,1:上传成功,2:上传失败)") - private Integer imageStatus; - - @ApiModelProperty("备注") - private String remark; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; - - @ApiModelProperty("镜像来源") - private Integer imageResource; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java deleted file mode 100644 index dff8f4b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtJobMetricsGrafanaVO.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; - -/** - * @description Pod Metrics Grafana url - * @date 2020-06-30 - */ -@Data -@Accessors(chain = true) -public class PtJobMetricsGrafanaVO implements Serializable { - - @ApiModelProperty("job监控地址") - private String jobMetricsGrafanaUrl; - - @ApiModelProperty("job对应k8s中的podName") - private String jobPodName; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java deleted file mode 100644 index 00ad5ca..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtMeasureQueryVO.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 返回度量查询结果 - * @date 2020-11-16 - */ -@Data -public class PtMeasureQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("id") - private Long id; - - @ApiModelProperty("度量名称") - private String name; - - @ApiModelProperty("度量文件url") - private String url; - - @ApiModelProperty("度量描述") - private String description; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java deleted file mode 100644 index aed416f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmQueryVO.java +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 训练算法返回列表 - * @date 2020-04-27 - */ -@Data -public class PtTrainAlgorithmQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "算法ID") - private Long id; - - @ApiModelProperty(value = "算法名称") - private String algorithmName; - - @ApiModelProperty(value = "描述信息") - private String description; - - @ApiModelProperty(value = "算法来源") - private Integer algorithmSource; - - @ApiModelProperty(value = "镜像名称") - private String imageName; - - @ApiModelProperty(value = "算法文件大小") - private String algorithmFileSize; - - @ApiModelProperty(value = "镜像版本") - private String imageTag; - - @ApiModelProperty(value = "代码目录") - private String codeDir; - - @ApiModelProperty(value = "运行命令") - private String runCommand; - - @ApiModelProperty(value = "运行参数") - private JSONObject runParams; - - @ApiModelProperty(value = "算法用途") - private String algorithmUsage; - - @ApiModelProperty(value = "精度") - private String accuracy; - - @ApiModelProperty(value = "P4推理速度") - private Integer p4InferenceSpeed; - - @ApiModelProperty(value = "创建时间") - private Timestamp createTime; - - @ApiModelProperty(value = "输出结果（1是，0否）") - private Boolean isTrainOut; - - @ApiModelProperty(value = "输出日志（1是，0否)") - private Boolean isTrainLog; - - @ApiModelProperty(value = "可视化日志（1是，0否）") - private Boolean isVisualizedLog; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java deleted file mode 100644 index ef7e38d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainAlgorithmUsageQueryVO.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 算法用途返回列表 - * @date 2020-06-23 - */ -@Data -public class PtTrainAlgorithmUsageQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty(value = "ID") - private Long id; - - @ApiModelProperty(value = "类型") - private String type; - - @ApiModelProperty(value = "辅助信息") - private String auxInfo; - - @ApiModelProperty(value = "创建时间") - private Timestamp createTime; - - @ApiModelProperty(value = "更新时间") - private Timestamp updateTime; - - @ApiModelProperty(value = "是否为默认值") - private Boolean isDefault; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java deleted file mode 100644 index 8c62602..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainDataSourceStatusQueryVO.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.Setter; - -import java.io.Serializable; - -/** - * @description 查询数据集状态查询结果 - * @date 2020-05-21 - */ -@Data -@Setter -public class PtTrainDataSourceStatusQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("数据集名称") - private String dataSourceName; - - @ApiModelProperty("数据集路径") - private String dataSourcePath; - - @ApiModelProperty("数据集状态(true为当前没有训练在使用，可以删除；false为有训练使用,不可以删除)") - private Boolean status; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java deleted file mode 100644 index 9156b58..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDeleteVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 返回删除训练任务结果 - * @date 2020-04-28 - */ -@Data -public class PtTrainJobDeleteVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("trainID,如果只传递trainID,代表删除该trainID下的所有job") - private Long trainId; - - @ApiModelProperty("训练作业jobID") - private Long id; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java deleted file mode 100644 index 48c8a78..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailQueryVO.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 根据jobId查询训练任务详情返回结果 - * @date 2020-06-12 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobDetailQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练作业名") - private String trainName; - - @ApiModelProperty("jobID") - private Long id; - - @ApiModelProperty("训练作业ID") - private Long trainId; - - @ApiModelProperty("训练作业job版本") - private String trainVersion; - - @ApiModelProperty("训练作业job父版本") - private String parentTrainVersion; - - @ApiModelProperty("训练作业jobName") - private String jobName; - - @ApiModelProperty("描述信息") - private String description; - - @ApiModelProperty("数据集名称") - private String dataSourceName; - - @ApiModelProperty("数据集路径") - private String dataSourcePath; - - @ApiModelProperty("训练时长") - private String runtime; - - @ApiModelProperty("训练输出位置") - private String outPath; - - @ApiModelProperty("日志输出路径") - private String logPath; - - @ApiModelProperty("可视化日志路径") - private String visualizedLogPath; - - @ApiModelProperty("规格名称") - private String trainJobSpecsName; - - @ApiModelProperty("类型(0为CPU，1为GPU)") - private Integer resourcesPoolType; - - @ApiModelProperty("节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") - private Integer trainStatus; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("算法ID") - private Long algorithmId; - - @ApiModelProperty("镜像名称") - private String imageName; - - @ApiModelProperty(value = "镜像Project") - private String imageNameProject; - - @ApiModelProperty(value = "镜像版本") - private String imageTag; - - @ApiModelProperty("运行命令,输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty("F1值") - private String paramF1; - - @ApiModelProperty("召回率") - private String paramCallback; - - @ApiModelProperty("精确率") - private String paramPrecise; - - @ApiModelProperty("准确率") - private String paramAccuracy; - - @ApiModelProperty("算法名称") - private String algorithmName; - - @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") - private Integer algorithmSource; - - @ApiModelProperty("算法用途") - private String algorithmUsage; - - @ApiModelProperty("算法精度") - private String accuracy; - - @ApiModelProperty("P4推理速度（ms）") - private Integer p4InferenceSpeed; - - @ApiModelProperty(value = "算法文件路径") - private String algorithmCodeDir; - - @ApiModelProperty("训练类型 0：普通训练，1：分布式训练") - private Integer trainType; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty("训练延时启动倒计时，单位：分钟") - private Integer delayCreateCountDown; - - @ApiModelProperty("训练自动停止倒计时，单位：分钟") - private Integer delayDeleteCountDown; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; - - @ApiModelProperty("训练信息(失败信息)") - private String trainMsg; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") - private Integer modelResource; - - @ApiModelProperty(value = "非炼知模型id") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - private Long modelBranchId; - - @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") - private String teacherModelIds; - - @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java deleted file mode 100644 index c023b51..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobDetailVO.java +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 返回训练版本查询详情 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobDetailVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练作业名") - private String trainName; - - @ApiModelProperty("jobID") - private Long id; - - @ApiModelProperty("训练作业ID") - private Long trainId; - - @ApiModelProperty("训练作业job版本") - private String trainVersion; - - @ApiModelProperty("训练作业job父版本") - private String parentTrainVersion; - - @ApiModelProperty("训练作业jobName") - private String jobName; - - @ApiModelProperty("描述信息") - private String description; - - @ApiModelProperty("数据集名称") - private String dataSourceName; - - @ApiModelProperty("数据集路径") - private String dataSourcePath; - - @ApiModelProperty("训练时长") - private String runtime; - - @ApiModelProperty("训练输出位置") - private String outPath; - - @ApiModelProperty("日志输出路径") - private String logPath; - - @ApiModelProperty("可视化日志路径") - private String visualizedLogPath; - - @ApiModelProperty("规格名称") - private String trainJobSpecsName; - - @ApiModelProperty("类型(0为CPU，1为GPU)") - private Integer resourcesPoolType; - - @ApiModelProperty("节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") - private Integer trainStatus; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("算法ID") - private Long algorithmId; - - @ApiModelProperty("镜像名称") - private String imageName; - - @ApiModelProperty(value = "镜像Project") - private String imageNameProject; - - @ApiModelProperty(value = "镜像版本") - private String imageTag; - - @ApiModelProperty("运行命令,输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty("F1值") - private String paramF1; - - @ApiModelProperty("召回率") - private String paramCallback; - - @ApiModelProperty("精确率") - private String paramPrecise; - - @ApiModelProperty("准确率") - private String paramAccuracy; - - @ApiModelProperty("算法名称") - private String algorithmName; - - @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") - private Integer algorithmSource; - - @ApiModelProperty("算法用途") - private String algorithmUsage; - - @ApiModelProperty("算法精度") - private String accuracy; - - @ApiModelProperty("P4推理速度（ms）") - private Integer p4InferenceSpeed; - - @ApiModelProperty(value = "算法文件路径") - private String algorithmCodeDir; - - @ApiModelProperty("训练类型") - private Integer trainType; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty("训练延时启动倒计时，单位：分钟") - private Integer delayCreateCountDown; - - @ApiModelProperty("训练自动停止倒计时，单位：分钟") - private Integer delayDeleteCountDown; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") - private Integer modelResource; - - @ApiModelProperty(value = "非炼知模型id") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - private Long modelBranchId; - - @ApiModelProperty(value = "训练信息(失败信息)") - private String trainMsg; - - @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") - private String teacherModelIds; - - @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") - private String studentModelIds; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java deleted file mode 100644 index d92b1e1..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobModelVO.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @description 返回停止训练任务结果 - * @date 2020-04-28 - */ -@Data -public class PtTrainJobModelVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("非炼知模型信息") - private List modelList; - - @ApiModelProperty("炼知教师模型信息") - private List teacherModelList; - - @ApiModelProperty("炼知学生模型信息") - private List studentModelList; -} \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java deleted file mode 100644 index d7237ef..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobSpecsQueryVO.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; - -/** - * @description 训练作业规格 - * @date 2020-05-06 - */ -@Data -@Accessors(chain = true) -public class PtTrainJobSpecsQueryVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练作业规格ID") - private Integer id; - - @ApiModelProperty("训练作业规格名称") - private String specsName; - - @ApiModelProperty(value = "规格类型(0为CPU, 1为GPU)") - private Integer resourcesPoolType; - - @ApiModelProperty(value = "规格信息") - private JSONObject specsInfo; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java deleted file mode 100644 index 8ef03be..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStatisticsMineVO.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 我的训练任务统计 - * @date 2020-07-15 - */ -@Data -@ApiModel(description = "我的训练任务统计结果") -public class PtTrainJobStatisticsMineVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("运行中的任务") - private Integer runJobCount; - - @ApiModelProperty("已经完成的任务") - private Integer finishJobCount; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java deleted file mode 100644 index 6a30cde..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainJobStopVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @description 返回停止训练任务结果 - * @date 2020-04-28 - */ -@Data -public class PtTrainJobStopVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("trainID,如果只传递trainID,代表停止该trainID下的所有job") - private Long trainId; - - @ApiModelProperty("训练作业jobID") - private Long id; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java deleted file mode 100644 index 1435eb0..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainLogQueryVO.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.util.List; - -/** - * @description 训练日志 查询VO - * @date 2020-05-08 - */ -@Data -@Accessors(chain = true) -public class PtTrainLogQueryVO { - private static final long serialVersionUID = 1L; - - @ApiModelProperty("log内容") - private List content; - - @ApiModelProperty(value = "任务名称") - private String jobName; - - @ApiModelProperty(value = "起始行") - private Integer startLine; - - @ApiModelProperty(value = "结束行") - private Integer endLine; - - @ApiModelProperty(value = "查询行数") - private Integer lines; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java deleted file mode 100644 index bbf3226..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainParamQueryVO.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import com.alibaba.fastjson.JSONObject; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.dubhe.base.BaseVO; - -import java.io.Serializable; - -/** - * @description 任务参数查询返回查询结果 - * @date 2020-04-27 - */ -@EqualsAndHashCode(callSuper = true) -@Data -public class PtTrainParamQueryVO extends BaseVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("任务参数ID") - private Long id; - - @ApiModelProperty("任务参数名称") - private String paramName; - - @ApiModelProperty("描述") - private String description; - - @ApiModelProperty("镜像名称") - private String imageName; - - @ApiModelProperty(value = "镜像Project") - private String imageNameProject; - - @ApiModelProperty(value = "镜像版本") - private String imageTag; - - @ApiModelProperty("算法ID") - private Long algorithmId; - - @ApiModelProperty("运行命令,输入长度不能超过128个字符") - private String runCommand; - - @ApiModelProperty("算法名称") - private String algorithmName; - - @ApiModelProperty("算法来源(1为我的算法，2为预置算法)") - private Integer algorithmSource; - - @ApiModelProperty("数据来源路径") - private String dataSourcePath; - - @ApiModelProperty("数据来源名称") - private String dataSourceName; - - @ApiModelProperty("运行参数(算法来源为我的算法时为调优参数，算法来源为预置算法时为运行参数)") - private JSONObject runParams; - - @ApiModelProperty("规格名称") - private String trainJobSpecsName; - - @ApiModelProperty("类型(0为CPU，1为GPU)") - private Integer resourcesPoolType; - - @ApiModelProperty("训练类型") - private Integer trainType; - - @ApiModelProperty("节点个数") - private Integer resourcesPoolNode; - - @ApiModelProperty("验证数据来源名称") - private String valDataSourceName; - - @ApiModelProperty("验证数据来源路径") - private String valDataSourcePath; - - @ApiModelProperty("是否验证数据集") - private Integer valType; - - @ApiModelProperty(value = "模型类型(0我的模型1预置模型2炼知模型)") - private Integer modelResource; - - @ApiModelProperty(value = "模型id") - private Long modelId; - - @ApiModelProperty(value = "我的模型版本对应的id") - private Long modelBranchId; - - @ApiModelProperty(value = "炼知教师模型ids,多个id之前用','隔开") - private String teacherModelIds; - - @ApiModelProperty(value = "炼知学生模型ids,多个id之前用','隔开") - private String studentModelIds; - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java deleted file mode 100644 index 8281c15..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/domain/vo/PtTrainVO.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.domain.vo; - -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.experimental.Accessors; - -import java.io.Serializable; -import java.sql.Timestamp; - -/** - * @description 训练查询结果 - * @date 2020-04-27 - */ -@Data -@Accessors(chain = true) -public class PtTrainVO implements Serializable { - - private static final long serialVersionUID = 1L; - - @ApiModelProperty("训练作业jobID") - private Long jobId; - - @ApiModelProperty("训练作业ID") - private Long trainId; - - @ApiModelProperty("训练作业名") - private String trainName; - - @ApiModelProperty("训练作业jobName") - private String jobName; - - @ApiModelProperty("训练作业版本数") - private Integer versionNum; - - @ApiModelProperty("训练时长") - private String runtime; - - @ApiModelProperty("训练作业job状态, 0为待处理，1为运行中，2为运行完成，3为失败，4为停止，5为未知，6为删除，7为创建失败") - private Integer trainStatus; - - @ApiModelProperty("创建时间") - private Timestamp createTime; - - @ApiModelProperty("数据来源路径") - private String dataSourcePath; - - @ApiModelProperty("数据来源名称") - private String dataSourceName; - - @ApiModelProperty("资源拥有者ID") - private Long originUserId; -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java deleted file mode 100644 index 11ac292..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/AlgorithmStatusEnum.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.enums; - -/** - * @description 算法状态枚举 - * @date 2020-08-19 - */ -public enum AlgorithmStatusEnum { - - - /** - * 创建中 - */ - MAKING(0, "创建中"), - /** - * 创建成功 - */ - SUCCESS(1, "创建成功"), - /** - * 创建失败 - */ - FAIL(2, "创建失败"); - - - /** - * 编码 - */ - private Integer code; - - /** - * 描述 - */ - private String description; - - AlgorithmStatusEnum(int code, String description) { - this.code = code; - this.description = description; - } - - public Integer getCode() { - return code; - } - - public String getDescription() { - return description; - } -} - diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java deleted file mode 100644 index 3e085e5..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/DistillCommandEnum.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -/** - * @description 蒸馏算法命令行枚举 - * @date 2021-01-05 - */ -public enum DistillCommandEnum { - - /** - * 软标签蒸馏 - */ - KD("KD", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_kd.sh %s %s %s %s %s"), - /** - * 软标签蒸馏，将BERT蒸馏到BiLSTM - */ - BILSTM("Distilled-BiLSTM", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_distilled_lstm.sh %s %s %s %s %s"), - /** - * 软标签蒸馏+层与层蒸馏 - */ - PKD("BERT-PKD", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_bert_pkd.sh %s %s %s %s %s"), - /** - * 软标签蒸馏+层与层蒸馏+注意力蒸馏 - */ - TINY("TinyBERT", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_student_tinybert.sh %s %s %s %s %s"), - /** - * 依照概率替换原有的BERT模块和Theseus的模块组成新的模型来训练 - */ - THESEUS("BERT-Theseus", "cd /usr/local/algorithm && bash run_train_teacher.sh %s %s %s %s %s && bash run_train_theseus.sh %s %s %s %s %s"); - - - /** - * 算法名 - */ - private String name; - - /** - * 命令行 - */ - private String command; - - DistillCommandEnum(String name, String command) { - this.name = name; - this.command = command; - } - - /** - * 根据名称获取命令行 - * @param name 蒸馏算法名称 - * @return 返回命令行 - */ - public static String getCommandByName(String name) { - for (DistillCommandEnum commandEnum : DistillCommandEnum.values()) { - if (commandEnum.getName().equals(name)) { - return commandEnum.getCommand(); - } - } - return ""; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java deleted file mode 100644 index e3a80a8..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/NoteBookStatusEnum.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - - -import org.dubhe.base.MagicNumConstant; -import org.dubhe.k8s.enums.PodPhaseEnum; - -import java.util.ArrayList; -import java.util.List; - -/** - * @description notebook 状态枚举 - * @create 2020-4-28 - */ -public enum NoteBookStatusEnum { - /** - * 运行 - */ - RUN(0, "运行中", 0), - /** - * 停止 - */ - STOP(1, "停止", 0), - /** - * 删除 - */ - DELETE(2, "删除", 0), - /** - * 启动中 - */ - STARTING(3, "启动中", 1), - /** - * 停止中 - */ - STOPPING(4, "停止中", 1), - /** - * 删除中 - */ - DELETING(5, "删除中", 1), - /** - * 运行异常 - */ - ERROR(6, "运行异常", 0); - /** - * 编码 - */ - private Integer code; - /** - * 描述 - */ - private String description; - /** - * 删除标识 - */ - private Integer deleteType; - - public Integer getCode() { - return code; - } - - public void setCode(Integer code) { - this.code = code; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Integer getDeleteType() { - return deleteType; - } - - public void setDeleteType(Integer deleteType) { - this.deleteType = deleteType; - } - - NoteBookStatusEnum(int code, String description, int deleteType) { - this.code = code; - this.description = description; - this.deleteType = deleteType; - } - - public static String getDescription(Integer code) { - if (code != null) { - for (NoteBookStatusEnum en : NoteBookStatusEnum.values()) { - if (en.getCode().equals(code)) { - return en.getDescription(); - } - } - } - return null; - } - - /** - * k8s状态转换成NoteBook状态 - * 当查询不到k8s状态时即为删除 - * - * @param bizPodPhase - * @return - */ - public static NoteBookStatusEnum convert(String bizPodPhase) { - if (bizPodPhase == null - || PodPhaseEnum.DELETED.getPhase().equals(bizPodPhase) - || PodPhaseEnum.FAILED.getPhase().equals(bizPodPhase)) { - return STOP; - } else if (PodPhaseEnum.PENDING.getPhase().equals(bizPodPhase)) { - return STARTING; - } else if (PodPhaseEnum.RUNNING.getPhase().equals(bizPodPhase)) { - return RUN; - } else if (PodPhaseEnum.SUCCEEDED.getPhase().equals(bizPodPhase)) { - return STOPPING; - } else { - return ERROR; - } - } - - - /** - * 查找可以删除的状态集合 - * - * @return List - */ - public static List getCanDeleteStatus() { - List list = new ArrayList<>(); - for (NoteBookStatusEnum en : NoteBookStatusEnum.values()) { - if (en.getDeleteType().equals(MagicNumConstant.ONE)) { - list.add(en.getCode()); - } - } - return list; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java deleted file mode 100644 index 52aac0b..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/OptimizeTypeEnum.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.enums; - -import lombok.Getter; - -/** - * @description 优化算法类型 - * @date 2021-01-06 - */ -@Getter -public enum OptimizeTypeEnum { - - SLIMMING(0, "剪枝", "/ChannelSlimming"), - - DISTILL(1, "蒸馏", "/distil"), - - QUANTIFY(2, "量化", "/quantization"), - ; - - /** - * 类型值 - */ - private Integer type; - - /** - * 类型名称 - */ - private String name; - - /** - * 代码目录 - */ - private String codeDir; - - OptimizeTypeEnum(Integer type, String name, String codeDir) { - this.type = type; - this.name = name; - this.codeDir = codeDir; - } - - - public static String getCodeDirByType(Integer type) { - for (OptimizeTypeEnum optimizeTypeEnum : OptimizeTypeEnum.values()) { - if (optimizeTypeEnum.getType().equals(type)) { - return optimizeTypeEnum.getCodeDir(); - } - } - return ""; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java deleted file mode 100644 index 3282f7d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/ResourcesPoolTypeEnum.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.enums; - -/** - * @description 规格类型 - * @date 2020-07-15 - */ -public enum ResourcesPoolTypeEnum { - - CPU(0, "CPU"), - GPU(1, "GPU"); - - - /** - * 编码 - */ - private Integer code; - - /** - * 描述 - */ - private String description; - - ResourcesPoolTypeEnum(int code, String description) { - this.code = code; - this.description = description; - } - - public Integer getCode() { - return code; - } - - public String getDescription() { - return description; - } - - /** - * 是否是GPU编码 - * @param code - * @return true 是 ，false 否 - */ - public static boolean isGpuCode(Integer code){ - return GPU.getCode().equals(code); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java deleted file mode 100644 index 5769737..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/enums/TrainTypeEnum.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.enums; - -import lombok.Getter; - -/** - * @description 训练类型 - * @date 2020-08-31 - */ -@Getter -public enum TrainTypeEnum { - - /** - * 普通训练 - */ - JOB(0,"普通训练"), - /** - * 分布式训练 - */ - DISTRIBUTE_TRAIN(1,"分布式训练"), - ; - - - private Integer code; - - private String name; - - TrainTypeEnum(Integer code, String name) { - this.code = code; - this.name = name; - } - - /** - * 判断是否是分布式训练 - * @param trainType 训练类型 - * @return true 分布式训练，false 普通训练 - */ - public static boolean isDistributeTrain(int trainType){ - return DISTRIBUTE_TRAIN.getCode() == trainType; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java deleted file mode 100644 index a9b7991..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/BatchServingController.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.BatchServingCreateDTO; -import org.dubhe.domain.dto.BatchServingDeleteDTO; -import org.dubhe.domain.dto.BatchServingDetailDTO; -import org.dubhe.domain.dto.BatchServingQueryDTO; -import org.dubhe.domain.dto.BatchServingStartDTO; -import org.dubhe.domain.dto.BatchServingStopDTO; -import org.dubhe.domain.dto.BatchServingUpdateDTO; -import org.dubhe.service.BatchServingService; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.annotation.Resource; - -/** - * @description 批量服务管理 - * @date 2020-08-27 - */ -@Api(tags = "云端Serving：批量服务管理") -@RestController -@RequestMapping("/api/batchServing") -public class BatchServingController { - - @Resource - private BatchServingService batchServingService; - - @ApiOperation("查询批量服务") - @GetMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody query(BatchServingQueryDTO batchServingQueryDTO) { - return new DataResponseBody(batchServingService.query(batchServingQueryDTO)); - } - - @ApiOperation("创建批量服务") - @PostMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody create(@Validated @RequestBody BatchServingCreateDTO batchServingCreateDTO) { - return new DataResponseBody(batchServingService.create(batchServingCreateDTO)); - } - - @ApiOperation("修改批量服务") - @PutMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody update(@Validated @RequestBody BatchServingUpdateDTO batchServingUpdateDTO) { - return new DataResponseBody(batchServingService.update(batchServingUpdateDTO)); - } - - @ApiOperation("删除批量服务") - @DeleteMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody delete(@Validated @RequestBody BatchServingDeleteDTO batchServingDeleteDTO) { - return new DataResponseBody(batchServingService.delete(batchServingDeleteDTO)); - } - - @ApiOperation("启动批量服务") - @PostMapping("/start") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody start(@Validated @RequestBody BatchServingStartDTO batchServingStartDTO) { - return new DataResponseBody(batchServingService.start(batchServingStartDTO)); - } - - @ApiOperation("停止批量服务") - @PostMapping("/stop") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody stop(@Validated @RequestBody BatchServingStopDTO batchServingStopDTO) { - return new DataResponseBody(batchServingService.stop(batchServingStopDTO)); - } - - @ApiOperation("获取批量服务详情") - @GetMapping("/detail") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getDetail(@Validated BatchServingDetailDTO batchServingDetailDTO) { - return new DataResponseBody(batchServingService.getDetail(batchServingDetailDTO)); - } - - @GetMapping("/pod/{id}") - @ApiOperation("获取批量服务下pod信息") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getServingLog(@PathVariable Long id) { - return new DataResponseBody(batchServingService.getPods(id)); - } - - - @GetMapping("/queryById/{id}") - @ApiOperation("获取批量服务状态及进度") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody queryStatusAndProgress(@PathVariable Long id) { - return new DataResponseBody(batchServingService.queryStatusAndProgress(id)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java deleted file mode 100644 index 99865f4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ForwardController.java +++ /dev/null @@ -1,126 +0,0 @@ -/** Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import cn.hutool.core.util.StrUtil; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.client.RestTemplate; - -import javax.servlet.http.HttpServletRequest; -import java.net.URI; -import java.net.URISyntaxException; - -/** - * @description 代理转发 - * @date 2020-06-23 - */ - -@RestController -public class ForwardController { - @Value("${dubhe-proxy.visual.keyword}") - private String visual; - @Value("${dubhe-proxy.visual.server}") - private String visualServer; - @Value("${dubhe-proxy.visual.port}") - private String visualPort; - @Value("${dubhe-proxy.refine.keyword}") - private String refine; - @Value("${dubhe-proxy.refine.server}") - private String refineServer; - @Value("${dubhe-proxy.refine.port}") - private String refinePort; - - RestTemplate restTemplate = new RestTemplate(); - - /** - * 根据不同的请求拼上对应的转发路径 - * - * @param request http请求 - * @return URI 用于restTemplate的请求路径 - **/ - private URI getURI(HttpServletRequest request) throws URISyntaxException { - String requestURI = request.getRequestURI(); - String server = null; - String prefix = ""; - int port = 0; - if (requestURI.startsWith(StrUtil.SLASH + visual)) { - prefix = visual; - server = visualServer; - port = Integer.parseInt(visualPort); - } else if (requestURI.startsWith(StrUtil.SLASH + refine)) { - prefix = refine; - server = refineServer; - port = Integer.parseInt(refinePort); - } - - return new URI("http", null, server, port, requestURI.substring(prefix.length() + 1), request.getQueryString(), null); - } - - /** - * 获取请求中的Cookie - * - * @param request http请求 - * @return HttpHeaders 用于restTemplate的header - **/ - private HttpHeaders getHeader(HttpServletRequest request) { - String cookie = request.getHeader("Cookie"); - HttpHeaders httpHeaders = new HttpHeaders(); - if (null != cookie) { - httpHeaders.set("Cookie", cookie); - } - return httpHeaders; - } - - /** - * 转发get请求 - * - * @param request http请求 - * @return ResponseEntity 返回给前端的响应实体 - **/ - @GetMapping({StrUtil.SLASH + "${dubhe-proxy.visual.keyword}" + StrUtil.SLASH + "**", StrUtil.SLASH + "${dubhe-proxy.refine.keyword}" + StrUtil.SLASH + "**"}) - @ResponseBody - public ResponseEntity mirrorRest(HttpServletRequest request) throws URISyntaxException { - URI uri = getURI(request); - HttpHeaders httpHeaders = getHeader(request); - ResponseEntity responseEntity = - restTemplate.exchange(uri, HttpMethod.GET, new HttpEntity(httpHeaders), String.class); - return responseEntity; - } - - /** - * 转发get请求 - * - * @param request http请求 - * @param method 请求方法 - * @param body 请求体 - * @return ResponseEntity 返回给前端的响应实体 - **/ - @RequestMapping({StrUtil.SLASH + "${dubhe-proxy.visual.keyword}" + StrUtil.SLASH + "**", StrUtil.SLASH + "${dubhe-proxy.refine.keyword}" + StrUtil.SLASH + "**"}) - @ResponseBody - public ResponseEntity mirrorRest(HttpMethod method, HttpServletRequest request, @RequestBody String body) throws URISyntaxException { - URI uri = getURI(request); - HttpHeaders httpHeaders = getHeader(request); - ResponseEntity responseEntity = - restTemplate.exchange(uri, method, new HttpEntity(body, httpHeaders), String.class); - return responseEntity; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java deleted file mode 100644 index d8538c4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackDeploymentController.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import org.dubhe.base.DataResponseBody; -import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.DeploymentCallbackAsyncService; -import org.dubhe.utils.K8sCallBackTool; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -import javax.annotation.Resource; - -import static org.dubhe.constant.StringConstant.K8S_CALLBACK_PATH_DEPLOYMENT; - -/** - * @description k8s deployment回调 - * @date 2020-11-27 - */ -@Api(tags = "k8s回调：deployment") -@RestController -@RequestMapping(K8S_CALLBACK_PATH_DEPLOYMENT) -public class K8sCallbackDeploymentController { - - @Resource(name = "servingAsyncService") - private DeploymentCallbackAsyncService servingAsyncService; - - /** - * 云端serving在线服务异步回调 - * - * @param k8sToken - * @param k8sDeploymentCallbackReq - * @return - */ - @PostMapping(value = "/serving") - @ApiOperation("云端serving deployment 回调") - public DataResponseBody servingPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, - @Validated @RequestBody ServingK8sDeploymentCallbackCreateDTO k8sDeploymentCallbackReq) { - servingAsyncService.deploymentCallBack(k8sDeploymentCallbackReq); - return DataResponseFactory.success("云端serving在线服务异步回调中"); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java deleted file mode 100644 index 4489d6d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/K8sCallbackPodController.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import org.dubhe.base.DataResponseBody; -import org.dubhe.dto.callback.AlgorithmK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.NotebookK8sPodCallbackCreateDTO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.utils.K8sCallBackTool; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.annotation.Resource; - -import static org.dubhe.constant.StringConstant.K8S_CALLBACK_URI; - -/** - * @description k8s Pod 异步回调处理类 - * @date 2020-05-28 - */ -@Api(tags = "k8s回调：Pod") -@RestController -@RequestMapping(K8S_CALLBACK_URI) -public class K8sCallbackPodController { - - @Resource(name = "noteBookAsyncServiceImpl") - private PodCallbackAsyncService noteBookAsyncService; - @Resource(name = "algorithmAsyncServiceImpl") - private PodCallbackAsyncService algorithmAsyncServiceImpl; - @Resource(name = "modelOptAsyncServiceImpl") - private PodCallbackAsyncService modelOptAsyncService; - @Resource(name = "batchServingAsyncService") - private PodCallbackAsyncService batchServingAsyncService; - - /** - * notebook异步回调 - * - * @param k8sToken - * @param k8sPodCallbackReq - * @return - */ - @PostMapping(value = "/notebook") - @ApiOperation("模型管理 pod 回调") - public DataResponseBody notebookPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken - , @Validated @RequestBody NotebookK8sPodCallbackCreateDTO k8sPodCallbackReq) { - noteBookAsyncService.podCallBack(k8sPodCallbackReq); - return DataResponseFactory.success("notebook正在异步处理pod中。"); - } - - /** - * algorithm异步回调 - * - * @param k8sToken - * @param k8sPodCallbackReq - * @return - */ - @PostMapping(value = "/algorithm") - @ApiOperation("算法管理 pod 回调") - public DataResponseBody algorithmPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken - , @Validated @RequestBody AlgorithmK8sPodCallbackCreateDTO k8sPodCallbackReq) { - algorithmAsyncServiceImpl.podCallBack(k8sPodCallbackReq); - return DataResponseFactory.success("算法管理异步回调处理中"); - } - - /** - * modelOpt异步回调 - * - * @param k8sToken - * @param k8sPodCallbackReq - * @return - */ - @PostMapping(value = "/modelopt") - @ApiOperation("模型优化管理 pod 回调") - public DataResponseBody modelOptPodCallBack(@ApiParam(type = "head") @RequestHeader(name= K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, - @Validated @RequestBody ModelOptK8sPodCallbackCreateDTO k8sPodCallbackReq) { - modelOptAsyncService.podCallBack(k8sPodCallbackReq); - return DataResponseFactory.success("model-opt 正在异步处理pod中。"); - } - - /** - * 云端serving批量服务异步回调 - * - * @param k8sToken - * @param k8sPodCallbackReq - * @return - */ - @PostMapping(value = "/batchserving") - @ApiOperation("云端serving pod 回调") - public DataResponseBody servingPodCallBack(@ApiParam(type = "head") @RequestHeader(name = K8sCallBackTool.K8S_CALLBACK_TOKEN) String k8sToken, - @Validated @RequestBody BatchServingK8sPodCallbackCreateDTO k8sPodCallbackReq) { - batchServingAsyncService.podCallBack(k8sPodCallbackReq); - return DataResponseFactory.success("云端serving批量服务异步回调处理中"); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java deleted file mode 100644 index 23f5dba..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskController.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; -import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.ModelOptTaskService; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.annotation.Resource; - -/** - * @description 模型优化任务 - * @date 2020-05-22 - */ -@Api(tags = "模型优化：任务") -@RestController -@RequestMapping("/api/modelOpt/task") -public class ModelOptTaskController { - - @Resource - private ModelOptTaskService modelOptTaskService; - - @GetMapping - @ApiOperation("任务列表分页查询") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getModelOptList(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { - return DataResponseFactory.success(modelOptTaskService.queryAll(modelOptTaskQueryDTO)); - } - - @PostMapping - @ApiOperation("创建任务") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody create(@Validated @RequestBody ModelOptTaskCreateDTO modelOptTaskCreateDTO) { - return DataResponseFactory.success(modelOptTaskService.create(modelOptTaskCreateDTO)); - } - - @PostMapping("/submit") - @ApiOperation("提交任务，创建实例") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody submit(@Validated @RequestBody ModelOptTaskSubmitDTO submitDTO) { - modelOptTaskService.submit(submitDTO); - return DataResponseFactory.success(); - } - - @PutMapping - @ApiOperation("编辑模型优化任务") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody update(@Validated @RequestBody ModelOptTaskUpdateDTO modelOptTaskUpdateDTO) { - return DataResponseFactory.success(modelOptTaskService.update(modelOptTaskUpdateDTO)); - } - - @DeleteMapping - @ApiOperation("删除模型优化任务") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody delete(@Validated @RequestBody ModelOptTaskDeleteDTO modelOptTaskDeleteDTO) { - modelOptTaskService.delete(modelOptTaskDeleteDTO); - return DataResponseFactory.success(); - } - - @GetMapping(value = "/getBuiltInModel") - @ApiOperation("获取全部内置模型") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getBuiltInModel(Integer type, String dataset, String algorithm) { - return DataResponseFactory.success(modelOptTaskService.getBuiltInModel(type, dataset, algorithm)); - } - - @GetMapping(value = "/getAlgorithm") - @ApiOperation("获取模型优化算法") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getAlgorithm(Integer type, String model, String dataset) { - return DataResponseFactory.success(modelOptTaskService.getAlgorithm(type, model, dataset)); - } - - @GetMapping(value = "/getDataset") - @ApiOperation("获取内置模型优化数据集") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getDataset(Integer type, String model, String algorithm) { - return DataResponseFactory.success(modelOptTaskService.getDataset(type, model, algorithm)); - } - - @GetMapping(value = "/MyDataset") - @ApiOperation("获取我的模型优化数据集") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getMyDataset() { - return DataResponseFactory.success(modelOptTaskService.getMyDataset()); - } - - @PostMapping(value = "/MyDataset") - @ApiOperation("创建我的模型优化数据集") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody createMyDataset(@Validated @RequestBody ModelOptDatasetCreateDTO modelOptDatasetCreateDTO) { - return DataResponseFactory.success(modelOptTaskService.createMyDataset(modelOptDatasetCreateDTO)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java deleted file mode 100644 index 82f487d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ModelOptTaskInstanceController.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.ModelOptTaskInstanceService; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.annotation.Resource; - -/** - * @description 模型优化任务实例 - * @date 2020-05-22 - */ -@Api(tags = "模型优化：任务实例") -@RestController -@RequestMapping("/api/modelOpt/taskInstance") -public class ModelOptTaskInstanceController { - - @Resource - private ModelOptTaskInstanceService modelOptTaskInstanceService; - - @ApiOperation("分页查询任务执行记录实例列表") - @GetMapping - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody query(ModelOptTaskInstanceQueryDTO instanceQueryDTO) { - return DataResponseFactory.success(modelOptTaskInstanceService.queryAll(instanceQueryDTO)); - } - - @ApiOperation("重新提交任务实例") - @PostMapping(value = "/resubmit") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody resubmit(@Validated @RequestBody ModelOptTaskInstanceResubmitDTO resubmitDTO) { - modelOptTaskInstanceService.resubmit(resubmitDTO); - return DataResponseFactory.success(); - } - - @ApiOperation("取消模型优化任务实例") - @PutMapping(value = "/cancel") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody cancel(@Validated @RequestBody ModelOptTaskInstanceCancelDTO cancelDTO) { - modelOptTaskInstanceService.cancel(cancelDTO); - return DataResponseFactory.success(); - } - - @ApiOperation("查看任务实例详情") - @GetMapping(value = "/detail") - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody getInstDetail(@Validated ModelOptTaskInstanceDetailDTO detailDTO) { - return DataResponseFactory.success(modelOptTaskInstanceService.getInstDetail(detailDTO)); - } - - @ApiOperation("删除任务实例") - @DeleteMapping - @RequiresPermissions(Permissions.MODEL_OPTIMIZE) - public DataResponseBody delete(@Validated @RequestBody ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO) { - modelOptTaskInstanceService.delete(modelOptTaskInstanceDeleteDTO); - return DataResponseFactory.success(); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java deleted file mode 100644 index 12d7202..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/NoteBookController.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import io.swagger.annotations.*; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.ResponseCode; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.domain.dto.NoteBookListQueryDTO; -import org.dubhe.domain.dto.SourceNoteBookDTO; -import org.dubhe.domain.dto.NoteBookCreateDTO; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.NoteBookService; -import org.dubhe.utils.NotebookUtil; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -import java.util.List; -import java.util.Set; - -/** - * @description notebook controller - * @date 2020-04-27 - */ -@Api(tags = "模型开发：Notebook") -@RestController -@RequestMapping("/api/{version}/notebook") -public class NoteBookController { - - @Autowired - private NoteBookService noteBookService; - - @ApiOperation("新增加notebook") - @PostMapping(value = "/notebooks") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody createNoteBook(@Validated @RequestBody NoteBookCreateDTO noteBookCreateDTO) { - long userId = NotebookUtil.getCurUserId(); - NoteBook noteBook = new NoteBook(); - BeanUtils.copyProperties(noteBookCreateDTO, noteBook); - noteBook.setCreateUserId(userId); - noteBook.setUserId(userId); - noteBook.setCreateResource(BizNfsEnum.NOTEBOOK.getCreateResource()); - return new DataResponseBody(noteBookService.createNoteBook(noteBook)); - } - - @ApiOperation("查询notebook") - @GetMapping(value = "/notebooks") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO) { - return new DataResponseBody(noteBookService.getNoteBookList(page, noteBookListQueryDTO)); - } - - @ApiOperation("异步批量删除notebook") - @DeleteMapping - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody deleteNoteBook(@RequestBody Set noteBookIds) { - List noteBookList = noteBookService.validateDeleteNoteBook(noteBookIds); - noteBookService.deleteNoteBooks(noteBookList); - return new DataResponseBody(ResponseCode.SUCCESS, "正在异步删除NoteBook中.."); - } - - @ApiOperation("启动notebook") - @PutMapping(value = "/start") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody startNotebook(@RequestParam(required = true) Long noteBookId) { - String resultInfo = noteBookService.startNoteBook(noteBookId); - return new DataResponseBody( - NotebookUtil.validateFailedInfo(resultInfo) ? ResponseCode.ERROR : ResponseCode.SUCCESS - , resultInfo); - } - - @ApiOperation("停止notebook") - @PutMapping(value = "/stop") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody stopNotebook(@RequestParam(required = true) Long noteBookId) { - String resultInfo = noteBookService.stopNoteBook(noteBookId); - return new DataResponseBody( - NotebookUtil.validateFailedInfo(resultInfo) ? ResponseCode.ERROR : ResponseCode.SUCCESS - , resultInfo); - } - - - @ApiOperation("打开notebook") - @GetMapping(value = "/{id}") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody openNotebook(@PathVariable(name = "id", required = true) Long noteBookId) { - return DataResponseFactory.success("获取成功", noteBookService.openNoteBook(noteBookId)); - } - - - @ApiOperation("第三方业务创建notebook") - @PostMapping(value = "/create/{source}") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody createByThirdParty(@PathVariable int source, @Validated @RequestBody SourceNoteBookDTO sourceNoteBookDTO) { - BizNfsEnum bizNfsEnum = BizNfsEnum.getByCreateResource(source); - if (bizNfsEnum == null || BizNfsEnum.NOTEBOOK == bizNfsEnum) { - return new DataResponseBody(ResponseCode.BADREQUEST, "不支持该渠道创建Notebook！"); - } - sourceNoteBookDTO.setCurUserId(NotebookUtil.getCurUserId()); - return DataResponseFactory.success("操作成功", noteBookService.createNoteBookByThirdParty(bizNfsEnum, sourceNoteBookDTO)); - } - - @ApiOperation("获取编辑地址") - @GetMapping(value = "/{id}/get-address") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getAddress(@PathVariable(name = "id", required = true) Long noteBookId) { - return new DataResponseBody(noteBookService.getAddress(noteBookId)); - } - - @ApiOperation("获取状态") - @GetMapping(value = "/status") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getNoteBookStatus() { - return new DataResponseBody(noteBookService.getNoteBookStatus()); - } - - - @ApiOperation("获取正在运行的notebook数量") - @GetMapping(value = "/run-number") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getNoteBookRunNumber() { - return new DataResponseBody(noteBookService.getNoteBookRunNumber()); - } - - - @ApiOperation("获取notebook模板") - @GetMapping(value = "/notebook-model") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getNoteBookModel() { - return new DataResponseBody(noteBookService.getNoteBookModel()); - } - - - @ApiOperation("获取notebook详情") - @PostMapping(value = "/detail") - @RequiresPermissions(Permissions.DEVELOPMENT_NOTEBOOK) - public DataResponseBody getNotebookDetail(@RequestBody Set noteBookIds) { - return new DataResponseBody(noteBookService.getNotebookDetail(noteBookIds)); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java deleted file mode 100644 index 08c289d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtImageController.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.*; -import org.dubhe.service.PtImageService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -/** - * @description 镜像 - * @date 2020-04-27 - */ -@Api(tags = "镜像：镜像管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/ptImage") -public class PtImageController { - - @Autowired - private PtImageService ptImageService; - - @GetMapping("/info") - @ApiOperation("查询镜像") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getImage(PtImageQueryDTO ptImageQueryDTO) { - return new DataResponseBody(ptImageService.getImage(ptImageQueryDTO)); - } - - @ApiOperation("通过projectName查询镜像") - @GetMapping - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getTagsByImageName(@RequestParam Integer projectType, @RequestParam String imageName) { - return new DataResponseBody(ptImageService.searchImages(projectType, imageName)); - } - - @PostMapping("uploadImage") - @ApiOperation("上传镜像包到harbor") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody uploadImage(@Validated @RequestBody PtImageUploadDTO ptImageUploadDTO) { - ptImageService.uploadImage(ptImageUploadDTO); - return new DataResponseBody(); - } - - @DeleteMapping - @ApiOperation("删除镜像") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody deleteTrainImage(@Validated @RequestBody PtImageDeleteDTO ptImageDeleteDTO) { - ptImageService.deleteTrainImage(ptImageDeleteDTO); - return new DataResponseBody(); - } - - @PutMapping - @ApiOperation("修改镜像信息") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody updateTrainImage(@Validated @RequestBody PtImageUpdateDTO ptImageUpdateDTO) { - ptImageService.updateTrainImage(ptImageUpdateDTO); - return new DataResponseBody(); - } - - @GetMapping("/imageNameList") - @ApiOperation("获取镜像名称列表") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getImageNameList(@RequestParam Integer projectType) { - return new DataResponseBody(ptImageService.getImageNameList(projectType)); - } - - @PutMapping("/imageResource") - @ApiOperation("修改镜像来源(notebook定制)") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody updateImageResource(@RequestParam Long id) { - ptImageService.updImageResource(id); - return new DataResponseBody(); - } - - @GetMapping("/imageUrl") - @ApiOperation("查询镜像url") - @RequiresPermissions(Permissions.TRAINING_IMAGE) - public DataResponseBody getImageUrl(PtImageQueryUrlDTO ptImageQueryUrlDTO) { - return new DataResponseBody(ptImageService.getImageUrl(ptImageQueryUrlDTO)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java deleted file mode 100644 index 81c1b24..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtMeasureController.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.domain.dto.PtMeasureDTO; -import org.dubhe.domain.dto.PtMeasureDeleteDTO; -import org.dubhe.domain.dto.PtMeasureQueryDTO; -import org.dubhe.domain.dto.PtMeasureUpdateDTO; -import org.dubhe.service.PtMeasureService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -/** - * @description 度量管理 - * @date 2020-11-16 - */ -@Api(tags = "度量：度量管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/ptMeasure") -public class PtMeasureController { - - @Autowired - private PtMeasureService ptMeasureService; - - @GetMapping - @ApiOperation("查询度量") - public DataResponseBody getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO) { - return new DataResponseBody(ptMeasureService.getMeasure(ptMeasureQueryDTO)); - } - - @PostMapping - @ApiOperation("新建度量") - public DataResponseBody addMeasure(@Validated @RequestBody PtMeasureDTO ptMeasureCreateDTO) { - ptMeasureService.createMeasure(ptMeasureCreateDTO); - return new DataResponseBody(); - } - - @PutMapping - @ApiOperation("修改度量") - public DataResponseBody updateMeasure(@Validated @RequestBody PtMeasureUpdateDTO ptMeasureUpdateDTO) { - ptMeasureService.updateMeasure(ptMeasureUpdateDTO); - return new DataResponseBody(); - } - - @DeleteMapping - @ApiOperation("删除度量") - public DataResponseBody deleteMeasure(@Validated @RequestBody PtMeasureDeleteDTO PtMeasureUpdateDTO) { - ptMeasureService.deleteMeasure(PtMeasureUpdateDTO); - return new DataResponseBody(); - } - - @GetMapping("/byName") - @ApiOperation("通过度量名称查询") - public DataResponseBody getMeasureByName(@RequestParam String name) { - return new DataResponseBody(ptMeasureService.getMeasureByName(name)); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java deleted file mode 100644 index 5cc6fec..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmController.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.*; -import org.dubhe.service.PtTrainAlgorithmService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -/** - * @description 训练算法 - * @date 2020-04-27 - */ -@Api(tags = "训练：算法管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/algorithm") -public class PtTrainAlgorithmController { - - @Autowired - private PtTrainAlgorithmService ptTrainAlgorithmService; - - @GetMapping - @ApiOperation("查询算法") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody getAlgorithms(@Validated PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO) { - return new DataResponseBody(ptTrainAlgorithmService.queryAll(ptTrainAlgorithmQueryDTO)); - } - - @GetMapping("/myAlgorithmCount") - @ApiOperation("查询当前用户的算法个数") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody getAlgorithmCount() { - return new DataResponseBody(ptTrainAlgorithmService.getAlgorithmCount()); - } - - @PostMapping - @ApiOperation("新增算法") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody create(@Validated @RequestBody PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { - return new DataResponseBody(ptTrainAlgorithmService.create(ptTrainAlgorithmCreateDTO)); - } - - @PutMapping - @ApiOperation("修改算法") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody update(@Validated @RequestBody PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { - return new DataResponseBody(ptTrainAlgorithmService.update(ptTrainAlgorithmUpdateDTO)); - } - - @DeleteMapping - @ApiOperation("删除算法") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody deleteAll(@Validated @RequestBody PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO) { - ptTrainAlgorithmService.deleteAll(ptTrainAlgorithmDeleteDTO); - return new DataResponseBody(); - } - - @PostMapping("/uploadAlgorithm") - @ApiOperation("模型优化上传算法") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody modelOptimizationUploadAlgorithm(@Validated @RequestBody PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO) { - return new DataResponseBody(ptTrainAlgorithmService.modelOptimizationUploadAlgorithm(ptModelAlgorithmCreateDTO)); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java deleted file mode 100644 index 299f3c4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainAlgorithmUsageController.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.constant.UserAuxiliaryInfoConstant; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageQueryDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.PtTrainAlgorithmUsageService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -/** - * @description 算法用途管理 - * @date 2020-06-19 - */ -@Api(tags = "训练：算法用途管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/algorithmUsage") -public class PtTrainAlgorithmUsageController { - - @Autowired - private PtTrainAlgorithmUsageService ptTrainAlgorithmUsageService; - - @GetMapping - @ApiOperation("算法用途列表展示") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody queryAll(@Validated PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO) { - ptTrainAlgorithmUsageQueryDTO.setType(UserAuxiliaryInfoConstant.ALGORITHM_USAGE); - return DataResponseFactory - .success(ptTrainAlgorithmUsageService.queryAll(ptTrainAlgorithmUsageQueryDTO)); - } - - @PostMapping - @ApiOperation("新增算法用途") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody create( - @Validated @RequestBody PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO) { - ptTrainAlgorithmUsageCreateDTO.setType(UserAuxiliaryInfoConstant.ALGORITHM_USAGE); - return DataResponseFactory.success(ptTrainAlgorithmUsageService.create(ptTrainAlgorithmUsageCreateDTO)); - } - - @DeleteMapping - @ApiOperation("删除算法用途") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody deleteAll(@Validated @RequestBody PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO) { - ptTrainAlgorithmUsageService.deleteAll(ptTrainAlgorithmUsageDeleteDTO); - return new DataResponseBody(); - } - - @PutMapping - @ApiOperation("修改算法用途") - @RequiresPermissions(Permissions.DEVELOPMENT_ALGORITHM) - public DataResponseBody update( - @Validated @RequestBody PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO) { - ptTrainAlgorithmUsageService.update(ptTrainAlgorithmUsageUpdateDTO); - return new DataResponseBody(); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java deleted file mode 100644 index 5a0dec7..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainJobController.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.Logical; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.*; -import org.dubhe.enums.TrainTypeEnum; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.PtTrainJobService; -import org.dubhe.service.PtTrainJobSpecsService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - - -/** - * @description 训练作业job - * @date 2020-04-27 - */ -@Api(tags = "训练：任务管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/trainJob") -public class PtTrainJobController { - - @Autowired - private PtTrainJobService ptTrainJobService; - - @Autowired - private PtTrainJobSpecsService ptTrainJobSpecsService; - - @GetMapping - @ApiOperation("作业列表展示") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainJob(@Validated PtTrainQueryDTO ptTrainQueryDTO) { - return new DataResponseBody(ptTrainJobService.getTrainJob(ptTrainQueryDTO)); - } - - @GetMapping("/trainJobSpecs") - @ApiOperation("规格展示") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainJobSpecs(@Validated PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO) { - return new DataResponseBody(ptTrainJobSpecsService.getTrainJobSpecs(ptTrainJobSpecsQueryDTO)); - } - - @GetMapping("/jobDetail") - @ApiOperation("根据jobId查询训练任务详情") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainJobDetail(@Validated PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO) { - return new DataResponseBody(ptTrainJobService.getTrainJobDetail(ptTrainJobDetailQueryDTO)); - } - - @GetMapping("/mine") - @ApiOperation(value = "我的训练任务统计", notes = "运行中的任务:PENDDING,RUNNING；完成的任务:其他状态") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody statisticsMine() { - return new DataResponseBody(ptTrainJobService.statisticsMine()); - } - - @GetMapping("/trainJobVersionDetail") - @ApiOperation("作业不同版本任务列表展示") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainJobVersion(@Validated PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO) { - return new DataResponseBody(ptTrainJobService.getTrainJobVersion(ptTrainJobVersionQueryDTO)); - } - - @GetMapping("/dataSourceStatus") - @ApiOperation("数据集状态展示") - @RequiresPermissions(value = {Permissions.TRAINING_JOB, Permissions.DATA}, logical = Logical.OR) - public DataResponseBody getTrainDataSourceStatus(@Validated PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO) { - return new DataResponseBody(ptTrainJobService.getTrainDataSourceStatus(ptTrainDataSourceStatusQueryDTO)); - } - - @PostMapping - @ApiOperation("创建训练任务") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody createTrainJob(@Validated @RequestBody PtTrainJobCreateDTO ptTrainJobCreateDTO) { - if (TrainTypeEnum.isDistributeTrain(ptTrainJobCreateDTO.getTrainType()) - && ptTrainJobCreateDTO.getResourcesPoolNode() < 2) { - // 分布式训练节点数校验补偿 - return DataResponseFactory.failed("分布式训练节点个数至少2个"); - } - return new DataResponseBody(ptTrainJobService.createTrainJobVersion(ptTrainJobCreateDTO)); - } - - @PutMapping - @ApiOperation("修改训练任务") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody updateTrainJob(@Validated @RequestBody PtTrainJobUpdateDTO ptTrainJobUpdateDTO) { - return new DataResponseBody(ptTrainJobService.updateTrainJob(ptTrainJobUpdateDTO)); - } - - @DeleteMapping - @ApiOperation("删除训练任务") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody deleteTrainJob(@Validated @RequestBody PtTrainJobDeleteDTO ptTrainJobDeleteDTO) { - return new DataResponseBody(ptTrainJobService.deleteTrainJob(ptTrainJobDeleteDTO)); - } - - @PostMapping("/stop") - @ApiOperation("停止训练任务") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody stopTrainJob(@Validated @RequestBody PtTrainJobStopDTO ptTrainJobStopDTO) { - return new DataResponseBody(ptTrainJobService.stopTrainJob(ptTrainJobStopDTO)); - } - - @PostMapping("/resume") - @ApiOperation("恢复训练任务") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody resumeTrainJob(@Validated @RequestBody PtTrainJobResumeDTO ptTrainJobResumeDTO) { - ptTrainJobService.resumeTrainJob(ptTrainJobResumeDTO); - return new DataResponseBody(); - } - - @GetMapping("/grafanaUrl/{jobId}") - @ApiOperation("获取job在grafana监控的地址") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getGrafanaUrl( @PathVariable Long jobId) { - return new DataResponseBody(ptTrainJobService.getGrafanaUrl(jobId)); - } - - @GetMapping("/model") - @ApiOperation("获取job使用的模型") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainJobModel(@Validated PtTrainModelDTO ptTrainModelDTO) { - return new DataResponseBody(ptTrainJobService.getTrainJobModel(ptTrainModelDTO)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java deleted file mode 100644 index f2b28c0..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainLogController.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.PtTrainLogQueryDTO; -import org.dubhe.domain.vo.PtTrainLogQueryVO; -import org.dubhe.factory.DataResponseFactory; -import org.dubhe.service.PtTrainLogService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @description 训练日志 控制器入口 - * @date 2020-05-08 - */ -@Api(tags = "训练：训练日志") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/trainLog") -public class PtTrainLogController { - - @Autowired - private PtTrainLogService ptTrainLogService; - - @GetMapping - @ApiOperation("训练日志查询") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainLog(@Validated PtTrainLogQueryDTO ptTrainLogQueryDTO) { - return new DataResponseBody(ptTrainLogService.queryTrainLog(ptTrainLogQueryDTO)); - } - - @GetMapping("/download") - @ApiOperation("训练日志下载") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody downLoadTrainLog(@Validated PtTrainLogQueryDTO ptTrainLogQueryDTO) { - - ptTrainLogQueryDTO.setStartLine(MagicNumConstant.ONE).setLines(MagicNumConstant.MILLION); - - PtTrainLogQueryVO ptTrainLogQueryVO = ptTrainLogService.queryTrainLog(ptTrainLogQueryDTO); - return new DataResponseBody(ptTrainLogService.getTrainLogString(ptTrainLogQueryVO.getContent())); - } - - @GetMapping("/pod/{id}") - @ApiOperation("获取pod节点") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getPods(@PathVariable Long id) { - return DataResponseFactory.success(ptTrainLogService.getPods(id)); - } - - - - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java deleted file mode 100644 index 05790fb..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/PtTrainParamController.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.annotation.ApiVersion; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.PtTrainParamCreateDTO; -import org.dubhe.domain.dto.PtTrainParamDeleteDTO; -import org.dubhe.domain.dto.PtTrainParamQueryDTO; -import org.dubhe.domain.dto.PtTrainParamUpdateDTO; -import org.dubhe.service.PtTrainParamService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.*; - -/** - * @description 任务参数 - * @date 2020-04-27 - */ -@Api(tags = "训练：任务参数管理") -@RestController -@ApiVersion(1) -@RequestMapping("/api/{version}/trainParams") -public class PtTrainParamController { - - @Autowired - private PtTrainParamService ptTrainParamService; - - @GetMapping - @ApiOperation("任务参数列表展示") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody getTrainParam(@Validated PtTrainParamQueryDTO ptTrainParamQueryDTO) { - return new DataResponseBody(ptTrainParamService.getTrainParam(ptTrainParamQueryDTO)); - } - - @PostMapping - @ApiOperation("保存任务参数") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody createTrainParam(@Validated @RequestBody PtTrainParamCreateDTO ptTrainParamCreateDTO) { - return new DataResponseBody(ptTrainParamService.createTrainParam(ptTrainParamCreateDTO)); - } - - @PutMapping - @ApiOperation("修改任务参数") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody updateTrainParam(@Validated @RequestBody PtTrainParamUpdateDTO ptTrainParamUpdateDTO) { - return new DataResponseBody(ptTrainParamService.updateTrainParam(ptTrainParamUpdateDTO)); - } - - @DeleteMapping - @ApiOperation("删除任务参数") - @RequiresPermissions(Permissions.TRAINING_JOB) - public DataResponseBody deleteTrainParam(@Validated @RequestBody PtTrainParamDeleteDTO ptTrainParamDeleteDTO) { - ptTrainParamService.deleteTrainParam(ptTrainParamDeleteDTO); - return new DataResponseBody(); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java deleted file mode 100644 index 09baeae..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/rest/ServingController.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.rest; - -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.dubhe.base.DataResponseBody; -import org.dubhe.constant.Permissions; -import org.dubhe.domain.dto.PredictParamDTO; -import org.dubhe.domain.dto.ServingInfoCreateDTO; -import org.dubhe.domain.dto.ServingInfoDeleteDTO; -import org.dubhe.domain.dto.ServingInfoDetailDTO; -import org.dubhe.domain.dto.ServingInfoQueryDTO; -import org.dubhe.domain.dto.ServingInfoUpdateDTO; -import org.dubhe.domain.dto.ServingStartDTO; -import org.dubhe.domain.dto.ServingStopDTO; -import org.dubhe.service.ServingService; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; - -import javax.annotation.Resource; - -/** - * @description 在线服务管理 - * @date 2020-08-24 - */ -@Api(tags = "云端Serving：在线服务管理") -@RestController -@RequestMapping("/api/serving") -public class ServingController { - - @Resource - private ServingService servingService; - - @ApiOperation("查询服务") - @GetMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody query(ServingInfoQueryDTO servingInfoQueryDTO) { - return new DataResponseBody(servingService.query(servingInfoQueryDTO)); - } - - @ApiOperation("创建服务") - @PostMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody create(@Validated @RequestBody ServingInfoCreateDTO servingInfoCreateDTO) { - return new DataResponseBody(servingService.create(servingInfoCreateDTO)); - } - - @ApiOperation("修改服务") - @PutMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody update(@Validated @RequestBody ServingInfoUpdateDTO servingInfoUpdateDTO) { - return new DataResponseBody(servingService.update(servingInfoUpdateDTO)); - } - - @ApiOperation("删除服务") - @DeleteMapping - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody delete(@Validated @RequestBody ServingInfoDeleteDTO servingInfoDeleteDTO) { - return new DataResponseBody(servingService.delete(servingInfoDeleteDTO)); - } - - @ApiOperation("获取服务详情") - @GetMapping("/detail") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getDetail(@Validated ServingInfoDetailDTO servingInfoDetailDTO) { - return new DataResponseBody(servingService.getDetail(servingInfoDetailDTO)); - } - - @ApiOperation("启动服务") - @PostMapping("/start") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody start(@Validated @RequestBody ServingStartDTO servingStartDTO) { - return new DataResponseBody(servingService.start(servingStartDTO)); - } - - @ApiOperation("停止服务") - @PostMapping("/stop") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody stop(@Validated @RequestBody ServingStopDTO servingStopDTO) { - return new DataResponseBody(servingService.stop(servingStopDTO)); - } - - @ApiOperation("获取接口参数") - @GetMapping("/predictParam") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getPredictParam(@Validated PredictParamDTO predictParamDTO) { - return new DataResponseBody(servingService.getPredictParam(predictParamDTO)); - } - - @GetMapping("/servingConfig/pod/{modelConfigId}") - @ApiOperation("获取modelConfigId下pod信息") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getServingLog(@PathVariable Long modelConfigId) { - return new DataResponseBody(servingService.getPods(modelConfigId)); - } - - @PostMapping("/predict") - @ApiOperation("预测") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody predict(MultipartFile[] files, Long id, String url) { - return new DataResponseBody(servingService.predict(id, url, files)); - } - - @GetMapping("/metrics/{servingId}") - @ApiOperation("获取服务的监控信息") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getServingMetrics(@PathVariable Long servingId) { - return new DataResponseBody(servingService.getMetricsDetail(servingId)); - } - - @GetMapping("/rollback/{servingId}") - @ApiOperation("获取回滚列表") - @RequiresPermissions(Permissions.SERVING_DEPLOYMENT) - public DataResponseBody getRollbackList(@PathVariable Long servingId) { - return new DataResponseBody(servingService.getRollbackList(servingId)); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java deleted file mode 100644 index cef17c0..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/BatchServingService.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import com.baomidou.mybatisplus.extension.service.IService; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.BatchServing; -import org.dubhe.domain.vo.*; -import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; -import org.dubhe.k8s.domain.vo.PodVO; - -import java.util.List; -import java.util.Map; - -/** - * @description 批量服务管理 - * @date 2020-08-26 - */ -public interface BatchServingService extends IService { - - /** - * 批量服务查询 - * - * @param batchServingQueryDTO 批量服务查询参数 - * @return Map 批量服务查询返回分页对象 - */ - Map query(BatchServingQueryDTO batchServingQueryDTO); - - /** - * 创建批量服务 - * - * @param batchServingCreateDTO 批量服务创建参数 - * @return BatchServingCreateVO 批量服务创建返回结果 - */ - BatchServingCreateVO create(BatchServingCreateDTO batchServingCreateDTO); - - /** - * 修改批量服务 - * - * @param batchServingUpdateDTO 批量服务修改参数 - * @return BatchServingUpdateVO 批量服务修改返回结果 - */ - BatchServingUpdateVO update(BatchServingUpdateDTO batchServingUpdateDTO); - - /** - * 删除批量服务 - * - * @param batchServingDeleteDTO 批量服务删除参数 - * @return BatchServingDeleteVO 批量服务删除返回结果 - */ - BatchServingDeleteVO delete(BatchServingDeleteDTO batchServingDeleteDTO); - - /** - * 启动批量服务 - * - * @param batchServingStartDTO 批量服务启动参数 - * @return BatchServingStartVO 批量服务启动返回结果 - */ - BatchServingStartVO start(BatchServingStartDTO batchServingStartDTO); - - /** - * 停止批量服务 - * - * @param batchServingStopDTO 批量服务停止参数 - * @return BatchServingStopVO 批量服务停止返回结果 - */ - BatchServingStopVO stop(BatchServingStopDTO batchServingStopDTO); - - /** - * 获取批量服务详情 - * - * @param batchServingDetailDTO 批量服务详情参数 - * @return BatchServingDetailVO 批量服务详情返回结果 - */ - BatchServingDetailVO getDetail(BatchServingDetailDTO batchServingDetailDTO); - - /** - * k8s回调批量服务状态 - * - * @param times 回调请求次数 - * @param req 回调请求对象 - * @return boolean 返回回调结果 - */ - boolean batchServingCallback(int times, BatchServingK8sPodCallbackCreateDTO req); - - /** - * @param id 服务配置id - * @return List 服务配下的pod信息 - */ - List getPods(Long id); - - /** - * 轮询返回状态及进度 - * @param id 批量服务id - * @return BatchServingQueryVO 返回查询结果 - */ - BatchServingQueryVO queryStatusAndProgress(Long id); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java deleted file mode 100644 index d302512..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/DeploymentCallbackAsyncService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; -import org.springframework.scheduling.annotation.Async; - -/** - * @description deployment 异步回调 - * @date 2020-11-27 - */ -public interface DeploymentCallbackAsyncService { - - /** - * deployment 异步回调 - * @param k8sDeploymentCallbackCreateDTO - */ - @Async - void deploymentCallBack (R k8sDeploymentCallbackCreateDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java deleted file mode 100644 index 0b6de2f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskInstanceService.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; -import org.dubhe.domain.entity.ModelOptTaskInstance; -import org.dubhe.domain.vo.ModelOptTaskInstanceQueryVO; -import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; - -import java.util.Map; - -/** - * @description 模型优化任务实例 - * @date 2020-05-22 - */ -public interface ModelOptTaskInstanceService { - /** - * 分页查询任务执行记录实例列表 - * - * @param instanceQueryDTO 模型优化实例查询参数包装类 - * @return Map 分页对象 - */ - Map queryAll(ModelOptTaskInstanceQueryDTO instanceQueryDTO); - - /** - * 新增任务实例 - * - * @param modelOptTaskInstance 模型优化实例对象 - */ - void create(ModelOptTaskInstance modelOptTaskInstance); - - /** - * 重新提交任务实例 - * - * @param resubmitDTO 重新提交任务实例参数 - */ - void resubmit(ModelOptTaskInstanceResubmitDTO resubmitDTO); - - /** - * 取消模型优化任务 - * - * @param cancelDTO 取消模型优化任务参数 - */ - void cancel(ModelOptTaskInstanceCancelDTO cancelDTO); - - /** - * 查看单个任务实例详情 - * - * @param detailDTO 查看任务实例详情参数 - * @return ModelOptTaskInstanceVO 任务实例对象 - */ - ModelOptTaskInstanceQueryVO getInstDetail(ModelOptTaskInstanceDetailDTO detailDTO); - - /** - * k8s回调模型优化方法 - * - * @param req 模型优化自定义回调参数类 - * @return 返回回调状态 - */ - boolean modelOptCallBack(ModelOptK8sPodCallbackCreateDTO req); - - /** - * 查询K8S并同步相关实例状态 - */ - void syncInstanceStatus(); - - /** - * 校验该任务是否存在进行中和等待中的实例 - * - * @param taskId 任务ID - * @return Boolean 是否存在进行中和等待中的实例 - */ - Boolean checkUnfinishedInst(Long taskId); - - /** - * 根据任务id删除实例 - * - * @param taskId 任务id - * @return Integer 删除实例数量 - */ - int deleteByTaskId(Long taskId); - - /** - * 删除任务实例 - * - * @param modelOptTaskInstanceDeleteDTO - */ - void delete(ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java deleted file mode 100644 index 8257dbf..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ModelOptTaskService.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; -import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; -import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; -import org.dubhe.domain.vo.ModelOptCreateVO; -import org.dubhe.domain.vo.ModelOptDatasetQueryVO; -import org.dubhe.domain.vo.ModelOptDatasetVO; -import org.dubhe.domain.vo.ModelOptModelQueryVO; -import org.dubhe.domain.vo.ModelOptUpdateVO; - -import java.util.List; -import java.util.Map; - -/** - * @description 模型优化任务 - * @date 2020-05-22 - */ -public interface ModelOptTaskService { - - /** - * 查询数据分页 - * - * @param modelOptTaskQueryDTO 模型优化任务查询参数 - * @return Map 模型优化任务分页对象 - */ - Map queryAll(ModelOptTaskQueryDTO modelOptTaskQueryDTO); - - /** - * 创建模型优化任务 - * - * @param modelOptTaskCreateDTO 模型优化任务创建对象 - */ - ModelOptCreateVO create(ModelOptTaskCreateDTO modelOptTaskCreateDTO); - - /** - * 提交模型优化任务，创建任务实例 - * - * @param submitDTO 任务提交参数 - */ - void submit(ModelOptTaskSubmitDTO submitDTO); - - /** - * 修改模型优化任务 - * - * @param modelOptTaskUpdateDTO 模型优化任务修改对象 - */ - ModelOptUpdateVO update(ModelOptTaskUpdateDTO modelOptTaskUpdateDTO); - - /** - * 删除模型优化任务 - * - * @param modelOptTaskDeleteDTO 模型优化任务删除参数 - */ - void delete(ModelOptTaskDeleteDTO modelOptTaskDeleteDTO); - - /** - * 获取内置模型 - * - * @param type 算法类型 - * @param dataset 数据集 - * @param algorithm 训练算法 - * @return List 内置模型列表 - */ - List getBuiltInModel(Integer type, String dataset, String algorithm); - - /** - * 获取优化算法 - * - * @param type 算法类型 - * @param model 模型 - * @param dataset 数据集 - * @return List 获取模型优化算法列表 - */ - List getAlgorithm(Integer type, String model, String dataset); - - /** - * 获取模型优化数据集 - * - * @param type 算法类型 - * @param model 模型 - * @param algorithm 训练算法 - * @return List 模型优化数据集列表 - */ - List getDataset(Integer type, String model, String algorithm); - - /** - * 获取我的模型优化数据集 - * - * @return List 我的模型优化数据集列表 - */ - List getMyDataset(); - - /** - * 创建我的模型优化数据集 - * - * @param modelOptDatasetCreateDTO 数据集创建参数 - * @return ModelOptDatasetVO 模型优化模块我的数据集查询 - */ - ModelOptDatasetVO createMyDataset(ModelOptDatasetCreateDTO modelOptDatasetCreateDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java deleted file mode 100644 index b876bb7..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/NoteBookService.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.domain.entity.NoteBookModel; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.vo.NoteBookVO; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.enums.NoteBookStatusEnum; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * @description notebook服务接口 - * @date 2020-04-28 - */ -public interface NoteBookService { - /** - * 分页查询所有notebook记录 - * - * @param page - * @param noteBookListQueryDTO - * @return Map - */ - Map getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO); - - /** - * 查询所有notebook记录 - * - * @param page - * @param noteBookQueryDTO - * @return List - */ - List getList(Page page, NoteBookQueryDTO noteBookQueryDTO); - - /** - * 新增加notebook - * - * @param noteBook - * @return NoteBookVO - */ - NoteBookVO createNoteBook(NoteBook noteBook); - - /** - * 验证notebook是否可删除 - * - * @param noteBookIds - * @return List - */ - List validateDeleteNoteBook(Set noteBookIds); - - /** - * 删除notebook异步方法 - * - * @param noteBookList - */ - void deleteNoteBooks(List noteBookList); - - - /** - * 启动notebook - * - * @param noteBookId - * @return String - */ - String startNoteBook(Long noteBookId); - - /** - * 停止notebook - * - * @param noteBookId - * @return String - */ - String stopNoteBook(Long noteBookId); - - /** - * update by ID - * - * @param noteBook - * @return noteBook - */ - NoteBook updateById(NoteBook noteBook); - - /** - * 打开notebook - * - * @param noteBookId - * @return String - */ - String openNoteBook(Long noteBookId); - - /** - * 获取notebook可访问URL - * - * @param noteBook - * @return String - */ - String getJupyterUrl(NoteBook noteBook); - - /** - * 查询notebook在K8s对应的状态 - * - * @param noteBook - * @return NoteBookStatusEnum - */ - NoteBookStatusEnum getStatus(NoteBook noteBook); - - /** - * 第三方创建notebook - * - * @param bizNfsEnum - * @param sourceNoteBookDTO - * @return NoteBookDTO - */ - NoteBookVO createNoteBookByThirdParty(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO); - - /** - * 获取编辑地址 - * - * @param noteBookId - * @return String - */ - String getAddress(Long noteBookId); - - /** - * 删除PVC - * - * @param noteBook - * @return String - */ - String deletePvc(NoteBook noteBook); - - /** - * 获取状态编码 - * - * @return List - */ - List getNoteBookStatus(); - - /** - * 获取正在运行的notebook数量 - * - * @return int - */ - int getNoteBookRunNumber(); - - /** - * 获取notebook模板 - * - * @return Map> - */ - Map> getNoteBookModel(); - - /** - * 刷新notebook对象状态 - * - * @param statusEnum - * @param noteBook - * @return true 无需更新 - */ - boolean refreshNoteBookStatus(NoteBookStatusEnum statusEnum, NoteBook noteBook); - - /** - * 根据notebook id更新训练id - * - * @param noteBookId - * @param algorithmId - * @return boolean - */ - boolean updateTrainIdByNoteBookId(Long noteBookId, Long algorithmId); - - /** - * 根据ID查询notebook详情 - * - * @param noteBookIds - * @return List - */ - List getNotebookDetail(Set noteBookIds); - - /** - * 获取已经运行却没有URL的notebook - * - * @param page - * @return List - */ - List getRunNotUrlList(Page page); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java deleted file mode 100644 index 62e8866..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PodCallbackAsyncService.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.springframework.scheduling.annotation.Async; - -/** - * @description Pod 异步回调处理接口 - * @date 2020-05-28 - */ -public interface PodCallbackAsyncService { - - /** - * pod 异步回调 - * @param k8sPodCallbackCreateDTO - */ - @Async - void podCallBack (R k8sPodCallbackCreateDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java deleted file mode 100644 index 5e75cfe..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtImageService.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.PtImage; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * @description 镜像服务service - * @date 2020-06-22 - */ -public interface PtImageService { - - /** - * 查询镜像 - * - * @param ptImageQueryDTO 查询条件 - * @return Map 镜像列表分页信息 - **/ - Map getImage(PtImageQueryDTO ptImageQueryDTO); - - - /** - * 上传镜像到harbor - * - * @param ptImageUploadDTO 上传条件 - */ - void uploadImage(PtImageUploadDTO ptImageUploadDTO); - - - /** - * 根据镜像获取信息 - * - * @param imageName 镜像名 - * @return List 镜像集合 - */ - List searchImages(Integer projectType, String imageName); - - /** - * 删除镜像 - * - * @param imageDeleteDTO 删除镜像条件参数 - */ - void deleteTrainImage(PtImageDeleteDTO imageDeleteDTO); - - /** - * 修改镜像信息 - * - * @param imageUpdateDTO 修改的镜像信息 - */ - void updateTrainImage(PtImageUpdateDTO imageUpdateDTO); - - - /** - * 获取镜像名称列表 - * @param projectType 镜像项目类型 - * @return Set 镜像列表 - */ - Set getImageNameList(Integer projectType); - - /** - * 修改镜像来源(notebook定制) - * - * @param id 镜像id - */ - void updImageResource(Long id); - - /** - * 获取镜像URL - * - * @param imageQueryUrlDTO 查询镜像路径DTO - * @return String 镜像完整路径 - */ - String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java deleted file mode 100644 index 41ca656..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtMeasureService.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.service; - -import org.dubhe.domain.dto.PtMeasureDTO; -import org.dubhe.domain.dto.PtMeasureDeleteDTO; -import org.dubhe.domain.dto.PtMeasureQueryDTO; -import org.dubhe.domain.dto.PtMeasureUpdateDTO; - -import java.util.Map; - -/** - * @description 度量管理service - * @date 2020-11-16 - */ -public interface PtMeasureService { - - - /** - * 查询度量信息 - * - * @param ptMeasureQueryDTO 查询条件 - * @return Map 度量列表分页信息 - */ - Map getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO); - - /** - * 新建度量 - * - * @param ptMeasureCreateDTO 新建度量入参DTO - */ - void createMeasure(PtMeasureDTO ptMeasureCreateDTO); - - /** - * 修改度量 - * - * @param ptMeasureUpdateDTO 修改度量入参DTO - */ - void updateMeasure(PtMeasureUpdateDTO ptMeasureUpdateDTO); - - /** - * 根据id删除度量 - * - * @param ptMeasureDeleteDTO 删除度量的条件DTO - */ - void deleteMeasure(PtMeasureDeleteDTO ptMeasureDeleteDTO); - - /** - * 根据度量名称返回度量文件信息 - * - * @param name 度量名称 - * @return String 度量文件json字符串 - */ - String getMeasureByName(String name); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java deleted file mode 100644 index 46ea712..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmService.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.PtTrainAlgorithm; - -import java.util.Map; - -/** - * @description 训练算法 服务类 - * @date 2020-04-27 - */ -public interface PtTrainAlgorithmService { - - /** - * 查询数据分页 - * - * @param criteria 分页参数条件 - * @return Map map - */ - Map queryAll(PtTrainAlgorithmQueryDTO criteria); - - /** - * 新增算法 - * - * @param resources 新增算法条件 - * @return PtTrainAlgorithmCreateVO 新建训练算法 - */ - Long create(PtTrainAlgorithmCreateDTO resources); - - /** - * 修改算法 - * - * @param resources 修改算法条件 - * @return PtTrainAlgorithmUpdateVO 修改训练算法 - */ - Long update(PtTrainAlgorithmUpdateDTO resources); - - /** - * 删除算法 - * - * @param ptTrainAlgorithmDeleteDTO 删除算法条件 - */ - void deleteAll(PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO); - - /** - * 查询当前用户的算法个数 - */ - Map getAlgorithmCount(); - - /** - * 模型优化上传算法 - * - * @param ptModelAlgorithmCreateDTO 模型优化上传算法入参 - * @return PtTrainAlgorithm 新增算法信息 - */ - PtTrainAlgorithm modelOptimizationUploadAlgorithm(PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java deleted file mode 100644 index a28d0df..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainAlgorithmUsageService.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageQueryDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; - -import java.util.List; -import java.util.Map; - -/** - * @description 算法用途 服务类 - * @date 2020-06-23 - */ -public interface PtTrainAlgorithmUsageService { - - /** - * 查询算法用途 - * - * @param ptTrainAlgorithmUsageQueryDTO 查询算法用途参数 - */ - Map queryAll(PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO); - - /** - * 新增算法用途 - * - * @param ptTrainAlgorithmUsageCreateDTO 新增算法用途参数 - */ - List create(PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO); - - /** - * 删除算法用途 - * - * @param ptTrainAlgorithmUsageDeleteDTO 删除算法用途参数 - */ - void deleteAll(PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO); - - /** - * 更新算法用途 - * - * @param ptTrainAlgorithmUsageUpdateDTO 更新算法用途参数 - */ - void update(PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java deleted file mode 100644 index 5c85338..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobService.java +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.*; -import org.dubhe.domain.vo.*; - -import java.util.List; -import java.util.Map; - -/** - * @description 训练作业job 服务类 - * @date 2020-04-27 - */ -public interface PtTrainJobService { - - - /** - * 作业列表展示 - * - * @param ptTrainQueryDTO 查询作业列表参数 - * @return Map 作业列表分页数据 - **/ - Map getTrainJob(PtTrainQueryDTO ptTrainQueryDTO); - - - /** - * 作业不同版本job列表展示 - * - * @param ptTrainJobVersionQueryDTO 查询作业不同版本job列表参数 - * @return List 训练版本查询详情集合 - **/ - List getTrainJobVersion(PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO); - - - /** - * 创建训练job - * - * @param ptTrainJobCreateDTO 创建训练job参数 - * @return List id集合 - */ - List createTrainJobVersion(PtTrainJobCreateDTO ptTrainJobCreateDTO); - - - /** - * 修改训练job - * - * @param ptTrainJobUpdateDTO 修改训练job参数 - * @return List id集合 - **/ - List updateTrainJob(PtTrainJobUpdateDTO ptTrainJobUpdateDTO); - - - /** - * 删除训练job - * - * @param ptTrainJobDeleteDTO 删除训练job参数 - * @return PtTrainJobDeleteVO 返回删除训练任务结果 - **/ - PtTrainJobDeleteVO deleteTrainJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO); - - - /** - * 停止训练job - * - * @param ptTrainJobStopDTO 停止训练job参数 - * @return PtTrainJobStopVO 停止训练任务结果 - **/ - PtTrainJobStopVO stopTrainJob(PtTrainJobStopDTO ptTrainJobStopDTO); - - - /** - * 查询训练作业job状态 - * - * @param ptTrainDataSourceStatusQueryDTO 查询训练作业job状态参数 - * @return HashedMap 数据集路径-是否可以删除 的map集合 - **/ - Map getTrainDataSourceStatus(PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO); - - - /** - * 我的训练任务统计 - * - * @return PtTrainJobStatisticsMineVO 统计信息 - **/ - PtTrainJobStatisticsMineVO statisticsMine(); - - /** - * 根据jobId查询训练任务详情查询 - * - * @param ptTrainJobDetailQueryDTO 根据jobId查询训练任务详情查询条件 - * @return PtTrainQueryJobDetailVO 根据jobId查询训练任务详情返回结果 - */ - PtTrainJobDetailQueryVO getTrainJobDetail(PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO); - - /** - * 恢复训练 - * - * @param ptTrainJobResumeDTO 恢复训练请求参数 - */ - void resumeTrainJob(PtTrainJobResumeDTO ptTrainJobResumeDTO); - - /** - * 获取job在grafana监控的地址 - * - * @param jobId 任务ID - * @return List Pod Metrics Grafana url - */ - List getGrafanaUrl(Long jobId); - - /** - * 获取训练使用的模型信息 - * - * @param ptTrainModelDTO - * @return PtTrainJobModelVO - */ - PtTrainJobModelVO getTrainJobModel(PtTrainModelDTO ptTrainModelDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java deleted file mode 100644 index 960567c..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainJobSpecsService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.PtTrainJobSpecsQueryDTO; -import org.dubhe.domain.vo.PtTrainJobSpecsQueryVO; - -import java.util.List; - -/** - * @description 训练作业规格服务类 - * @date 2020-05-06 - */ -public interface PtTrainJobSpecsService { - - - /** - * 查询规格表 - * - * @param ptTrainJobSpecsQueryDTO 查询规格表参数 - * @return List list - **/ - List getTrainJobSpecs(PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java deleted file mode 100644 index 023e14f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainLogService.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import java.util.List; - -import org.dubhe.domain.dto.PtTrainLogQueryDTO; -import org.dubhe.k8s.domain.vo.PodVO; -import org.dubhe.domain.vo.PtTrainLogQueryVO; - -/** - * @description 训练日志服务类 - * @date 2020-05-08 - */ -public interface PtTrainLogService { - - /** - * 查询训练任务运行日志 - * - * @param ptTrainLogQueryDTO 训练日志查询 - * @return PtTrainLogQueryVO 返回训练日志查询 - **/ - PtTrainLogQueryVO queryTrainLog(PtTrainLogQueryDTO ptTrainLogQueryDTO); - - /** - * - * 字符串换行 - * @param content 个数 - * @return String 字符串 - */ - String getTrainLogString(List content); - - /** - * 获取训练任务的Pod - * - * @param id 训练作业job表 id - * @return 训练任务的Pod - */ - List getPods(Long id); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java deleted file mode 100644 index 1ba625f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/PtTrainParamService.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.PtTrainParamCreateDTO; -import org.dubhe.domain.dto.PtTrainParamDeleteDTO; -import org.dubhe.domain.dto.PtTrainParamQueryDTO; -import org.dubhe.domain.dto.PtTrainParamUpdateDTO; - -import java.util.List; -import java.util.Map; - -/** - * @description 任务参数 服务类 - * @date 2020-04-27 - */ -public interface PtTrainParamService { - - /** - * 任务参数列表展示 - * - * @param ptTrainParamQueryDTO 任务参数列表展示条件 - * @return Map 任务参数列表分页数据 - **/ - Map getTrainParam(PtTrainParamQueryDTO ptTrainParamQueryDTO); - - /** - * 保存任务参数 - * - * @param ptTrainParamCreateDTO 保存任务参数条件 - * @return List 保存任务参数id集合 - **/ - List createTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO); - - /** - * 修改任务参数 - * - * @param ptTrainParamUpdateDTO 修改任务参数条件 - * @return List 修改任务参数id集合 - **/ - List updateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO); - - /** - * 删除任务参数 - * - * @param ptTrainParamDeleteDTO 删除任务参数条件 - **/ - void deleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java deleted file mode 100644 index d093fa9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/ServingService.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service; - -import org.dubhe.domain.dto.PredictParamDTO; -import org.dubhe.domain.dto.ServingInfoCreateDTO; -import org.dubhe.domain.dto.ServingInfoDeleteDTO; -import org.dubhe.domain.dto.ServingInfoDetailDTO; -import org.dubhe.domain.dto.ServingInfoQueryDTO; -import org.dubhe.domain.dto.ServingInfoUpdateDTO; -import org.dubhe.domain.dto.ServingStartDTO; -import org.dubhe.domain.dto.ServingStopDTO; -import org.dubhe.domain.entity.ServingInfo; -import org.dubhe.domain.vo.PredictParamVO; -import org.dubhe.domain.vo.ServingInfoCreateVO; -import org.dubhe.domain.vo.ServingInfoDeleteVO; -import org.dubhe.domain.vo.ServingInfoDetailVO; -import org.dubhe.domain.vo.ServingInfoUpdateVO; -import org.dubhe.domain.vo.ServingMetricsVO; -import org.dubhe.domain.vo.ServingModelConfigVO; -import org.dubhe.domain.vo.ServingStartVO; -import org.dubhe.domain.vo.ServingStopVO; -import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; -import org.dubhe.k8s.domain.vo.PodVO; -import org.springframework.web.multipart.MultipartFile; - -import java.util.List; -import java.util.Map; - -/** - * @description 云端服务管理 - * @date 2020-08-25 - */ -public interface ServingService { - /** - * 查询分页数据 - * - * @param servingInfoQueryDTO 服务查询参数 - * @return Map 云端服务分页对象 - */ - Map query(ServingInfoQueryDTO servingInfoQueryDTO); - - /** - * 创建服务 - * - * @param servingInfoCreateDTO 服务创建参数 - * @return ServingInfoCreateVO 服务创建返回对象 - */ - ServingInfoCreateVO create(ServingInfoCreateDTO servingInfoCreateDTO); - - /** - * 修改服务 - * - * @param servingInfoUpdateDTO 服务对象修改 - * @return ServingInfoUpdateVO 服务修改返回对象 - */ - ServingInfoUpdateVO update(ServingInfoUpdateDTO servingInfoUpdateDTO); - - /** - * 删除服务 - * - * @param servingInfoDeleteDTO 服务对象删除 - * @return ServingInfoDeleteVO 服务删除返回对象 - */ - ServingInfoDeleteVO delete(ServingInfoDeleteDTO servingInfoDeleteDTO); - - /** - * 获取服务详情 - * - * @param servingInfoDetailDTO 获取服务详情参数 - * @return ServingInfoDetailVO 服务详情返回对象 - */ - ServingInfoDetailVO getDetail(ServingInfoDetailDTO servingInfoDetailDTO); - - /** - * 启动服务 - * - * @param servingStartDTO 启动服务参数 - * @return ServingStartVO 启动服务返回对象 - */ - ServingStartVO start(ServingStartDTO servingStartDTO); - - /** - * 预测 - * - * @param id 预测服务ID - * @param url 预测地址 - * @param files 需要预测的图片文件 - * @return ServingPredictVO 预测返回对象 - */ - String predict(Long id, String url, MultipartFile[] files); - - /** - * 停止服务 - * - * @param servingStopDTO 停止服务参数 - * @return ServingStopVO 停止服务返回对象 - */ - ServingStopVO stop(ServingStopDTO servingStopDTO); - - /** - * 获取预测参数 - * - * @param predictParamDTO 获取预测参数服务 - * @return PredictParamVO 预测参数返回对象 - */ - PredictParamVO getPredictParam(PredictParamDTO predictParamDTO); - - /** - * 发送路由更新消息 - * - * @param saveIdList 新增的路由ID列表 - * @param deleteIdList 删除的路由ID列表 - */ - void notifyUpdateServingRoute(List saveIdList, List deleteIdList); - - /** - * 获取modelConfigId下pod信息 - * - * @param id 服务配置id - * @return 服务配下的pod信息 - */ - List getPods(Long id); - - /** - * 获取在线服务的监控信息 - * - * @param id 模型部署信息id - * @return ServingMetricsVO 返回监控信息对象 - */ - ServingMetricsVO getMetricsDetail(Long id); - - /** - * k8s回调在线服务状态 - * - * @param times 回调请求次数 - * @param req 回调请求对象 - * @return boolean 返回是否回调成功 - */ - boolean servingCallback(int times, ServingK8sDeploymentCallbackCreateDTO req); - - /** - * 获取在线服务回滚信息列表 - * - * @param servingId 在线服务id - * @return Map> 返回回滚信息列表 - */ - Map> getRollbackList(Long servingId); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java deleted file mode 100644 index bf0be16..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractDeploymentCallback.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.abstracts; - -import org.dubhe.constant.NumberConstant; -import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.DeploymentCallbackAsyncService; -import org.dubhe.utils.K8sCallBackTool; -import org.dubhe.utils.LogUtil; - -import javax.annotation.Resource; - -/** - * @description 公共 deployment回调 - * @date 2020-11-27 - */ -public abstract class AbstractDeploymentCallback implements DeploymentCallbackAsyncService { - - @Resource - private K8sCallBackTool k8sCallBackTool; - - /** - * 公共 失败重试策略 - * - * @param k8sDeploymentCallbackCreateDTO - * @param - */ - @Override - public void deploymentCallBack(R k8sDeploymentCallbackCreateDTO) { - int tryTime = 1; - while (!doCallback(tryTime, k8sDeploymentCallbackCreateDTO)) { - if (k8sCallBackTool.continueRetry(++tryTime)) { - // 继续重试 tryTime重试次数+1 - try { - Thread.sleep(tryTime * NumberConstant.NUMBER_1000); - } catch (InterruptedException e) { - LogUtil.error(LogEnum.NOTE_BOOK, "AbstractDeploymentCallback deploymentCallBack InterruptedException : {}", e); - // Restore interrupted state...       - Thread.currentThread().interrupt(); - } - } else { - // 重试超限 tryTime重试次数+1未尝试，因此需要tryTime重试次数-1 - callbackFailed(--tryTime, k8sDeploymentCallbackCreateDTO); - break; - } - } - } - - - /** - * deployment 异步回调具体实现处理类 - * - * @param times 第n次处理 - * @param k8sDeploymentCallbackCreateDTO k8s回调实体类 - * @param BaseK8sDeploymentCallbackReq k8s回调基类 - * @return true：处理成功 false：处理失败 - */ - public abstract boolean doCallback(int times, R k8sDeploymentCallbackCreateDTO); - - - /** - * deployment 异步回调具体实现处理类 - * - * @param retryTimes 总处理次数 - * @param k8sDeploymentCallbackCreateDTO k8s回调实体类 - * @param BaseK8sDeploymentCallbackReq k8s回调基类 - */ - public abstract void callbackFailed(int retryTimes, R k8sDeploymentCallbackCreateDTO); -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java deleted file mode 100644 index e14a456..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/abstracts/AbstractPodCallback.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.abstracts; - -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.utils.K8sCallBackTool; -import org.dubhe.utils.LogUtil; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * @description pod 异步回调抽象处理类 - * @date 2020-05-29 - */ -public abstract class AbstractPodCallback implements PodCallbackAsyncService { - - @Autowired - private K8sCallBackTool k8sCallBackTool; - - /** - * 公共 失败重试策略 - * - * @param k8sPodCallbackCreateDTO - * @param - */ - @Override - public void podCallBack(R k8sPodCallbackCreateDTO) { - int tryTime = 1; - while (!doCallback(tryTime,k8sPodCallbackCreateDTO)){ - if (k8sCallBackTool.continueRetry(++tryTime)){ - // 继续重试 tryTime重试次数+1 - try { - Thread.sleep(tryTime * 1000); - continue; - } catch (InterruptedException e) { - LogUtil.error(LogEnum.NOTE_BOOK,"AbstractPodCallback podCallBack InterruptedException : {}", e); - // Restore interrupted state...       - Thread.currentThread().interrupt(); - } - }else { - // 重试超限 tryTime重试次数+1未尝试，因此需要tryTime重试次数-1 - callbackFailed(--tryTime,k8sPodCallbackCreateDTO); - break; - } - } - } - - /** - * pod 异步回调具体实现处理类 - * @param times 第n次处理 - * @param k8sPodCallbackCreateDTO k8s回调实体类 - * @param BaseK8sPodCallbackReq k8s回调基类 - * @return true：处理成功 false：处理失败 - */ - public abstract boolean doCallback(int times,R k8sPodCallbackCreateDTO); - - - /** - * pod 异步回调具体实现处理类 - * @param retryTimes 总处理次数 - * @param k8sPodCallbackCreateDTO k8s回调实体类 - * @param BaseK8sPodCallbackReq k8s回调基类 - */ - public abstract void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO); - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java deleted file mode 100644 index 092fb92..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/NoteBookConvert.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.convert; - -import org.dubhe.base.BaseConvert; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.domain.vo.NoteBookVO; -import org.mapstruct.Mapper; -import org.mapstruct.ReportingPolicy; - -/** - * @description notebook 转化器 - * @create 2020-04-28 - */ -@Mapper(componentModel = "spring", unmappedTargetPolicy = ReportingPolicy.IGNORE) -public interface NoteBookConvert extends BaseConvert { -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java deleted file mode 100644 index 77c9d00..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/convert/PtJupyterResourceConvert.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.convert; - -import org.dubhe.base.MagicNumConstant; -import org.dubhe.enums.BizEnum; -import org.dubhe.k8s.domain.bo.PtJupyterResourceBO; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.utils.K8sNameTool; - -/** - * @description PtJupyterResourceBO 转换器 - * @date 2020-04-30 - */ -public class PtJupyterResourceConvert { - - private PtJupyterResourceConvert(){ - - } - - /** - * NoteBook 转换为 PtJupyterResourceBO - * - * @param noteBook - * @param k8sNameTool - * @param notebookDelayDeleteTime - * @return PtJupyterResourceBO - */ - public static PtJupyterResourceBO toPtJupyterResourceBo(NoteBook noteBook, K8sNameTool k8sNameTool, Integer notebookDelayDeleteTime){ - if (noteBook == null){ - return null; - } - PtJupyterResourceBO bo = new PtJupyterResourceBO(); - bo.setNamespace(noteBook.getK8sNamespace()) - .setName(noteBook.getK8sResourceName()) - .setCpuNum(noteBook.getCpuNum()* MagicNumConstant.ONE_THOUSAND) - .setGpuNum(noteBook.getGpuNum()< MagicNumConstant.ONE?null:noteBook.getGpuNum()) - .setMemNum(noteBook.getMemNum()* MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR) - .setImage(noteBook.getK8sImageName()) - .setWorkspaceDir(k8sNameTool.getAbsoluteNfsPath(noteBook.getK8sPvcPath())) - .setWorkspaceMountPath(noteBook.getK8sMountPath()) - // request和limit先一致 - .setWorkspaceRequest(noteBook.getDiskMemNum()*MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR+"Mi") - .setWorkspaceLimit(noteBook.getDiskMemNum()*MagicNumConstant.ONE_THOUSAND_TWENTY_FOUR+"Mi") - .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK)) - .setDatasetDir(k8sNameTool.getAbsoluteNfsPath(noteBook.getDataSourcePath())) - .setDatasetMountPath(k8sNameTool.getDatasetPath()) - .setDatasetReadOnly(true) - .setDelayDeleteTime(notebookDelayDeleteTime) - ; - return bo; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java deleted file mode 100644 index 3627773..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/AlgorithmAsyncServiceImpl.java +++ /dev/null @@ -1,260 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.dao.PtJobParamMapper; -import org.dubhe.dao.PtTrainJobMapper; -import org.dubhe.domain.entity.PtJobParam; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.dto.callback.AlgorithmK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.TrainJobStatusEnum; -import org.dubhe.enums.TrainTypeEnum; -import org.dubhe.k8s.api.LogMonitoringApi; -import org.dubhe.k8s.api.PodApi; -import org.dubhe.k8s.domain.bo.LogMonitoringBO; -import org.dubhe.k8s.domain.resource.BizPod; -import org.dubhe.k8s.enums.K8sKindEnum; -import org.dubhe.k8s.enums.ContainerStatusesStateEnum; -import org.dubhe.k8s.utils.PodUtil; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.service.abstracts.AbstractPodCallback; -import org.dubhe.utils.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - - -/** - * @description 训练任务回调 - * @date 2020-06-03 - */ -@Service(value = "algorithmAsyncServiceImpl") -public class AlgorithmAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { - - @Autowired - private PtTrainJobMapper ptTrainJobMapper; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private PodApi podApi; - - @Autowired - private PtJobParamMapper ptJobParamMapper; - - @Autowired - private LogMonitoringApi logMonitoringApi; - - @Autowired - private RedisUtils redisUtils; - - /** - * pod 异步回调具体实现处理类 - * - * @param times 第n次处理 - * @param k8sPodCallbackCreateDTO k8s回调实体类 - * @param BaseK8sPodCallbackReq k8s回调基类 - * @return boolean true：处理成功 false：处理失败 - */ - @Override - public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { - // 强制转型 - AlgorithmK8sPodCallbackCreateDTO req = (AlgorithmK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.BIZ_TRAIN, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); - // 匹配训练任务 - PtTrainJob ptTrainJob = getPtTrainJob(req); - if (null == ptTrainJob) { - LogUtil.error(LogEnum.BIZ_TRAIN, "req={} not found", req); - return false; - } - String phase = req.getPhase(); - if (TrainJobStatusEnum.isEnd(ptTrainJob.getTrainStatus())) { - // 对于当前状态为结束状态的任务不做处理 - return true; - } - // 处理启动异常日志 - dealFailed(req,times); - if (undoDistributeTrain(ptTrainJob, req)){ - // 不需要做回调处理的分布式训练场景 - return true; - } - // 如果上报状态是结束状态并没指定过运行时间，则更新运行时间 - if (TrainJobStatusEnum.isEnd(phase) && TrainUtil.INIT_RUNTIME.equals(ptTrainJob.getRuntime())) { - //获取训练运行参数 - QueryWrapper jobParamQueryWrapper = new QueryWrapper<>(); - jobParamQueryWrapper.eq("train_job_id", ptTrainJob.getId()).last(" limit 1 "); - PtJobParam ptJobParam = ptJobParamMapper.selectOne(jobParamQueryWrapper); - if (ptJobParam == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "the data of table pt_job_param queried by trainJobId does not exist {}", ptTrainJob.getId()); - return false; - } - //判断训练是否延时启动，并更新运行时长 - long timeDelta = (ptJobParam.getDelayCreateTime() != null && ptJobParam.getDelayCreateTime().getTime() > ptTrainJob.getUpdateTime().getTime()) ? System.currentTimeMillis() - ptJobParam.getDelayCreateTime().getTime() : System.currentTimeMillis() - ptTrainJob.getUpdateTime().getTime(); - String runTime = String.format(TrainUtil.RUNTIME, - TimeUnit.MILLISECONDS.toHours(timeDelta), - TimeUnit.MILLISECONDS.toMinutes(timeDelta) % TimeUnit.HOURS.toMinutes(1), - TimeUnit.MILLISECONDS.toSeconds(timeDelta) % TimeUnit.MINUTES.toSeconds(1) - ); - ptTrainJob.setRuntime(runTime); - } - // 更新job运行时间和状态 - ptTrainJob.setTrainStatus(TrainJobStatusEnum.transferStatus(phase).getStatus()); - int updateResult = ptTrainJobMapper.updateById(ptTrainJob); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "update trainJob_id={} failed, phase={}", ptTrainJob.getId(), req.getPhase()); - return false; - } - return true; - } - - /** - * 记录异常情况的日志 - * @param req - * @param times 尝试次数 - */ - private void dealFailed(AlgorithmK8sPodCallbackCreateDTO req, int times) { - if (times != 1){ - // 仅第一次执行，避免重复产生日志 - return; - } - TrainJobStatusEnum trainJobStatusEnum = TrainJobStatusEnum.getByMessage(req.getPhase()); - if(TrainJobStatusEnum.FAILED != trainJobStatusEnum || StringUtils.isBlank(req.getMessages())){ - // 必须是回调FAILED且有日志才执行日志记录 - return; - } - // 生成资源唯一标识，避免并发调用重复执行 - String key = req.getNamespace() + "#" + req.getResourceName(); - // 线程唯一身份标识 - String uuid = UUID.randomUUID().toString(); - try { - if (!redisUtils.getDistributedLock(key,uuid,MagicNumConstant.TEN)){ - return; - } - if(logMonitoringApi.searchLogByPodName( - 0, - 1, - new LogMonitoringBO(req.getNamespace(),req.getResourceName()) - ).getTotalLogs() > 0){ - // 已有失败日志，不执行 - return; - } - List logList = new ArrayList<>(2); - logList.add(DateUtil.getCurrentTimeStr() + ": Pod startup failure!"); - logList.add("Reason: "+ ContainerStatusesStateEnum.getStateMessage(req.getMessages())); - logMonitoringApi.addLogsToEs(req.getPodName(), req.getNamespace(),logList); - }finally { - redisUtils.releaseDistributedLock(key,uuid); - } - } - - /** - * 验证是否是不需要做回调处理的分布式训练场景 - * 1，RUNNING回调时有Pod还没启动成功 - * 2，非 Master Pod的回调的结束状态状态 - * @param ptTrainJob - * @param req - * @return true 不需要做回调处理，false，需要做回调处理 - */ - private boolean undoDistributeTrain(PtTrainJob ptTrainJob,AlgorithmK8sPodCallbackCreateDTO req){ - String phase = req.getPhase(); - if (TrainTypeEnum.isDistributeTrain(ptTrainJob.getTrainType())) { - // 分布式训练 - if (ptTrainJob.getResourcesPoolNode() > MagicNumConstant.ONE - && TrainJobStatusEnum.RUNNING == TrainJobStatusEnum.getByMessage(phase) - && !validateDistributedRunningPod(req.getNamespace(), ptTrainJob)) { - // 节点数大于1 且 其回调状态为RUNNING时，需要做 多节点是否都已RUNNING的判断，以保证分布式训练任务已经处于运行状态 - // 没有启动完毕,等待下次Pod回调 - return true; - } - if (TrainJobStatusEnum.isEnd(phase) - && !PodUtil.isMaster(req.getPodName())) { - // 仅是主节点结束状态才需要更新分布式训练结束状态信息 - return true; - } - } - return false; - } - - /** - * 匹配训练任务 - * @param req - * @return PtTrainJob - */ - private PtTrainJob getPtTrainJob(AlgorithmK8sPodCallbackCreateDTO req) { - // 根据namespace和podName找到job - Long userId = k8sNameTool.getUserIdFromNamespace(req.getNamespace()); - QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); - queryTrainJonWrapper.eq("create_user_id", userId); - if (K8sKindEnum.DISTRIBUTETRAIN.getKind().equals(req.getPodParentType()) - || K8sKindEnum.JOB.getKind().equals(req.getPodParentType())) { - queryTrainJonWrapper.eq("k8s_job_name", req.getPodParentName()); - } else { - LogUtil.error(LogEnum.BIZ_TRAIN, "Pod parent type [{}] not support in callback!", req.getPodParentType()); - return null; - } - return ptTrainJobMapper.selectOne(queryTrainJonWrapper); - } - - - /** - * 验证分布式训练节点是否都已启动 - * @param namespace - * @param ptTrainJob - * @return true 完全启动，false 没有启动完毕 - */ - private boolean validateDistributedRunningPod(String namespace, PtTrainJob ptTrainJob) { - List podList = podApi.getListByResourceName(namespace, ptTrainJob.getJobName()); - if (podList.size() != ptTrainJob.getResourcesPoolNode()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "k8s pod num ne resources pod num {}/{} ！", podList.size(), ptTrainJob.getResourcesPoolNode()); - return false; - } - int runningPodSize = podList.stream() - .filter(p -> TrainJobStatusEnum.RUNNING == TrainJobStatusEnum.getByMessage(p.getPhase())) - .collect(Collectors.toList()) - .size(); - if (runningPodSize != ptTrainJob.getResourcesPoolNode()) { - LogUtil.warn(LogEnum.BIZ_TRAIN, "k8s running pod num {}/{} ", runningPodSize, ptTrainJob.getResourcesPoolNode()); - return false; - } - return true; - } - - /** - * pod 异步回调具体实现处理类 - * - * @param retryTimes 总处理次数 - * @param k8sPodCallbackCreateDTO k8s回调实体类 - * @param BaseK8sPodCallbackReq k8s回调基类 - */ - @Override - public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { - // 强制转型 - AlgorithmK8sPodCallbackCreateDTO req = (AlgorithmK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.BIZ_TRAIN, "Thread {}try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java deleted file mode 100644 index 0eafe5d..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingAsyncServiceImpl.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.BatchServingService; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.service.abstracts.AbstractPodCallback; -import org.dubhe.utils.LogUtil; -import org.springframework.stereotype.Service; -import javax.annotation.Resource; - -/** - * @description 云端Serving批量服务回调 - * @date 2020-11-27 - */ -@Service(value = "batchServingAsyncService") -public class BatchServingAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { - - @Resource - private BatchServingService batchServingService; - - @Override - public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { - // 强制转型 - BatchServingK8sPodCallbackCreateDTO req = (BatchServingK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.SERVING, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); - //批量服务回调 - return batchServingService.batchServingCallback(times, req); - } - - @Override - public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { - BatchServingK8sPodCallbackCreateDTO req = (BatchServingK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java deleted file mode 100644 index 5a4b65f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/BatchServingServiceImpl.java +++ /dev/null @@ -1,673 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.ServingConstant; -import org.dubhe.constant.StringConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.BatchServingMapper; -import org.dubhe.dao.PtModelBranchMapper; -import org.dubhe.dao.PtModelInfoMapper; -import org.dubhe.domain.PtModelBranch; -import org.dubhe.domain.PtModelInfo; -import org.dubhe.domain.dto.BatchServingCreateDTO; -import org.dubhe.domain.dto.BatchServingDeleteDTO; -import org.dubhe.domain.dto.BatchServingDetailDTO; -import org.dubhe.domain.dto.BatchServingQueryDTO; -import org.dubhe.domain.dto.BatchServingStartDTO; -import org.dubhe.domain.dto.BatchServingStopDTO; -import org.dubhe.domain.dto.BatchServingUpdateDTO; -import org.dubhe.domain.dto.RecycleTaskCreateDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.BatchServing; -import org.dubhe.domain.vo.BatchServingCreateVO; -import org.dubhe.domain.vo.BatchServingDeleteVO; -import org.dubhe.domain.vo.BatchServingDetailVO; -import org.dubhe.domain.vo.BatchServingQueryVO; -import org.dubhe.domain.vo.BatchServingStartVO; -import org.dubhe.domain.vo.BatchServingStopVO; -import org.dubhe.domain.vo.BatchServingUpdateVO; -import org.dubhe.domain.vo.ServingInfoQueryVO; -import org.dubhe.dto.callback.BatchServingK8sPodCallbackCreateDTO; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.RecycleModuleEnum; -import org.dubhe.enums.RecycleResourceEnum; -import org.dubhe.enums.RecycleTypeEnum; -import org.dubhe.enums.ServingErrorEnum; -import org.dubhe.enums.ServingStatusEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.domain.dto.PodQueryDTO; -import org.dubhe.k8s.domain.vo.PodVO; -import org.dubhe.k8s.enums.PodPhaseEnum; -import org.dubhe.k8s.service.PodService; -import org.dubhe.service.BatchServingService; -import org.dubhe.service.RecycleTaskService; -import org.dubhe.service.UserService; -import org.dubhe.task.DeployServingAsyncTask; -import org.dubhe.utils.DateUtil; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.K8sUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.MinioUtil; -import org.dubhe.utils.NfsUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.ReflectionUtils; -import org.dubhe.utils.StringUtils; -import org.dubhe.utils.WrapperHelp; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import javax.annotation.Resource; -import java.text.DecimalFormat; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @description 批量服务管理 - * @date 2020-08-26 - */ -@Service -public class BatchServingServiceImpl extends ServiceImpl implements BatchServingService { - - @Resource - private BatchServingMapper batchServingMapper; - - @Resource - private PtModelInfoMapper ptModelInfoMapper; - - @Resource - private PtModelBranchMapper ptModelBranchMapper; - - @Resource - private DeployServingAsyncTask deployServingAsyncTask; - - @Resource - private NfsUtil nfsUtil; - - @Resource - private K8sUtil k8sUtil; - - @Resource - private MinioUtil minioUtil; - - @Resource - private K8sNameTool k8sNameTool; - - @Resource - private PodService podService; - - @Resource - private RecycleTaskService recycleTaskService; - - @Resource - private UserService userService; - - @Value("${minio.bucketName}") - private String bucketName; - - - public final static List FILE_NAMES; - - static { - FILE_NAMES = ReflectionUtils.getFieldNames(BatchServingQueryVO.class); - } - - /** - * 批量服务查询 - * - * @param batchServingQueryDTO 批量服务查询参数 - * @return Map 批量服务查询返回分页对象 - */ - @Override - public Map query(BatchServingQueryDTO batchServingQueryDTO) { - String name = batchServingQueryDTO.getName(); - //批量服务名称或id条件非空 - if (StringUtils.isNotBlank(name)) { - //整数匹配 - if (StringConstant.PATTERN_NUM.matcher(name).matches()) { - batchServingQueryDTO.setId(Long.parseLong(name)); - batchServingQueryDTO.setName(null); - Map map = queryBatchServing(batchServingQueryDTO); - if (((List) map.get(StringConstant.RESULT)).size() > NumberConstant.NUMBER_0) { - return map; - } else { - batchServingQueryDTO.setId(null); - batchServingQueryDTO.setName(name); - } - } - } - return queryBatchServing(batchServingQueryDTO); - } - - /** - * 批量服务查询 - * - * @param batchServingQueryDTO 批量服务查询参数 - * @return Map 批量服务查询返回分页对象 - */ - public Map queryBatchServing(BatchServingQueryDTO batchServingQueryDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - LogUtil.info(LogEnum.SERVING, "User {} queried the online service list with the query of{}", userDTO.getUsername(), JSONObject.toJSONString(batchServingQueryDTO)); - - QueryWrapper wrapper = WrapperHelp.getWrapper(batchServingQueryDTO); - wrapper.eq("create_user_id", userDTO.getId()); - if (StringUtils.isNotBlank(batchServingQueryDTO.getName())) { - //按名称搜索时不区分大小写 - wrapper.like("lower(name)", batchServingQueryDTO.getName().toLowerCase()); - } - Page page = new Page(null == batchServingQueryDTO.getCurrent() ? NumberConstant.NUMBER_1 : batchServingQueryDTO.getCurrent(), - null == batchServingQueryDTO.getSize() ? NumberConstant.NUMBER_10 : batchServingQueryDTO.getSize()); - try { - //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 - String column = batchServingQueryDTO.getSort() != null && FILE_NAMES.contains(batchServingQueryDTO.getSort()) ? StringUtils.humpToLine(batchServingQueryDTO.getSort()) : "update_time"; - //排序方式 - boolean isAsc = StringUtils.isBlank(batchServingQueryDTO.getOrder()) || StringUtils.equals(batchServingQueryDTO.getOrder(), StringConstant.SORT_DESC) ? false : true; - wrapper.orderBy(true, isAsc, column); - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "Query online service with an exception, query info:{}，exception info:{}", JSONObject.toJSONString(batchServingQueryDTO), e); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - IPage batchServings = batchServingMapper.selectPage(page, wrapper); - List queryVOList = batchServings.getRecords().stream().map(batchServing -> { - BatchServingQueryVO batchServingQueryVO = new BatchServingQueryVO(); - BeanUtils.copyProperties(batchServing, batchServingQueryVO); - return batchServingQueryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.SERVING, "User {} queried batching service list, the number of batching service is {}", userDTO.getUsername(), queryVOList.size()); - return PageUtil.toPage(page, queryVOList); - } - - /** - * 创建批量服务 - * - * @param batchServingCreateDTO 批量服务创建参数 - * @return BatchServingCreateVO 返回创建后结果 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public BatchServingCreateVO create(BatchServingCreateDTO batchServingCreateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - //参数校验 - assert Objects.nonNull(userDTO); - checkNameExist(batchServingCreateDTO.getName(), userDTO.getId()); - Integer frameType = getFrameType(batchServingCreateDTO.getModelId(), batchServingCreateDTO.getModelAddress()); - checkResourceType(frameType, batchServingCreateDTO.getResourcesPoolType()); - checkInputExist(batchServingCreateDTO.getInputPath()); - BatchServing batchServing = new BatchServing(); - BeanUtils.copyProperties(batchServingCreateDTO, batchServing); - batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - batchServing.setFrameType(frameType); - String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; - batchServing.setOutputPath(outputPath); - saveBatchServing(userDTO, batchServing); - //输入文件定时清理 - createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); - deployServingAsyncTask.deployBatchServing(userDTO, batchServing); - return new BatchServingCreateVO(batchServing.getId(), batchServing.getStatus()); - } - - /** - * 创建批量服务并保存数据 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - */ - @Transactional(rollbackFor = Exception.class) - public void saveBatchServing(UserDTO userDTO, BatchServing batchServing) { - if (!save(batchServing)) { - LogUtil.error(LogEnum.SERVING, "User {} failed to save the batching service info to the database, service name：{}", userDTO.getUsername(), batchServing.getName()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 校验资源类型 - * - * @param frameType 模型框架 - * @param resourcePoolType 节点类型 - */ - public void checkResourceType(Integer frameType, Integer resourcePoolType) { - // oneflow 暂不支持cpu - if (NumberConstant.NUMBER_1 == frameType && NumberConstant.NUMBER_0 == resourcePoolType) { - throw new BusinessException(ServingErrorEnum.CPU_NOT_SUPPORTED_BY_ONEFLOW); - } - } - - /** - * 文件定时清理 - * - * @param path 文件路径 - * @param day 保留天数 - */ - public void createRecycleTask(String path, Integer day) { - RecycleTaskCreateDTO recycleTaskCreateDTO = new RecycleTaskCreateDTO() - .setRecycleCustom(RecycleResourceEnum.SERVING_RECYCLE_FILE.getClassName()) - .setRecycleCondition(k8sUtil.getAbsoluteNfsPath(path)) - .setRecycleDelayDate(day) - .setRecycleType(RecycleTypeEnum.FILE.getCode()) - .setRecycleModule(RecycleModuleEnum.BIZ_SERVING.getValue()) - .setRecycleNote(RecycleResourceEnum.SERVING_RECYCLE_FILE.getMessage()); - recycleTaskService.createRecycleTask(recycleTaskCreateDTO); - } - - /** - * 校验名称是否存在 - * - * @param name 服务名称 - * @param userId 用户ID - */ - public void checkNameExist(String name, Long userId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(BatchServing::getName, name); - wrapper.eq(BatchServing::getCreateUserId, userId); - int count = batchServingMapper.selectCount(wrapper); - if (count > NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.SERVING_NAME_EXIST); - } - } - - /** - * 获取模型框架 - * - * @param modelId 模型ID - * @param modelAddress 模型路径 - * @return 模型框架 - */ - public Integer getFrameType(Long modelId, String modelAddress) { - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(modelId); - String path = k8sUtil.getAbsoluteNfsPath(modelAddress); - if (ptModelInfo == null || nfsUtil.fileOrDirIsEmpty(path)) { - throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); - } - if (ptModelInfo.getFrameType() > NumberConstant.NUMBER_4) { - throw new BusinessException(ServingErrorEnum.MODEL_FRAME_TYPE_NOT_SUPPORTED); - } - return ptModelInfo.getFrameType(); - } - - /** - * 更新批量服务 - * - * @param batchServingUpdateDTO 批量服务修改参数 - * @return BatchServingUpdateVO 返回更新后结果 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public BatchServingUpdateVO update(BatchServingUpdateDTO batchServingUpdateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - BatchServing batchServing = checkBatchServingExist(batchServingUpdateDTO.getId(), userDTO.getId()); - checkBatchServingStatus(batchServing.getStatus()); - //修改输入路径时，定时删除之前路径下的文件 - if (!batchServing.getInputPath().equals(batchServingUpdateDTO.getInputPath())) { - createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); - } - createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); - //修改模型或版本时，重新校验模型 - if (!(batchServing.getModelId().equals(batchServingUpdateDTO.getModelId()) && - batchServing.getModelAddress().equals(batchServingUpdateDTO.getModelAddress()))) { - Integer frameType = getFrameType(batchServingUpdateDTO.getModelId(), batchServingUpdateDTO.getModelAddress()); - batchServing.setFrameType(frameType); - } - checkResourceType(batchServing.getFrameType(), batchServing.getResourcesPoolType()); - BeanUtils.copyProperties(batchServingUpdateDTO, batchServing); - batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - batchServing.setUpdateTime(DateUtil.getCurrentTimestamp()); - String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; - batchServing.setOutputPath(outputPath); - updateBatchServing(userDTO, batchServing); - deployServingAsyncTask.deleteBatchServing(userDTO, batchServing); - deployServingAsyncTask.deployBatchServing(userDTO, batchServing); - createRecycleTask(batchServingUpdateDTO.getInputPath(), NumberConstant.NUMBER_1); - return new BatchServingUpdateVO(batchServing.getId(), batchServing.getStatus()); - } - - /** - * 修改批量服务并保存数据 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - */ - @Transactional(rollbackFor = Exception.class) - public void updateBatchServing(UserDTO userDTO, BatchServing batchServing) { - int result = batchServingMapper.updateById(batchServing); - if (result < 1) { - LogUtil.error(LogEnum.SERVING, "User {} failed modifying the batching service in the database, service id={}", userDTO.getUsername(), batchServing.getId()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 校验批量服务是否存在 - * - * @param id 批量服务id - * @param userId 用户id - * @return BatchServing 批量服务对象 - */ - BatchServing checkBatchServingExist(Long id, Long userId) { - BatchServing batchServing = batchServingMapper.selectById(id); - //校验服务是否存在，或是否为当前用户创建 - if (batchServing == null || (!userId.equals(batchServing.getCreateUserId()))) { - throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); - } - return batchServing; - } - - /** - * 校验批量服务状态是否为停止状态 - * - * @param status 批量服务状态 - */ - void checkBatchServingStatus(String status) { - if (ServingStatusEnum.WORKING.getStatus().equals(status) || - ServingStatusEnum.IN_DEPLOYMENT.getStatus().equals(status)) { - throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); - } - } - - /** - * 校验输入图片是否为空 - * - * @param inputPath 图片路径 - */ - void checkInputExist(String inputPath) { - if (nfsUtil.fileOrDirIsEmpty(k8sUtil.getAbsoluteNfsPath(inputPath))) { - throw new BusinessException(ServingErrorEnum.INPUT_FILE_NOT_EXIST); - } - } - - /** - * 删除批量服务 - * - * @param batchServingDeleteDTO 批量服务删除参数 - * @return BatchServingDeleteVO 返回删除后结果 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public BatchServingDeleteVO delete(BatchServingDeleteDTO batchServingDeleteDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - BatchServing batchServing = checkBatchServingExist(batchServingDeleteDTO.getId(), userDTO.getId()); - checkBatchServingStatus(batchServing.getStatus()); - deleteBatchServing(batchServingDeleteDTO, userDTO); - createRecycleTask(batchServing.getInputPath(), NumberConstant.NUMBER_1); - createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); - return new BatchServingDeleteVO(batchServing.getId()); - } - - /** - * 删除批量服务并保存数据 - * - * @param batchServingDeleteDTO 批量服务信息 - * @param userDTO 用户信息 - */ - @Transactional(rollbackFor = Exception.class) - public void deleteBatchServing(BatchServingDeleteDTO batchServingDeleteDTO, UserDTO userDTO) { - if (!removeById(batchServingDeleteDTO.getId())) { - LogUtil.error(LogEnum.SERVING, "User {} failed deleting the batching service in the database, service id={}", userDTO.getUsername(), batchServingDeleteDTO.getId()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 启动批量服务 - * - * @param batchServingStartDTO 批量服务启动参数 - * @return BatchServingStartVO 返回启动后信息 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public BatchServingStartVO start(BatchServingStartDTO batchServingStartDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - BatchServing batchServing = checkBatchServingExist(batchServingStartDTO.getId(), userDTO.getId()); - if (StringUtils.equalsAny(batchServing.getStatus(), ServingStatusEnum.IN_DEPLOYMENT.getStatus(), ServingStatusEnum.WORKING.getStatus(), ServingStatusEnum.COMPLETED.getStatus())) { - LogUtil.error(LogEnum.SERVING, "User {} failed starting the batching service, service id={}, service name:{}, service status：{}", userDTO.getUsername(), batchServing.getId(), batchServing.getName(), batchServing.getStatus()); - throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); - } - //删除之前的推理结果，重新推理 - createRecycleTask(batchServing.getOutputPath(), NumberConstant.NUMBER_1); - batchServing.setProgress(SymbolConstant.ZERO); - batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - //生成新的输出路径 - String outputPath = ServingConstant.OUTPUT_NFS_PATH + userDTO.getId() + SymbolConstant.SLASH + StringUtils.getTimestamp() + SymbolConstant.SLASH; - batchServing.setOutputPath(outputPath); - updateBatchServing(userDTO, batchServing); - deployServingAsyncTask.deployBatchServing(userDTO, batchServing); - return new BatchServingStartVO(batchServing.getId(), batchServing.getStatus(), batchServing.getProgress()); - } - - /** - * 停止批量服务 - * - * @param batchServingStopDTO 批量服务停止参数 - * @return BatchServingStopVO 返回停止后信息 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public BatchServingStopVO stop(BatchServingStopDTO batchServingStopDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - BatchServing batchServing = checkBatchServingExist(batchServingStopDTO.getId(), userDTO.getId()); - if (ServingStatusEnum.STOP.getStatus().equals(batchServing.getStatus()) || ServingStatusEnum.EXCEPTION.equals(batchServing.getStatus())) { - LogUtil.error(LogEnum.SERVING, "The service is not running, user {} failed stopping the service. Service id={}, service name:{}, service status:{}", - userDTO.getUsername(), batchServing.getId(), batchServing.getName(), batchServing.getStatus()); - throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); - } - updateBatchServing(userDTO, batchServing); - deployServingAsyncTask.deleteBatchServing(userDTO, batchServing); - return new BatchServingStopVO(batchServing.getId(), batchServing.getStatus()); - } - - /** - * 获取批量服务详情 - * - * @param batchServingDetailDTO 批量服务详情参数 - * @return BatchServingDetailVO 返回批量服务详情 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public BatchServingDetailVO getDetail(BatchServingDetailDTO batchServingDetailDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - BatchServing batchServing = checkBatchServingExist(batchServingDetailDTO.getId(), userDTO.getId()); - BatchServingDetailVO batchServingDetailVO = new BatchServingDetailVO(); - BeanUtils.copyProperties(batchServing, batchServingDetailVO); - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(batchServingDetailVO.getModelId()); - if (ptModelInfo == null) { - LogUtil.error(LogEnum.SERVING, "User {} failed deploying the batch service. The model is not exist: {}, Service id={}", userDTO.getUsername(), ptModelInfo.getName(), batchServingDetailDTO.getId()); - throw new BusinessException(ServingErrorEnum.MODEL_NOT_EXIST); - } - batchServingDetailVO.setModelName(ptModelInfo.getName()); - LambdaQueryWrapper wrapper = new LambdaQueryWrapper(); - wrapper.eq(PtModelBranch::getParentId, batchServingDetailVO.getModelId()); - wrapper.eq(PtModelBranch::getModelAddress, batchServingDetailVO.getModelAddress()); - PtModelBranch ptModelBranch = ptModelBranchMapper.selectOne(wrapper); - if (ptModelBranch != null) { - batchServingDetailVO.setModelVersion(ptModelBranch.getVersionNum()); - } - //运行中的任务计算进度 - if (batchServing.getStatus().equals(ServingStatusEnum.WORKING.getStatus())) { - String progress = queryProgressByMinIO(batchServing); - batchServingDetailVO.setProgress(progress); - } - return batchServingDetailVO; - } - - /** - * 批量服务回调 - * - * @param times 回调请求次数 - * @param req 回调请求对象 - * @return boolean 返回回调结果 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public boolean batchServingCallback(int times, BatchServingK8sPodCallbackCreateDTO req) { - // 不处理slave - if (req.getPodName().contains(ServingConstant.SLAVE_POD)) { - return true; - } - // 根据namespace和podName找到模型配置 - String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.BATCH_SERVING, req.getResourceName()); - if (StringUtils.isBlank(resourceInfo)) { - LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig ID! Request: {}", Thread.currentThread(), times, req.toString()); - return false; - } - Long id = Long.parseLong(resourceInfo.substring(NumberConstant.NUMBER_4)); - BatchServing batchServing = batchServingMapper.selectById(id); - if (Objects.isNull(batchServing)) { - LogUtil.warn(LogEnum.SERVING, "Cannot find batchServing! Request: {}", Thread.currentThread(), times, req.toString()); - return false; - } - // 对于当前状态是已完成的不处理 - if (ServingStatusEnum.COMPLETED.getStatus().equals(batchServing.getStatus())) { - return true; - } - if (PodPhaseEnum.PENDING.getPhase().equals(req.getPhase())) { - batchServing.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - } - if (PodPhaseEnum.RUNNING.getPhase().equals(req.getPhase())) { - //从部署中状态转变为运行中时设置开始时间 - if (ServingStatusEnum.IN_DEPLOYMENT.getStatus().equals(batchServing.getStatus())) { - batchServing.setStartTime(DateUtil.getCurrentTimestamp()); - } - batchServing.setStatus(ServingStatusEnum.WORKING.getStatus()); - } - if (PodPhaseEnum.SUCCEEDED.getPhase().equals(req.getPhase())) { - batchServing.setEndTime(DateUtil.getCurrentTimestamp()); - batchServing.setStatus(ServingStatusEnum.COMPLETED.getStatus()); - batchServing.setProgress(String.valueOf(NumberConstant.NUMBER_100)); - // 批量推理成功异步发送短信通知 - deployServingAsyncTask.asyncSendServingMail(userService.findById(batchServing.getCreateUserId()).getEmail(), batchServing.getId()); - } - if (PodPhaseEnum.FAILED.getPhase().equals(req.getPhase())) { - String progress = queryProgressByMinIO(batchServing); - batchServing.setProgress(progress); - batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - } - if (PodPhaseEnum.DELETED.getPhase().equals(req.getPhase())) { - String progress = queryProgressByMinIO(batchServing); - batchServing.setProgress(progress); - batchServing.setStatus(ServingStatusEnum.STOP.getStatus()); - } - if (PodPhaseEnum.UNKNOWN.getPhase().equals(req.getPhase())) { - batchServing.setStatus(ServingStatusEnum.UNKNOWN.getStatus()); - } - return updateById(batchServing); - } - - /** - * 获取批量服务所有POD - * - * @param id 服务配置id - * @return List 返回POD列表 - */ - @Override - public List getPods(Long id) { - BatchServing batchServing = batchServingMapper.selectById(id); - if (batchServing == null) { - return Collections.emptyList(); - } - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(user); - String nameSpace = k8sUtil.getNamespace(user); - return podService.getPods(new PodQueryDTO(nameSpace, k8sUtil.getBatchResourceName(batchServing.getResourceInfo()))); - } - - /** - * 查询批量服务状态与进度 - * - * @param id 批量服务id - * @return BatchServingQueryVO 返回查询结果 - */ - @Override - public BatchServingQueryVO queryStatusAndProgress(Long id) { - BatchServing batchServing = batchServingMapper.selectById(id); - if (Objects.isNull(batchServing)) { - throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); - } - String progress = queryProgressByMinIO(batchServing); - return BatchServingQueryVO.builder() - .id(id) - .name(batchServing.getName()) - .description(batchServing.getDescription()) - .status(batchServing.getStatus()) - .progress(progress) - .startTime(batchServing.getStartTime()) - .endTime(batchServing.getEndTime()) - .outputPath(batchServing.getOutputPath()) - .build(); - } - - /** - * 通过minio查询推理进度 - * - * @param batchServing 批量服务信息 - * @return String 返回进度结果 - */ - private String queryProgressByMinIO(BatchServing batchServing) { - DecimalFormat df = new DecimalFormat(String.valueOf(NumberConstant.NUMBER_0)); - int inputCount = queryCount(batchServing.getInputPath()); - int outputCount = queryCount(batchServing.getOutputPath()); - String progress = String.valueOf(NumberConstant.NUMBER_0); - if (inputCount != NumberConstant.NUMBER_0) { - progress = df.format((float) outputCount / inputCount * NumberConstant.NUMBER_100); - } - return progress; - } - - /** - * 通过minio查询文件数量 - * - * @param path 文件路径 - * @return int 返回文件数量 - */ - private int queryCount(String path) { - try { - if (nfsUtil.fileOrDirIsEmpty(k8sUtil.getAbsoluteNfsPath(path))) { - return 0; - } - return minioUtil.getCount(bucketName, path); - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "query count failed by path in minio: {}", path, e); - } - return NumberConstant.NUMBER_0; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java deleted file mode 100644 index 531b0f9..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptAsyncServiceImpl.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import lombok.extern.slf4j.Slf4j; -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.ModelOptTaskInstanceService; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.service.abstracts.AbstractPodCallback; -import org.dubhe.utils.LogUtil; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; - -/** - * @description 模型优化-k8s回调处理类 - * @date 2020-06-05 - */ -@Service(value = "modelOptAsyncServiceImpl") -@Slf4j -public class ModelOptAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { - - @Resource - private ModelOptTaskInstanceService modelOptTaskInstanceService; - - @Override - public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { - try { - // 强制转型 - ModelOptK8sPodCallbackCreateDTO req = (ModelOptK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.MODEL_OPT, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); - return modelOptTaskInstanceService.modelOptCallBack(req); - } catch (Exception e) { - LogUtil.error(LogEnum.MODEL_OPT, "ModelOpt doCallback error!", e); - return false; - } - } - - @Override - public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { - ModelOptK8sPodCallbackCreateDTO req = (ModelOptK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java deleted file mode 100644 index 1d38dba..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskInstanceServiceImpl.java +++ /dev/null @@ -1,568 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import org.apache.commons.lang3.StringUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.ModelOptConstant; -import org.dubhe.constant.ModelOptErrorEnum; -import org.dubhe.constant.ModelOptInstanceStatusEnum; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.ModelOptTaskInstanceMapper; -import org.dubhe.domain.dto.DictDTO; -import org.dubhe.domain.dto.DictDetailDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceCancelDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceDetailDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskInstanceResubmitDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.ModelOptTaskInstance; -import org.dubhe.domain.vo.ModelOptResultQueryVO; -import org.dubhe.domain.vo.ModelOptTaskInstanceQueryVO; -import org.dubhe.dto.callback.ModelOptK8sPodCallbackCreateDTO; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.enums.DistillCommandEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.OptimizeTypeEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.ModelOptJobApi; -import org.dubhe.k8s.api.NamespaceApi; -import org.dubhe.k8s.domain.bo.PtModelOptimizationJobBO; -import org.dubhe.k8s.domain.dto.PodQueryDTO; -import org.dubhe.k8s.domain.resource.BizJob; -import org.dubhe.k8s.domain.resource.BizNamespace; -import org.dubhe.k8s.domain.vo.PodVO; -import org.dubhe.k8s.enums.K8sResponseEnum; -import org.dubhe.k8s.enums.PodPhaseEnum; -import org.dubhe.k8s.service.PodService; -import org.dubhe.service.DictService; -import org.dubhe.service.ModelOptTaskInstanceService; -import org.dubhe.utils.DateUtil; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.K8sUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.MinioUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.WrapperHelp; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; - -import javax.annotation.Resource; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @description 模型优化任务实例 - * @date 2020-05-22 - */ -@Service -public class ModelOptTaskInstanceServiceImpl extends ServiceImpl implements ModelOptTaskInstanceService { - - @Resource - private ModelOptTaskInstanceMapper modelOptTaskInstanceMapper; - - @Resource - private ModelOptJobApi modelOptJobApi; - - @Resource - private NamespaceApi namespaceApi; - - @Resource - private DictService dictService; - - @Resource - private K8sNameTool k8sNameTool; - - @Resource - private K8sUtil k8sUtil; - - @Resource - private PodService podService; - - @Resource - private MinioUtil minioUtil; - - @Value("${minio.bucketName}") - private String bucket; - - @Value("${optimize.image}") - private String optimizeImage; - - /** - * 分页查询任务执行记录实例列表 - * - * @param instanceQueryDTO 查询条件 - * @return Map 分页对象 - */ - @Override - public Map queryAll(ModelOptTaskInstanceQueryDTO instanceQueryDTO) { - QueryWrapper wrapper = WrapperHelp.getWrapper(instanceQueryDTO); - wrapper.orderByDesc("id"); - Page page = new Page(null == instanceQueryDTO.getCurrent() ? MagicNumConstant.ONE : instanceQueryDTO.getCurrent() - , null == instanceQueryDTO.getSize() ? MagicNumConstant.TEN : instanceQueryDTO.getSize()); - IPage modelOptTaskInstances = modelOptTaskInstanceMapper.selectPage(page, wrapper); - List list = modelOptTaskInstances.getRecords().stream().map(modelOptTaskInstance -> { - ModelOptTaskInstanceQueryVO modelOptTaskInstanceVO = buildResult(modelOptTaskInstance); - return modelOptTaskInstanceVO; - }).collect(Collectors.toList()); - return PageUtil.toPage(modelOptTaskInstances, list); - } - - /** - * 新增任务实例 - * - * @param modelOptTaskInstance 任务实例对象 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void create(ModelOptTaskInstance modelOptTaskInstance) { - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.WAITING.getValue()); - if (StringUtils.isBlank(modelOptTaskInstance.getAlgorithmPath()) || StringUtils.isBlank(modelOptTaskInstance.getAlgorithmPath())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化实例id={}的算法路径为空，算法：{}", modelOptTaskInstance.getId(), modelOptTaskInstance.getAlgorithmPath()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - modelOptTaskInstanceMapper.insert(modelOptTaskInstance); - if (!runTask(modelOptTaskInstance)) { - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 重新提交任务实例 - * - * @param resubmitDTO 重新提交任务实例参数 - */ - @Override - public void resubmit(ModelOptTaskInstanceResubmitDTO resubmitDTO) { - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(resubmitDTO.getId()); - checkInstExist(modelOptTaskInstance); - // 校验该任务是否存在进行中和等待中的实例 - if (checkUnfinishedInst(modelOptTaskInstance.getTaskId())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务实例状态:{}，提交失败，实例id={}", modelOptTaskInstance.getStatus(), resubmitDTO.getId()); - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_EXIST); - } - ModelOptTaskInstance newInstance = rebuild(modelOptTaskInstance); - create(newInstance); - } - - /** - * 取消模型优化任务 - * - * @param cancelDTO 取消模型优化任务参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void cancel(ModelOptTaskInstanceCancelDTO cancelDTO) { - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(cancelDTO.getId()); - checkInstExist(modelOptTaskInstance); - //等待、进行中的实例才能取消 - if (!StringUtils.equalsAny(modelOptTaskInstance.getStatus(), ModelOptInstanceStatusEnum.WAITING.getValue(), ModelOptInstanceStatusEnum.RUNNING.getValue())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务实例状态:{}，取消失败，实例id={}", modelOptTaskInstance.getStatus(), cancelDTO.getId()); - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_STATUS_ERROR); - } - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - // 调用k8s删除相关资源 - assert userDTO != null; - modelOptJobApi.deleteByResourceName(getNamespace(userDTO), k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, cancelDTO.getId().toString())); - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.CANCELED.getValue()); - modelOptTaskInstance.setUpdateTime(new Timestamp(System.currentTimeMillis())); - modelOptTaskInstanceMapper.updateById(modelOptTaskInstance); - } - - /** - * 查看单个任务实例详情 - * - * @param detailDTO 查看任务实例详情参数 - * @return ModelOptTaskInstanceVO 任务实例对象 - */ - @Override - public ModelOptTaskInstanceQueryVO getInstDetail(ModelOptTaskInstanceDetailDTO detailDTO) { - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(detailDTO.getId()); - checkInstExist(modelOptTaskInstance); - return buildResult(modelOptTaskInstance); - } - - /** - * k8s回调模型优化方法 - * - * @param req 模型优化自定义回调参数类 - * @return - */ - @Override - @Transactional(rollbackFor = Exception.class) - public boolean modelOptCallBack(ModelOptK8sPodCallbackCreateDTO req) { - String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.MODEL_OPT, req.getResourceName()); - if (StringUtils.isBlank(resourceInfo)) { - LogUtil.error(LogEnum.MODEL_OPT, "can't resolve resourceName {}", req.getResourceName()); - return false; - } - Long instId = Long.valueOf(resourceInfo); - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(instId); - checkInstExist(modelOptTaskInstance); - //对当前状态是已完成或已取消的不执行回调操作 - if (ModelOptInstanceStatusEnum.COMPLETED.getValue().equals(modelOptTaskInstance.getStatus()) - || ModelOptInstanceStatusEnum.CANCELED.getValue().equals(modelOptTaskInstance.getStatus())) { - return true; - } - if (PodPhaseEnum.RUNNING.getPhase().equals(req.getPhase())) { - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.RUNNING.getValue()); - modelOptTaskInstance.setStartTime(DateUtil.getCurrentTimestamp()); - } - if (PodPhaseEnum.FAILED.getPhase().equals(req.getPhase())) { - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.EXEC_FAILED.getValue()); - } - if (PodPhaseEnum.DELETED.getPhase().equals(req.getPhase())) { - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.CANCELED.getValue()); - } - if (PodPhaseEnum.SUCCEEDED.getPhase().equals(req.getPhase())) { - modelOptTaskInstance.setStatus(ModelOptInstanceStatusEnum.COMPLETED.getValue()); - modelOptTaskInstance.setEndTime(DateUtil.getCurrentTimestamp()); - if (StringUtils.isBlank(modelOptTaskInstance.getOptResultBefore()) - || StringUtils.isBlank(modelOptTaskInstance.getOptResultAfter())) { - try { - String optResultBefore = minioUtil.readString(bucket, modelOptTaskInstance.getOptResultJsonPathBefore() + ModelOptConstant.OPTIMIZE_JSON_NAME); - String optResultAfter = minioUtil.readString(bucket, modelOptTaskInstance.getOptResultJsonPathAfter() + ModelOptConstant.OPTIMIZE_JSON_NAME); - modelOptTaskInstance.setOptResultBefore(optResultBefore); - modelOptTaskInstance.setOptResultAfter(optResultAfter); - } catch (Exception e) { - LogUtil.error(LogEnum.MODEL_OPT, "Read json file field by k8s callback，instance id={},Exception={}", instId, e); - } - } - } - return updateById(modelOptTaskInstance); - } - - /** - * 查询K8S同步相关实例状态 - */ - @Override - public void syncInstanceStatus() { - // 获取等待状态超过五分钟的任务实例 - List taskInstanceList = modelOptTaskInstanceMapper.selectWaitingFor5MinutesInstances(); - // 若相应实例已启动失败更新相关实例状态 - for (ModelOptTaskInstance instance : taskInstanceList) { - UserDTO tempUser = new UserDTO(); - tempUser.setId(instance.getCreateUserId()); - tempUser.setUsername("更新优化实例状态定时任务"); - String namespace = getNamespace(tempUser); - String resourceName = k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, instance.getId().toString()); - BizJob bizJob = modelOptJobApi.getWithResourceName(namespace, resourceName); - if (K8sResponseEnum.NOT_FOUND.getCode().equals(bizJob.getCode())) { - // k8s侧无pod存在当做实例已停止 - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(instance.getId()); - checkInstExist(modelOptTaskInstance); - modelOptTaskInstanceMapper.markInstanceExecFailed(instance.getId()); - } - } - } - - /** - * 校验任务实例是否存在 - * - * @param modelOptTaskInstance 模型优化实例对象 - */ - public void checkInstExist(ModelOptTaskInstance modelOptTaskInstance) { - if (modelOptTaskInstance == null) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_ABSENT); - } - } - - /** - * 重新构建任务实例 - * - * @param instance 任务实例参数来源 - * @return 填充参数后的实例对象 - */ - private ModelOptTaskInstance rebuild(ModelOptTaskInstance instance) { - ModelOptTaskInstance newInstance = new ModelOptTaskInstance(); - newInstance.setTaskId(instance.getTaskId()) - .setTaskName(instance.getTaskName()) - .setIsBuiltIn(instance.getIsBuiltIn()) - .setModelId(instance.getModelId()) - .setModelName(instance.getModelName()) - .setModelAddress(instance.getModelAddress()) - .setAlgorithmId(instance.getAlgorithmId()) - .setAlgorithmType(instance.getAlgorithmType()) - .setAlgorithmName(instance.getAlgorithmName()) - .setAlgorithmPath(instance.getAlgorithmPath()) - .setDatasetName(instance.getDatasetName()) - .setDatasetPath(instance.getDatasetPath()) - .setCommand(instance.getCommand()) - .setParams(instance.getParams()); - return newInstance; - } - - /** - * @param instance 运行实例 - * @return boolean 调用k8s是否成功 - */ - private boolean runTask(ModelOptTaskInstance instance) { - PtModelOptimizationJobBO jobBo = new PtModelOptimizationJobBO(); - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - // 检查namespace,不存在就创建 - assert currentUser != null; - String namespace = getNamespace(currentUser); - String command = getCommand(instance); - LogUtil.info(LogEnum.MODEL_OPT, "用户{}执行模型优化输入的命令为{}，实例id={}", currentUser.getUsername(), command, instance.getId()); - String logPath = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_LOG; - String outputModelDir = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_MODEL; - String optResultJsonPathBefore = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_JSON_BEFORE; - String optResultJsonPathAfter = SymbolConstant.SLASH + BizNfsEnum.MODEL_OPT.getBizNfsPath() + SymbolConstant.SLASH + currentUser.getId() + SymbolConstant.SLASH + instance.getTaskId() + SymbolConstant.SLASH + instance.getId() + ModelOptConstant.OPTIMIZE_JSON_AFTER; - jobBo.setNamespace(namespace); - jobBo.setName(k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, instance.getId().toString())); - jobBo.setCpuNum(ModelOptConstant.CPU_NUM); - jobBo.setMemNum(ModelOptConstant.MEMORY_NUM); - jobBo.setGpuNum(ModelOptConstant.GPU_NUM); - jobBo.setCmdLines(Arrays.asList("-c", command)); - jobBo.putNfsMounts(ModelOptConstant.DATASET_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getDatasetPath())); - jobBo.putNfsMounts(ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getModelAddress())); - jobBo.putNfsMounts(ModelOptConstant.ALGORITHM_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(instance.getAlgorithmPath())); - jobBo.putNfsMounts(ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(logPath)); - jobBo.putNfsMounts(ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(outputModelDir)); - jobBo.putNfsMounts(ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(optResultJsonPathBefore)); - jobBo.putNfsMounts(ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH, k8sNameTool.getAbsoluteNfsPath(optResultJsonPathAfter)); - jobBo.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.MODEL_OPT)); - jobBo.setImage(optimizeImage); - - // 调用k8s接口 - BizJob bizJob = modelOptJobApi.create(jobBo); - if (null == bizJob || !bizJob.isSuccess()) { - String message = null == bizJob ? "未知的错误" : bizJob.getMessage(); - LogUtil.error( - LogEnum.MODEL_OPT, - "用户{}创建模型优化任务实例, k8s创建过程中失败, 实例id={}, 传递参数为{}, 错误的信息为{}", - currentUser.getUsername(), instance.getId(), instance, message); - return false; - } - instance.setLogPath(logPath); - instance.setOutputModelDir(outputModelDir); - instance.setOptResultJsonPathBefore(optResultJsonPathBefore); - instance.setOptResultJsonPathAfter(optResultJsonPathAfter); - return updateById(instance); - } - - /** - * 获取模型优化执行命令行 - * - * @param instance 模型优化实例 - * @return 返回执行命令行 - */ - private String getCommand(ModelOptTaskInstance instance) { - String command = SymbolConstant.BLANK; - if (instance.getIsBuiltIn()) { - // 内置为默认命令 - if (OptimizeTypeEnum.DISTILL.getType().equals(instance.getAlgorithmType())) { - command = DistillCommandEnum.getCommandByName(instance.getAlgorithmName()); - assert command != null; - command = String.format(command, ModelOptConstant.DATASET_MOUNT_PATH, - ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, - ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, - instance.getDatasetName(), - ModelOptConstant.DATASET_MOUNT_PATH, - ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH + ModelOptConstant.OPTIMIZE_JSON_AFTER, - ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH, - instance.getDatasetName()); - } - if (OptimizeTypeEnum.SLIMMING.getType().equals(instance.getAlgorithmType())) { - command = String.format(ModelOptConstant.OPT_START_SLIMMING_COMMAND, - ModelOptConstant.NEURONAL_PRUNING.equals(instance.getAlgorithmName()) ? "bn" : instance.getAlgorithmName(), - instance.getModelName(), - instance.getDatasetName(), - ModelOptConstant.DATASET_MOUNT_PATH, - ModelOptConstant.OUTPUT_MODEL_MOUNT_PATH, - ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH); - } - if (OptimizeTypeEnum.QUANTIFY.getType().equals(instance.getAlgorithmType())) { - command = String.format(ModelOptConstant.OPT_START_QUANTIFY_COMMAND, instance.getModelName(), - ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, - ModelOptConstant.DATASET_MOUNT_PATH, - ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_BEFORE_MOUNT_PATH, - instance.getModelName(), - ModelOptConstant.INPUT_MODEL_BEFORE_MOUNT_PATH, - ModelOptConstant.DATASET_MOUNT_PATH, - ModelOptConstant.OUTPUT_LOG_MOUNT_PATH, - ModelOptConstant.OUTPUT_RESULT_AFTER_MOUNT_PATH); - } - } else { - // 非内置为用户输入命令 - command = String.format(ModelOptConstant.MY_OPT_COMMAND, - ModelOptConstant.ALGORITHM_MOUNT_PATH, - instance.getCommand() + getParams(instance.getParams())); - } - return command; - } - - /** - * 解析运行参数 - * - * @return String 运行参数字符串 - */ - public String getParams(JSONObject params) { - StringBuilder paramStr = new StringBuilder(); - for (String key : params.keySet()) { - if (Objects.nonNull(params.get(key))) { - paramStr.append(" --").append(key).append(SymbolConstant.FLAG_EQUAL + SymbolConstant.APOSTROPHE).append(params.get(key)).append(SymbolConstant.APOSTROPHE); - } - } - return paramStr.toString(); - } - - /** - * @param currentUser 当前用户 - * @return namespace k8s的命名空间 - */ - private String getNamespace(UserDTO currentUser) { - String namespaceStr = k8sNameTool.generateNamespace(currentUser.getId()); - BizNamespace bizNamespace = namespaceApi.get(namespaceStr); - if (null == bizNamespace) { - BizNamespace namespace = namespaceApi.create(namespaceStr, null); - if (null == namespace || !namespace.isSuccess()) { - LogUtil.error(LogEnum.MODEL_OPT, "用户{}启动k8s模型优化失败，namespace为空", currentUser.getUsername()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - } - return namespaceStr; - } - - /** - * 校验该任务是否存在进行中和等待中的实例 - * - * @param taskId 任务id - * @return Boolean 是否存在进行中和等待中的实例 - */ - @Override - public Boolean checkUnfinishedInst(Long taskId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ModelOptTaskInstance::getTaskId, taskId) - .in(ModelOptTaskInstance::getStatus, Arrays.asList(ModelOptInstanceStatusEnum.WAITING.getValue(), ModelOptInstanceStatusEnum.RUNNING.getValue())); - List list = modelOptTaskInstanceMapper.selectList(wrapper); - return CollectionUtil.isNotEmpty(list); - } - - /** - * 根据任务id删除实例 - * - * @param taskId 任务id - * @return Integer 删除实例数量 - */ - @Override - public int deleteByTaskId(Long taskId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ModelOptTaskInstance::getTaskId, taskId); - return modelOptTaskInstanceMapper.delete(wrapper); - } - - @Override - @Transactional(rollbackFor = Exception.class) - public void delete(ModelOptTaskInstanceDeleteDTO modelOptTaskInstanceDeleteDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ModelOptTaskInstance modelOptTaskInstance = modelOptTaskInstanceMapper.selectById(modelOptTaskInstanceDeleteDTO.getId()); - checkInstExist(modelOptTaskInstance); - if (StringUtils.equalsAny(modelOptTaskInstance.getStatus(), ModelOptInstanceStatusEnum.RUNNING.getValue(), ModelOptInstanceStatusEnum.WAITING.getValue())) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_STATUS_ERROR); - } - modelOptTaskInstanceMapper.deleteById(modelOptTaskInstanceDeleteDTO.getId()); - // 调用k8s删除相关资源 - modelOptJobApi.deleteByResourceName(getNamespace(userDTO), - k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, modelOptTaskInstanceDeleteDTO.getId().toString())); - } - - /** - * 构建优化结果 - * - * @param modelOptTaskInstance 任务实例对象 - * @return ModelOptTaskInstanceQueryVO 模型优化任务实例 - */ - public ModelOptTaskInstanceQueryVO buildResult(ModelOptTaskInstance modelOptTaskInstance) { - //有返回结果，构造返回结果 - ModelOptTaskInstanceQueryVO modelOptTaskInstanceVO = new ModelOptTaskInstanceQueryVO(); - BeanUtils.copyProperties(modelOptTaskInstance, modelOptTaskInstanceVO); - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(user); - String nameSpace = k8sUtil.getNamespace(user); - List pods = podService.getPods(new PodQueryDTO(nameSpace, k8sNameTool.generateResourceName(BizEnum.MODEL_OPT, modelOptTaskInstance.getId().toString()))); - if (!CollectionUtils.isEmpty(pods)) { - modelOptTaskInstanceVO.setPodName(pods.get(NumberConstant.NUMBER_0).getPodName()); - } - if (StringUtils.isEmpty(modelOptTaskInstance.getOptResultBefore()) || - StringUtils.isEmpty(modelOptTaskInstance.getOptResultAfter())) { - return modelOptTaskInstanceVO; - } - //获取模型优化任务结果字典 - DictDTO dictDTO = dictService.findByName(ModelOptConstant.TASK_RESULT_DICT_NAME); - List resultList = new ArrayList<>(); - if (dictDTO == null || CollectionUtils.isEmpty(dictDTO.getDictDetails())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务结果字典为空，opt_result字典为{}", JSONObject.toJSONString(dictDTO)); - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_RESULT_DICT_EMPTY); - } - //解析优化结果字符串 - JSONObject before = JSONObject.parseObject(modelOptTaskInstance.getOptResultBefore()); - JSONObject after = JSONObject.parseObject(modelOptTaskInstance.getOptResultAfter()); - for (DictDetailDTO dictDetail : dictDTO.getDictDetails()) { - if (StringUtils.isEmpty(dictDetail.getLabel()) || StringUtils.isEmpty(dictDetail.getValue())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务结果字典详情为空，opt_result字典详情为{}", JSONObject.toJSONString(dictDetail)); - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_RESULT_DICT_EMPTY); - } - //构造模型优化结果 - ModelOptResultQueryVO modelOptResultVO = new ModelOptResultQueryVO(); - modelOptResultVO.setName(dictDetail.getLabel()); - //去掉模型优化结果中的单位 - String beforeStr = before.getOrDefault(dictDetail.getLabel(), SymbolConstant.ZERO).toString().replace(dictDetail.getValue(), SymbolConstant.BLANK); - String afterStr = after.getOrDefault(dictDetail.getLabel(), SymbolConstant.ZERO).toString().replace(dictDetail.getValue(), SymbolConstant.BLANK); - Float beforeValue = Float.parseFloat(beforeStr); - Float afterValue = Float.parseFloat(afterStr); - if (ModelOptConstant.ACCURACY.equals(dictDetail.getLabel())) { - beforeValue = beforeValue * NumberConstant.NUMBER_100; - afterValue = afterValue * NumberConstant.NUMBER_100; - } - modelOptResultVO.setBefore(beforeValue); - modelOptResultVO.setAfter(afterValue); - modelOptResultVO.setUnit(dictDetail.getValue()); - modelOptResultVO.setPositive(Integer.toString(afterValue.compareTo(beforeValue))); - resultList.add(modelOptResultVO); - } - modelOptTaskInstanceVO.setOptResult(resultList); - return modelOptTaskInstanceVO; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java deleted file mode 100644 index 3cc0f7f..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ModelOptTaskServiceImpl.java +++ /dev/null @@ -1,560 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.collection.CollectionUtil; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.config.NfsConfig; -import org.dubhe.constant.ModelOptConstant; -import org.dubhe.constant.ModelOptErrorEnum; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.ModelOptBuiltInMapper; -import org.dubhe.dao.ModelOptDatasetMapper; -import org.dubhe.dao.ModelOptTaskMapper; -import org.dubhe.dao.PtModelInfoMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.PtModelInfo; -import org.dubhe.domain.dto.ModelOptDatasetCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskCreateDTO; -import org.dubhe.domain.dto.ModelOptTaskDeleteDTO; -import org.dubhe.domain.dto.ModelOptTaskQueryDTO; -import org.dubhe.domain.dto.ModelOptTaskSubmitDTO; -import org.dubhe.domain.dto.ModelOptTaskUpdateDTO; -import org.dubhe.domain.dto.PtModelAlgorithmCreateDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.ModelOptDataset; -import org.dubhe.domain.entity.ModelOptTask; -import org.dubhe.domain.entity.ModelOptTaskInstance; -import org.dubhe.domain.entity.PtTrainAlgorithm; -import org.dubhe.domain.vo.ModelOptAlgorithmQueryVO; -import org.dubhe.domain.vo.ModelOptCreateVO; -import org.dubhe.domain.vo.ModelOptDatasetQueryVO; -import org.dubhe.domain.vo.ModelOptDatasetVO; -import org.dubhe.domain.vo.ModelOptModelQueryVO; -import org.dubhe.domain.vo.ModelOptTaskQueryVO; -import org.dubhe.domain.vo.ModelOptUpdateVO; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.OptimizeTypeEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.ModelOptTaskInstanceService; -import org.dubhe.service.ModelOptTaskService; -import org.dubhe.service.PtTrainAlgorithmService; -import org.dubhe.utils.DateUtil; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sUtil; -import org.dubhe.utils.LocalFileUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NfsUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.PtModelUtil; -import org.dubhe.utils.ReflectionUtils; -import org.dubhe.utils.StringUtils; -import org.dubhe.utils.WrapperHelp; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @description 模型优化任务 - * @date 2020-05-22 - */ -@Service -public class ModelOptTaskServiceImpl implements ModelOptTaskService { - - @Resource - private ModelOptTaskMapper modelOptTaskMapper; - - @Resource - private ModelOptTaskInstanceService modelOptTaskInstanceService; - - @Resource - private ModelOptBuiltInMapper modelOptBuiltInMapper; - - @Resource - private PtTrainAlgorithmService ptTrainAlgorithmService; - - @Resource - private NfsUtil nfsUtil; - - @Resource - private K8sUtil k8sUtil; - - @Resource - private LocalFileUtil localFileUtil; - - @Resource - private NfsConfig nfsConfig; - - @Resource - private PtModelInfoMapper ptModelInfoMapper; - - @Resource - private ModelOptDatasetMapper modelOptDatasetMapper; - - public final static List FILED_MANES; - - static { - FILED_MANES = ReflectionUtils.getFieldNames(ModelOptTaskQueryVO.class); - } - - /** - * 查询数据分页 - * - * @param modelOptTaskQueryDTO 模型优化任务查询参数 - * @return Map 模型优化任务分页对象 - */ - @Override - public Map queryAll(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { - String name = modelOptTaskQueryDTO.getName(); - //任务名称或id条件非空 - if (StringUtils.isNotEmpty(name)) { - //整数匹配 - if (Constant.PATTERN_NUM.matcher(name).matches()) { - //查询条件为整数时，先根据id精确查询 - modelOptTaskQueryDTO.setId(Long.parseLong(name)); - modelOptTaskQueryDTO.setName(null); - Map map = queryTask(modelOptTaskQueryDTO); - //根据id查询结果为空时，再根据任务名称模糊查询 - if (((List) map.get(Constant.RESULT)).size() > NumberConstant.NUMBER_0) { - return map; - } else { - modelOptTaskQueryDTO.setName(name); - } - } - } - return queryTask(modelOptTaskQueryDTO); - } - - /** - * 模型优化任务查询 - * - * @param modelOptTaskQueryDTO 模型优化任务查询条件DTO - * @return Map 分页查询结果 - */ - public Map queryTask(ModelOptTaskQueryDTO modelOptTaskQueryDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.MODEL_OPT, "用户{}查询模型优化任务列表，查询条件为{}", userDTO.getUsername(), JSONObject.toJSONString(modelOptTaskQueryDTO)); - QueryWrapper wrapper = WrapperHelp.getWrapper(modelOptTaskQueryDTO); - wrapper.eq("create_user_id", userDTO.getId()); - if (StringUtils.isNotEmpty(modelOptTaskQueryDTO.getName())) { - //任务名称模糊搜索时不区分大小写 - wrapper.like("lower(name)", modelOptTaskQueryDTO.getName().toLowerCase()); - } - Page page = new Page(null == modelOptTaskQueryDTO.getCurrent() ? MagicNumConstant.ONE : modelOptTaskQueryDTO.getCurrent(), - null == modelOptTaskQueryDTO.getSize() ? MagicNumConstant.TEN : modelOptTaskQueryDTO.getSize()); - try { - //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 - String column = modelOptTaskQueryDTO.getSort() != null && FILED_MANES.contains(modelOptTaskQueryDTO.getSort()) ? StringUtils.humpToLine(modelOptTaskQueryDTO.getSort()) : "update_time"; - //排序方式 - boolean isAsc = !StringUtils.isEmpty(modelOptTaskQueryDTO.getOrder()) && !StringUtils.equals(modelOptTaskQueryDTO.getOrder(), Constant.SORT_DESC); - wrapper.orderBy(true, isAsc, column); - } catch (Exception e) { - LogUtil.error(LogEnum.MODEL_OPT, "查询模型优化任务列表展示异常,请求信息:{}，异常信息:{}", JSONObject.toJSONString(modelOptTaskQueryDTO), e); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - IPage modelOptTasks = modelOptTaskMapper.selectPage(page, wrapper); - List queryVOList = modelOptTasks.getRecords().stream().map(modelOptTask -> { - ModelOptTaskQueryVO queryVO = new ModelOptTaskQueryVO(); - BeanUtils.copyProperties(modelOptTask, queryVO); - return queryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.MODEL_OPT, "用户{}查询模型优化任务列表，任务数={}", userDTO.getUsername(), queryVOList.size()); - return PageUtil.toPage(modelOptTasks, queryVOList); - } - - /** - * 创建模型优化任务 - * - * @param modelOptTaskCreateDTO 模型优化任务创建对象 - * @return 返回创建成功的对象 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ModelOptCreateVO create(ModelOptTaskCreateDTO modelOptTaskCreateDTO) { - //参数校验 - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - checkDatasetExist(modelOptTaskCreateDTO.getDatasetPath()); - assert userDTO != null; - checkNameExist(modelOptTaskCreateDTO.getName(), userDTO.getId()); - ModelOptTask modelOptTask = new ModelOptTask(); - BeanUtils.copyProperties(modelOptTaskCreateDTO, modelOptTask); - if (!modelOptTaskCreateDTO.getIsBuiltIn()) { - checkModelExist(modelOptTask, modelOptTaskCreateDTO.getModelId()); - //选择非内置，且算法为内置算法时（算法id为空），需要添加到我的算法中 - if (modelOptTaskCreateDTO.getAlgorithmId() == null) { - PtTrainAlgorithm algorithm = forkAlgorithm(modelOptTaskCreateDTO.getAlgorithmType(), modelOptTaskCreateDTO.getAlgorithmName(), modelOptTaskCreateDTO.getAlgorithmPath(), modelOptTaskCreateDTO.getModelAddress()); - modelOptTask.setAlgorithmType(modelOptTaskCreateDTO.getAlgorithmType()); - modelOptTask.setAlgorithmId(algorithm.getId()); - modelOptTask.setAlgorithmName(algorithm.getAlgorithmName()); - modelOptTask.setAlgorithmPath(algorithm.getCodeDir()); - saveTask(userDTO, modelOptTask); - return new ModelOptCreateVO(algorithm.getId(), algorithm.getCodeDir()); - } - //非内置，算法为我的算法 - localFileUtil.copyFile(nfsConfig.getBucket() + modelOptTaskCreateDTO.getModelAddress(), nfsConfig.getBucket() + modelOptTaskCreateDTO.getAlgorithmPath() + ModelOptConstant.COPY_MODEL_POSTFIX); - saveTask(userDTO, modelOptTask); - return new ModelOptCreateVO(modelOptTaskCreateDTO.getAlgorithmId(), modelOptTaskCreateDTO.getAlgorithmPath()); - } - //内置 - saveTask(userDTO, modelOptTask); - return null; - } - - /** - * 保存模型任务 - * - * @param userDTO 用户信息 - * @param modelOptTask 模型任务信息 - */ - private void saveTask(UserDTO userDTO, ModelOptTask modelOptTask) { - int result = modelOptTaskMapper.insert(modelOptTask); - if (result < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.MODEL_OPT, "用户{}创建模型优化任务:{}, 数据库操作失败", userDTO.getUsername(), modelOptTask.getName()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * fork内置算法到我的算法 - * - * @param algorithmName 内置算法名称 - * @param algorithmPath 内置算法路径 - * @param modelAddress 模型路径 - * @return PtTrainAlgorithm 我的算法对象 - */ - private PtTrainAlgorithm forkAlgorithm(Integer type, String algorithmName, String algorithmPath, String modelAddress) { - String newAlgorithmName = algorithmName + SymbolConstant.HYPHEN + DateUtil.getTimestampStr(); - String newAlgorithmPath = ModelOptConstant.MY_OPT_ALGORITHM_ROOT_PATH + newAlgorithmName; - String codeDir = SymbolConstant.BLANK; - if (type != null) { - codeDir = OptimizeTypeEnum.getCodeDirByType(type); - } - //copy算法 - localFileUtil.copyFile(nfsConfig.getBucket() + algorithmPath, nfsConfig.getBucket() + newAlgorithmPath); - //剪枝算法 我的模型需要拷贝到算法目录下 - if (OptimizeTypeEnum.SLIMMING.getType().equals(type)) { - localFileUtil.copyDir(nfsConfig.getBucket() + modelAddress, nfsConfig.getBucket() + newAlgorithmPath + codeDir + ModelOptConstant.COPY_MODEL_POSTFIX); - } - PtModelAlgorithmCreateDTO algorithmCreateDTO = new PtModelAlgorithmCreateDTO(); - algorithmCreateDTO.setName(newAlgorithmName); - algorithmCreateDTO.setPath(newAlgorithmPath + codeDir); - return ptTrainAlgorithmService.modelOptimizationUploadAlgorithm(algorithmCreateDTO); - } - - /** - * 提交模型优化任务，创建任务实例 - * - * @param submitDTO 任务提交参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void submit(ModelOptTaskSubmitDTO submitDTO) { - ModelOptTask modelOptTask = modelOptTaskMapper.selectById(submitDTO.getId()); - checkTaskExist(modelOptTask); - if (modelOptTaskInstanceService.checkUnfinishedInst(submitDTO.getId())) { - LogUtil.error(LogEnum.MODEL_OPT, "模型优化任务存在等待或进行中的实例，提交失败，任务id={}", submitDTO.getId()); - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_INSTANCE_EXIST); - } - ModelOptTaskInstance modelOptTaskInstance = buildInstance(modelOptTask); - modelOptTaskInstanceService.create(modelOptTaskInstance); - } - - /** - * 修改模型优化任务 - * - * @param modelOptTaskUpdateDTO 模型优化任务修改对象 - * @return 返回修改后信息 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ModelOptUpdateVO update(ModelOptTaskUpdateDTO modelOptTaskUpdateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - ModelOptTask oldTask = modelOptTaskMapper.selectById(modelOptTaskUpdateDTO.getId()); - checkTaskExist(oldTask); - //用户不能修改其他用户的任务 - assert userDTO != null; - if (!userDTO.getId().equals(oldTask.getCreateUserId())) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); - } - //如果修改了任务名，校验新的任务名是否存在 - if (!modelOptTaskUpdateDTO.getName().equals(oldTask.getName())) { - checkNameExist(modelOptTaskUpdateDTO.getName(), userDTO.getId()); - } - checkDatasetExist(modelOptTaskUpdateDTO.getDatasetPath()); - ModelOptTask newTask = new ModelOptTask(); - BeanUtils.copyProperties(modelOptTaskUpdateDTO, newTask); - if (!modelOptTaskUpdateDTO.getIsBuiltIn()) { - checkModelExist(newTask, modelOptTaskUpdateDTO.getModelId()); - //选择非内置，且算法为内置算法时（算法id为空），需要添加到我的算法中 - if (modelOptTaskUpdateDTO.getAlgorithmId() == null) { - PtTrainAlgorithm algorithm = forkAlgorithm(modelOptTaskUpdateDTO.getAlgorithmType(), modelOptTaskUpdateDTO.getAlgorithmName(), modelOptTaskUpdateDTO.getAlgorithmPath(), modelOptTaskUpdateDTO.getModelAddress()); - newTask.setAlgorithmId(algorithm.getId()); - newTask.setAlgorithmName(algorithm.getAlgorithmName()); - newTask.setAlgorithmPath(algorithm.getCodeDir()); - updateTask(userDTO, newTask); - return new ModelOptUpdateVO(algorithm.getId(), algorithm.getCodeDir()); - } - //非内置，算法为我的算法 - localFileUtil.copyFile(nfsConfig.getBucket() + modelOptTaskUpdateDTO.getModelAddress(), nfsConfig.getBucket() + modelOptTaskUpdateDTO.getAlgorithmPath() + "/model"); - updateTask(userDTO, newTask); - return new ModelOptUpdateVO(modelOptTaskUpdateDTO.getAlgorithmId(), modelOptTaskUpdateDTO.getAlgorithmPath()); - } - //内置 - updateTask(userDTO, newTask); - return null; - } - - /** - * 修改任务 - * - * @param userDTO 用户 - * @param newTask 任务 - */ - private void updateTask(UserDTO userDTO, ModelOptTask newTask) { - int result = modelOptTaskMapper.updateById(newTask); - if (result < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.MODEL_OPT, "用户{}修改模型优化任务, 数据库操作失败，任务id={}， 任务名:{}", userDTO.getUsername(), - newTask.getId(), newTask.getName()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 删除模型优化任务 - * - * @param modelOptTaskDeleteDTO 模型优化任务删除参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void delete(ModelOptTaskDeleteDTO modelOptTaskDeleteDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - ModelOptTask modelOptTask = modelOptTaskMapper.selectById(modelOptTaskDeleteDTO.getId()); - //用户不能删除其他用户的任务 - assert userDTO != null; - if (!userDTO.getId().equals(modelOptTask.getCreateUserId())) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); - } - checkTaskExist(modelOptTask); - if (modelOptTaskInstanceService.checkUnfinishedInst(modelOptTaskDeleteDTO.getId())) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_DELETE_ERROR); - } - int result = modelOptTaskMapper.deleteById(modelOptTaskDeleteDTO.getId()); - if (result < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.MODEL_OPT, "用户{}删除模型优化任务, 数据库操作失败，任务id={}， 任务名:{}", userDTO.getUsername(), modelOptTaskDeleteDTO.getId(), modelOptTask.getName()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - //删除该任务对应的实例 - modelOptTaskInstanceService.deleteByTaskId(modelOptTaskDeleteDTO.getId()); - } - - /** - * 获取内置模型 - * - * @param type 算法类型 - * @param dataset 数据集 - * @param algorithm 训练算法 - * @return List 内置模型列表 - */ - @Override - public List getBuiltInModel(Integer type, String dataset, String algorithm) { - return modelOptBuiltInMapper.getModel(type, dataset, algorithm); - } - - /** - * 获取优化算法 - * - * @param type 算法类型 - * @param model 模型 - * @param dataset 数据集 - * @return List 获取模型优化算法列表 - */ - @Override - public List getAlgorithm(Integer type, String model, String dataset) { - return modelOptBuiltInMapper.getAlgorithm(type, model, dataset); - } - - /** - * 获取模型优化数据集 - * - * @param type 算法类型 - * @param model 模型 - * @param algorithm 训练算法 - * @return List 模型优化数据集列表 - */ - @Override - public List getDataset(Integer type, String model, String algorithm) { - return modelOptBuiltInMapper.getDataset(type, model, algorithm); - } - - /** - * 获取我的模型优化数据集 - * - * @return List 我的模型优化数据集列表 - */ - @Override - public List getMyDataset() { - List list = modelOptDatasetMapper.getAllDataset(); - if (CollectionUtils.isEmpty(list)) { - return new ArrayList<>(); - } - List voList = new ArrayList<>(); - list.forEach(modelOptDataset -> { - ModelOptDatasetVO vo = new ModelOptDatasetVO(); - BeanUtils.copyProperties(modelOptDataset, vo); - voList.add(vo); - }); - return voList; - } - - /** - * 创建我的模型优化数据集 - * - * @param modelOptDatasetCreateDTO 数据集创建参数 - * @return ModelOptDatasetVO 模型优化模块我的数据集查询 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ModelOptDatasetVO createMyDataset(ModelOptDatasetCreateDTO modelOptDatasetCreateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ModelOptDataset::getName, modelOptDatasetCreateDTO.getName()); - assert userDTO != null; - wrapper.eq(ModelOptDataset::getCreateUserId, userDTO.getId()); - List list = modelOptDatasetMapper.selectList(wrapper); - if (!CollectionUtils.isEmpty(list)) { - throw new BusinessException(ModelOptErrorEnum.DATASET_NAME_EXIST); - } - //源文件路径 - String sourcePath = nfsConfig.getBucket() + modelOptDatasetCreateDTO.getPath(); - if (nfsUtil.fileOrDirIsEmpty(sourcePath)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} upload path or source path {} does not exist", userDTO.getUsername(), sourcePath); - throw new BusinessException("源文件或路径不存在"); - } - ModelOptDataset modelOptDataset = new ModelOptDataset(); - modelOptDataset.setName(modelOptDatasetCreateDTO.getName()); - //校验path是否带有压缩文件，如有，则解压至当前文件夹并删除压缩文件 - String targetPath = ModelOptConstant.MY_OPT_DATASET_ROOT_PATH + StringUtils.getTimestamp() + SymbolConstant.SLASH; - if (modelOptDatasetCreateDTO.getPath().endsWith(PtModelUtil.ZIP)) { - //目标路径 - boolean unzip = localFileUtil.unzipLocalPath(sourcePath, nfsConfig.getBucket() + targetPath); - if (!unzip) { - LogUtil.error(LogEnum.BIZ_MODEL, "用户{}解压模型文件失败", userDTO.getUsername()); - throw new BusinessException("数据集文件解压失败"); - } - modelOptDataset.setPath(targetPath); - } - - int result = modelOptDatasetMapper.insert(modelOptDataset); - if (result < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.MODEL_OPT, "用户{}创建模型优化数据集:{}, 数据库操作失败", userDTO.getUsername(), modelOptDatasetCreateDTO.getName()); - throw new BusinessException(ModelOptErrorEnum.INTERNAL_SERVER_ERROR); - } - return new ModelOptDatasetVO(modelOptDataset); - } - - /** - * 校验任务是否存在 - * - * @param modelOptTask 模型优化任务对象 - */ - public void checkTaskExist(ModelOptTask modelOptTask) { - if (modelOptTask == null) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_ABSENT); - } - } - - /** - * 校验任务名称是否已存在 - * - * @param name 任务名称 - * @param userId 用户id - */ - public void checkNameExist(String name, Long userId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ModelOptTask::getName, name); - wrapper.eq(ModelOptTask::getCreateUserId, userId); - List list = modelOptTaskMapper.selectList(wrapper); - if (CollectionUtil.isNotEmpty(list)) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_TASK_NAME_EXIST); - } - } - - /** - * 校验数据集是否存在 - * - * @param datasetPath 数据集路径 - */ - public void checkDatasetExist(String datasetPath) { - String path = k8sUtil.getAbsoluteNfsPath(datasetPath); - if (nfsUtil.fileOrDirIsEmpty(path)) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_DATASET_ABSENT); - } - } - - /** - * 校验模型是否存在 - * - * @param modelOptTask 模型优化任务 - * @param modelId 模型id - */ - public void checkModelExist(ModelOptTask modelOptTask, Long modelId) { - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(modelId); - if (ptModelInfo == null) { - throw new BusinessException(ModelOptErrorEnum.MODEL_OPT_MODEL_NOT_EXIST); - } - modelOptTask.setModelName(ptModelInfo.getName()); - } - - /** - * 根据模型优化任务，生成任务实例 - * - * @param task 模型优化任务对象 - * @return ModelOptTaskInstance 模型优化任务实例 - */ - private ModelOptTaskInstance buildInstance(ModelOptTask task) { - ModelOptTaskInstance instance = new ModelOptTaskInstance(); - instance.setTaskId(task.getId()); - instance.setTaskName(task.getName()); - instance.setIsBuiltIn(task.getIsBuiltIn()); - instance.setModelId(task.getModelId()); - instance.setModelName(task.getModelName()); - instance.setModelAddress(task.getModelAddress()); - instance.setAlgorithmType(task.getAlgorithmType()); - instance.setAlgorithmId(task.getAlgorithmId()); - instance.setAlgorithmName(task.getAlgorithmName()); - instance.setAlgorithmPath(task.getAlgorithmPath()); - instance.setDatasetName(task.getDatasetName()); - instance.setDatasetPath(task.getDatasetPath()); - instance.setCommand(task.getCommand()); - instance.setParams(task.getParams()); - return instance; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java deleted file mode 100644 index b80261e..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookAsyncServiceImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import org.dubhe.dao.NoteBookMapper; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.NoteBookStatusEnum; -import org.dubhe.dto.callback.BaseK8sPodCallbackCreateDTO; -import org.dubhe.dto.callback.NotebookK8sPodCallbackCreateDTO; -import org.dubhe.service.NoteBookService; -import org.dubhe.service.PodCallbackAsyncService; -import org.dubhe.service.abstracts.AbstractPodCallback; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NotebookUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * @description 异步刪除接口实现 - * @date 2020-05-28 - */ -@Service(value = "noteBookAsyncServiceImpl") -public class NoteBookAsyncServiceImpl extends AbstractPodCallback implements PodCallbackAsyncService { - - @Autowired - private NoteBookService noteBookService; - - @Autowired - private NoteBookMapper noteBookMapper; - - - @Override - public boolean doCallback(int times, R k8sPodCallbackCreateDTO) { - try { - // 强制转型 - NotebookK8sPodCallbackCreateDTO req = (NotebookK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.NOTE_BOOK, "NoteBookAsyncServiceImpl try {} time.Request: {}", times, req.toString()); - NoteBook notebook = noteBookMapper.findByNamespaceAndResourceName(req.getNamespace(), req.getResourceName(),NoteBookStatusEnum.DELETE.getCode()); - if (notebook == null) { - LogUtil.warn(LogEnum.NOTE_BOOK, "Cannot find notebook! Request: {}", Thread.currentThread(), times, req.toString()); - return true; - } - NoteBookStatusEnum statusEnum = NoteBookStatusEnum.convert(k8sPodCallbackCreateDTO.getPhase()); - if (noteBookService.refreshNoteBookStatus(statusEnum, notebook)) { - return true; - } - notebook.setK8sStatusCode(req.getPhase()); - notebook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(req.getMessages())); - noteBookService.updateById(notebook); - return true; - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "NoteBook doCallback error!{}", e); - return false; - } - } - - @Override - public void callbackFailed(int retryTimes, R k8sPodCallbackCreateDTO) { - // 强制转型 - NotebookK8sPodCallbackCreateDTO req = (NotebookK8sPodCallbackCreateDTO) k8sPodCallbackCreateDTO; - LogUtil.info(LogEnum.NOTE_BOOK, "Thread {}try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); - // 目前利用定时补偿补充处理，无需做callbackFailed - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java deleted file mode 100644 index 0398f52..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/NoteBookServiceImpl.java +++ /dev/null @@ -1,737 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.util.RandomUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang3.StringUtils; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.NoteBookMapper; -import org.dubhe.dao.NoteBookModelMapper; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.domain.entity.NoteBookModel; -import org.dubhe.domain.vo.NoteBookVO; -import org.dubhe.enums.*; -import org.dubhe.exception.BusinessException; -import org.dubhe.exception.NotebookBizException; -import org.dubhe.k8s.api.JupyterResourceApi; -import org.dubhe.k8s.api.NamespaceApi; -import org.dubhe.k8s.api.PodApi; -import org.dubhe.k8s.domain.PtBaseResult; -import org.dubhe.k8s.domain.resource.BizNamespace; -import org.dubhe.k8s.domain.resource.BizPod; -import org.dubhe.k8s.domain.vo.PtJupyterDeployVO; -import org.dubhe.k8s.enums.K8sResponseEnum; -import org.dubhe.service.NoteBookService; -import org.dubhe.service.PtImageService; -import org.dubhe.service.convert.NoteBookConvert; -import org.dubhe.service.convert.PtJupyterResourceConvert; -import org.dubhe.utils.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * @description notebook服务实现 - * @date 2020-04-28 - */ -@Service -public class NoteBookServiceImpl implements NoteBookService { - - @Autowired - private NoteBookMapper noteBookMapper; - - @Autowired - private NoteBookModelMapper noteBookModelMapper; - - @Autowired - private NoteBookConvert noteBookConvert; - - @Autowired - private JupyterResourceApi jupyterResourceApi; - - @Autowired - private PodApi podApi; - - @Autowired - private NamespaceApi namespaceApi; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private PtImageService ptImageService; - - @Value("${delay.notebook.delete}") - private Integer notebookDelayDeleteTime; - - private static final String BLANK = SymbolConstant.BLANK; - - /** - * 分页查询所有 notebook 记录 - * - * @param page - * @param noteBookListQueryDTO - * @return Map - */ - @Override - public Map getNoteBookList(Page page, NoteBookListQueryDTO noteBookListQueryDTO) { - QueryWrapper queryWrapper = WrapperHelp.getWrapper(noteBookListQueryDTO); - queryWrapper.ne(true, NoteBook.COLUMN_STATUS, NoteBookStatusEnum.DELETE.getCode()) - .ne(true, "deleted", NoteBookStatusEnum.STOP.getCode()); - if (noteBookListQueryDTO.getStatus() != null) { - if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.RUN.getCode())) { - //运行中的notebook必须有url - queryWrapper.eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.RUN.getCode()) - .ne(NoteBook.COLUMN_URL, SymbolConstant.BLANK); - } else if (noteBookListQueryDTO.getStatus().equals(NoteBookStatusEnum.STARTING.getCode())) { - //启动中的notebook还包括运行中但没有url - queryWrapper.and((qw) -> - qw.eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.RUN.getCode()).eq(NoteBook.COLUMN_URL, SymbolConstant.BLANK) - .or() - .eq(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.STARTING.getCode()) - ); - } else { - // 其他状态照常 - queryWrapper.eq(NoteBook.COLUMN_STATUS, noteBookListQueryDTO.getStatus()); - } - } - queryWrapper.orderBy(true, false, "id"); - IPage noteBookPage = noteBookMapper.selectPage(page, queryWrapper); - return PageUtil.toPage(noteBookPage, noteBookConvert::toDto); - } - - /** - * 查询所有 notebook 记录 - * - * @param page - * @param noteBookQueryDTO - * @return List - */ - @Override - public List getList(Page page, NoteBookQueryDTO noteBookQueryDTO) { - return noteBookMapper.selectPage(page, WrapperHelp.getWrapper(noteBookQueryDTO)).getRecords(); - } - - /** - * 获取镜像路径 - * - * @return String 镜像路径 - */ - private String getDefaultImage() { - PtImageQueryUrlDTO imageQueryUrlDTO = new PtImageQueryUrlDTO(); - imageQueryUrlDTO.setProjectType(ImageTypeEnum.NOTEBOOK.getType()) - .setImageResource(ImageSourceEnum.PRE.getCode()); - String imageUrl = ptImageService.getImageUrl(imageQueryUrlDTO); - if (StrUtil.isEmpty(imageUrl)) { - throw new BusinessException(ImageTypeEnum.NOTEBOOK.getCode() + "未配置默认镜像！"); - } - - return imageUrl; - } - - /** - * 新增加 notebook - * - * @param noteBook - * @return NoteBookVO - * - */ - @Override - @Transactional(rollbackFor = Exception.class) - public NoteBookVO createNoteBook(NoteBook noteBook) { - if (noteBookMapper.findByNameAndStatus(noteBook.getNoteBookName(), NoteBookStatusEnum.DELETE.getCode()) != null) { - throw new NotebookBizException("Notebook名称已使用过！请重新提交。"); - } - if (StringUtils.isEmpty(noteBook.getName())) { - noteBook.setName(k8sNameTool.getK8sName()); - } - noteBook.setK8sNamespace(k8sNameTool.generateNamespace(noteBook.getCreateUserId())); - noteBook.setK8sResourceName(k8sNameTool.generateResourceName(BizEnum.NOTEBOOK, noteBook.getName())); - if (StringUtils.isBlank(noteBook.getK8sPvcPath())) { - //20200618 修改为 使用训练路劲 - noteBook.setK8sPvcPath(k8sNameTool.getNfsPath(BizNfsEnum.ALGORITHM, noteBook.getCreateUserId())); - } - noteBook.setK8sMountPath(NotebookUtil.getK8sMountPath()); - if (start(noteBook)) { - noteBook.setStatus(NoteBookStatusEnum.STARTING.getCode()); - } else { - noteBook.setStatus(NoteBookStatusEnum.STOP.getCode()); - } - noteBookMapper.insert(noteBook); - return noteBookConvert.toDto(noteBook); - } - - /** - * 初始化namespace - * - * @param noteBook - * @param labels - * @return boolean - */ - private boolean initNameSpace(NoteBook noteBook, Map labels) { - try { - BizNamespace result = namespaceApi.create(noteBook.getK8sNamespace(), labels); - noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); - return (HttpUtils.isSuccess(result.getCode()) - || K8sResponseEnum.EXISTS.getCode().equals(result.getCode())); - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "createNoteBook调用jupyterResourceApi.createWithPvc异常！{}", e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - return false; - } - } - - /** - * 验证 notebook 是否可删除 - * - * @param noteBookIds - * @return List - */ - @Override - public List validateDeleteNoteBook(Set noteBookIds) { - for (Long noteBookId : noteBookIds) { - NumberUtil.isNumber(noteBookId); - } - List deleteTypeList = NoteBookStatusEnum.getCanDeleteStatus(); - List noteBookList = noteBookMapper.selectBatchIds(noteBookIds); - for (NoteBook noteBook : noteBookList) { - if (deleteTypeList.contains(noteBook.getStatus())) { - throw new NotebookBizException("不可删除正在运行的notebook！"); - } - } - return noteBookList; - } - - /** - * 删除notebook异步方法 - * - * @param noteBookList - */ - @Override - @Async - @Transactional(rollbackFor = Exception.class) - public void deleteNoteBooks(List noteBookList) { - if (CollUtil.isNotEmpty(noteBookList)) { - for (NoteBook noteBook : noteBookList) { - if (noteBook.getStatus().equals(NoteBookStatusEnum.STOP.getCode())) { - deleteNoteBook(noteBook); - } - } - } - } - - /** - * 删除notebook实现逻辑 - * - * @param noteBook - * @return String - */ - private String deleteNoteBook(NoteBook noteBook) { - if (noteBook == null) { - throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); - } - String returnStr; - NoteBookStatusEnum statusEnum = getStatus(noteBook); - if (NoteBookStatusEnum.STOP == statusEnum) { - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(BLANK); - noteBook.setUrl(BLANK); - returnStr = this.deletePvc(noteBook); - } else { - try { - PtBaseResult result = jupyterResourceApi.delete(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); - noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); - if (HttpUtils.isSuccess(result.getCode())) { - noteBook.setStatus(NoteBookStatusEnum.DELETING.getCode()); - // 添加超时时间点 - noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); - noteBook.setUrl(BLANK); - returnStr = NoteBookStatusEnum.DELETING.getDescription(); - } else if (K8sResponseEnum.REPEAT.getCode().equals(result.getCode())) { - // 重复提交停止指令，无需再次停止，直接删除PVC文件 - noteBook.setUrl(BLANK); - returnStr = deletePvc(noteBook); - } else { - // 其他失败编码 -> 删除失败,保留原状态 - returnStr = "删除失败"; - } - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "deleteNoteBook调用jupyterResourceApi.delete异常！{}", e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - returnStr = "删除失败"; - } - } - this.updateById(noteBook); - return returnStr; - } - - /** - * 启动notebook - * - * @param noteBookId - * @return String - */ - @Override - @Transactional(rollbackFor = Exception.class) - public String startNoteBook(Long noteBookId) { - NumberUtil.isNumber(noteBookId); - NoteBook noteBook = noteBookMapper.selectById(noteBookId); - return startNoteBook(noteBook); - } - - /** - * 具体启动notebook实现 - * - * @param noteBook - * @return String - */ - private String startNoteBook(NoteBook noteBook) { - if (noteBook == null) { - throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); - } - if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { - return "notebook " + NoteBookStatusEnum.RUN.getDescription(); - } else if (NoteBookStatusEnum.STARTING.getCode().equals(noteBook.getStatus())) { - return "notebook " + NoteBookStatusEnum.STARTING.getDescription(); - } else if (!NoteBookStatusEnum.STOP.getCode().equals(noteBook.getStatus())) { - throw new NotebookBizException("notebook【" + noteBook.getName() + "】当前状态：" + NoteBookStatusEnum.getDescription(noteBook.getStatus()) + ",无法再次启动。"); - } - String returnStr; - if (start(noteBook)) { - noteBook.setStatus(NoteBookStatusEnum.STARTING.getCode()); - returnStr = NoteBookStatusEnum.STARTING.getDescription(); - } else { - // 重启notebook状态沿用历史状态 - returnStr = "启动" + NotebookUtil.FAILED; - } - this.updateById(noteBook); - return returnStr; - } - - /** - * 更新notebook - * - * @param noteBook - * @return NoteBook - */ - @Override - @Transactional(rollbackFor = Exception.class) - public NoteBook updateById(NoteBook noteBook) { - noteBook.setUpdateTime(null); - noteBook.setUpdateUserId(NotebookUtil.getCurUserId()); - noteBookMapper.updateById(noteBook); - return noteBook; - } - - /** - * 启动notebook - * - * @param noteBook - * @return true 启动成功；false 启动失败 - */ - private boolean start(NoteBook noteBook) { - // 添加启动时间 - noteBook.setLastStartTime(new Date()); - // 添加超时时间点 - noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); - if (initNameSpace(noteBook, null)) { - try { - //20200618 修改为 创建时不创建PVC - PtJupyterDeployVO result = jupyterResourceApi.create(PtJupyterResourceConvert.toPtJupyterResourceBo(noteBook, k8sNameTool, notebookDelayDeleteTime)); - noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); - return HttpUtils.isSuccess(result.getCode()); - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "notebook调用jupyterResourceApi.createWithPvc异常！{}", e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - return false; - } - } else { - return false; - } - } - - /** - * 停止notebook - * - * @param noteBookId - * @return String - */ - @Override - @Transactional(rollbackFor = Exception.class) - public String stopNoteBook(Long noteBookId) { - NumberUtil.isNumber(noteBookId); - NoteBook noteBook = noteBookMapper.selectById(noteBookId); - if (noteBook == null) { - throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); - } - if (!NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { - throw new NotebookBizException("notebook正在运行,不能停止"); - } - String returnStr; - NoteBookStatusEnum statusEnum = getStatus(noteBook); - if (NoteBookStatusEnum.STOP == statusEnum) { - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(BLANK); - noteBook.setUrl(BLANK); - returnStr = "已停止"; - } else { - try { - PtBaseResult result = jupyterResourceApi.delete(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); - noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); - if (HttpUtils.isSuccess(result.getCode())) { - noteBook.setStatus(NoteBookStatusEnum.STOPPING.getCode()); - // 添加超时时间点 - noteBook.setLastOperationTimeout(NotebookUtil.getTimeoutSecondLong()); - noteBook.setUrl(BLANK); - returnStr = NoteBookStatusEnum.STOPPING.getDescription(); - } else if (K8sResponseEnum.REPEAT.getCode().equals(result.getCode())) { - // 重复提交停止指令，无需再次停止，直接标记停止 - noteBook.setStatus(NoteBookStatusEnum.STOP.getCode()); - noteBook.setUrl(BLANK); - returnStr = NoteBookStatusEnum.STOP.getDescription(); - } else { - // 其他失败编码 -> 停止失败,保留原状态 - returnStr = "停止" + NotebookUtil.FAILED; - } - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "停止notebook调用jupyterResourceApi.delete异常！{}", e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - returnStr = "停止" + NotebookUtil.FAILED; - } - } - this.updateById(noteBook); - return returnStr; - } - - /** - * 开启notebook - * - * @param noteBookId - * @return String - */ - @Override - public String openNoteBook(Long noteBookId) { - NumberUtil.isNumber(noteBookId); - NoteBook noteBook = noteBookMapper.selectById(noteBookId); - if (noteBook == null) { - throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); - } else if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { - if (NotebookUtil.checkUrlContainsToken(noteBook.getUrl())) { - return noteBook.getUrl(); - } else { - // 补偿:已启动notebook获取可访问地址 - String jupyterUrlWithToken = this.getJupyterUrl(noteBook); - if (NotebookUtil.checkUrlContainsToken(jupyterUrlWithToken)) { - noteBook.setUrl(jupyterUrlWithToken); - this.updateById(noteBook); - return noteBook.getUrl(); - } else { - throw new NotebookBizException("notebook已启动 获取URL失败！"); - } - } - } else { - throw new NotebookBizException("notebook 尚未启动成功,无法打开。"); - } - } - - /** - * 获取jupyter 地址 - * - * @param noteBook - * @return String - */ - @Override - public String getJupyterUrl(NoteBook noteBook) { - try { - return podApi.getUrlByResourceName(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 获取URL失败！", noteBook.getK8sNamespace(), noteBook.getK8sResourceName(), e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - return null; - } - } - - /** - * 获取notebook状态 - * - * @param noteBook - * @return NoteBookStatusEnum - */ - @Override - public NoteBookStatusEnum getStatus(NoteBook noteBook) { - try { - BizPod result = podApi.getWithResourceName(noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); - noteBook.setK8sStatusCode(result.getCode() == null ? BLANK : result.getCode()); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(result)); - if (K8sResponseEnum.NOT_FOUND.getCode().equals(result.getCode())) { - // 结果不存在当已停止 - return NoteBookStatusEnum.STOP; - } else if (!HttpUtils.isSuccess(result.getCode())) { - LogUtil.warn(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 查询失败！", noteBook.getK8sNamespace(), noteBook.getK8sResourceName()); - return null; - } - return NoteBookStatusEnum.convert(result.getPhase()); - } catch (Exception e) { - LogUtil.error(LogEnum.NOTE_BOOK, "notebook nameSpace:{} resourceName:{} 查询异常！{}", noteBook.getK8sNamespace(), noteBook.getK8sResourceName(), e); - noteBook.setK8sStatusCode(BLANK); - noteBook.setK8sStatusInfo(NotebookUtil.getK8sStatusInfo(e)); - return null; - } - } - - /** - * 第三方创建notebook - * - * @param bizNfsEnum - * @param sourceNoteBookDTO - * @return NoteBookVO - */ - @Override - public NoteBookVO createNoteBookByThirdParty(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO) { - String k8sPvcPath = sourceNoteBookDTO.getSourceFilePath(); - NoteBook noteBook = noteBookMapper.selectOne(WrapperHelp.getWrapper(new NoteBookQueryDTO( - NoteBookStatusEnum.DELETE.getCode(), - k8sPvcPath, - sourceNoteBookDTO.getCurUserId()) - )); - if (noteBook == null) { - return this.createNoteBook(initSourceReqNoteBook(bizNfsEnum, sourceNoteBookDTO, k8sPvcPath)); - } else { - if (!NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { - this.startNoteBook(noteBook); - } - return noteBookConvert.toDto(noteBook); - } - } - - /** - * 初始化第三方请求的notebook - * - * @param bizNfsEnum - * @param sourceNoteBookDTO - * @param k8sPvcPath - * @return NoteBook - */ - private NoteBook initSourceReqNoteBook(BizNfsEnum bizNfsEnum, SourceNoteBookDTO sourceNoteBookDTO, String k8sPvcPath) { - NoteBook noteBook = new NoteBook(); - noteBook.setCreateUserId(sourceNoteBookDTO.getCurUserId()); - noteBook.setUserId(sourceNoteBookDTO.getCurUserId()); - noteBook.setCreateResource(bizNfsEnum.getCreateResource()); - noteBook.setDescription(bizNfsEnum.getBizName()); - noteBook.setName(k8sNameTool.getK8sName()); - String notebookName = NotebookUtil.generateName(bizNfsEnum, sourceNoteBookDTO.getSourceId()); - if (noteBookMapper.findByNameAndStatus(notebookName, NoteBookStatusEnum.DELETE.getCode()) != null) { - // 重名随机符号拼接 - notebookName += RandomUtil.randomString(MagicNumConstant.TWO); - } - - noteBook.setNoteBookName(notebookName); - noteBook.setCpuNum(MagicNumConstant.ONE); - noteBook.setGpuNum(MagicNumConstant.ZERO); - noteBook.setMemNum(MagicNumConstant.ONE); - noteBook.setDiskMemNum(MagicNumConstant.ONE); - noteBook.setAlgorithmId(sourceNoteBookDTO.getSourceId()); - - noteBook.setK8sPvcPath(k8sPvcPath); - noteBook.setK8sImageName(getDefaultImage()); - return noteBook; - } - - /** - * 获取地址 - * - * @param noteBookId - * @return String - */ - @Override - @Transactional(rollbackFor = Exception.class) - public String getAddress(Long noteBookId) { - NumberUtil.isNumber(noteBookId); - NoteBook noteBook = noteBookMapper.selectById(noteBookId); - if (noteBook == null) { - throw new NotebookBizException(NotebookUtil.NOTEBOOK_NOT_EXISTS); - } else if (NoteBookStatusEnum.RUN.getCode().equals(noteBook.getStatus())) { - if (NotebookUtil.checkUrlContainsToken(noteBook.getUrl())) { - return noteBook.getUrl(); - } else { - // 补偿:已启动notebook获取可访问地址 - String jupyterUrlWithToken = this.getJupyterUrl(noteBook); - if (NotebookUtil.checkUrlContainsToken(jupyterUrlWithToken)) { - noteBook.setUrl(jupyterUrlWithToken); - this.updateById(noteBook); - return noteBook.getUrl(); - } - } - } - return null; - } - - /** - * 删除PVC - * - * @param noteBook - * @return String - */ - @Override - public String deletePvc(NoteBook noteBook) { - noteBook.setStatus(NoteBookStatusEnum.DELETE.getCode()); - noteBook.setDeleted(true); - return NoteBookStatusEnum.DELETE.getDescription(); - } - - /** - * 获取notebook所有状态 - * - * @return List - */ - @Override - public List getNoteBookStatus() { - List noteBookStatusDtoList = new ArrayList<>(); - for (NoteBookStatusEnum noteBookStatusEnum : NoteBookStatusEnum.values()) { - if (noteBookStatusEnum != NoteBookStatusEnum.DELETE) { - NoteBookStatusDTO noteBookStatusDTO = new NoteBookStatusDTO(); - noteBookStatusDTO.setStatusCode(noteBookStatusEnum.getCode()); - noteBookStatusDTO.setStatusName(noteBookStatusEnum.getDescription()); - noteBookStatusDtoList.add(noteBookStatusDTO); - } - } - return noteBookStatusDtoList; - } - - /** - * 获取正在运行的notebook数量 - * - * @return int - */ - @Override - public int getNoteBookRunNumber() { - return noteBookMapper.selectRunNoteBookNum(NoteBookStatusEnum.RUN.getCode()); - } - - /** - * 获取notebook配置的模式数据 - * - * @return Map> - */ - @Override - public Map> getNoteBookModel() { - List noteBookModelList = noteBookModelMapper.selectAllNoteBookModel(); - return CollUtil.isEmpty(noteBookModelList) ? - new HashMap<>(MagicNumConstant.EIGHT) : - noteBookModelList.stream().collect(Collectors.groupingBy(NoteBookModel::getModelType)); - } - - /** - * 刷新notebook状态 - * - * @param statusEnum - * @param noteBook - * @return boolean - */ - @Override - public boolean refreshNoteBookStatus(NoteBookStatusEnum statusEnum, NoteBook noteBook) { - if (NoteBookStatusEnum.RUN == statusEnum) { - if (NoteBookStatusEnum.STARTING.getCode().equals(noteBook.getStatus())) { - // append jupyter url+token - noteBook.setUrl(this.getJupyterUrl(noteBook)); - // 仅启动中可切换为启动状态 - noteBook.setStatus(statusEnum.getCode()); - } else { - return true; - } - } else if (NoteBookStatusEnum.STOP == statusEnum) { - if (NoteBookStatusEnum.DELETING.getCode().equals(noteBook.getStatus())) { - this.deletePvc(noteBook); - // deletePVC方法内部已设置状态，无需这边设置 - } else { - noteBook.setStatus(statusEnum.getCode()); - } - noteBook.setUrl(BLANK); - } - return false; - } - - /** - * 支持双向更新，如果不为0 ，则是绑定关联关系，反之解除绑定关系 - * - * @param noteBookId - * @param algorithmId - * @return boolean - */ - @Override - @Transactional(rollbackFor = Exception.class) - public boolean updateTrainIdByNoteBookId(Long noteBookId, Long algorithmId) { - if (noteBookId != null) { - NumberUtil.isNumber(noteBookId); - NoteBook noteBook = noteBookMapper.selectById(noteBookId); - if (noteBook != null) { - noteBook.setAlgorithmId(algorithmId == null ? MagicNumConstant.ZERO : algorithmId); - noteBookMapper.updateById(noteBook); - return true; - } - } - return false; - } - - /** - * 获取notebook详情 - * - * @param noteBookIds - * @return List - */ - @Override - public List getNotebookDetail(Set noteBookIds) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.in("id", noteBookIds); - queryWrapper.ne(NoteBook.COLUMN_STATUS, NoteBookStatusEnum.DELETE.getCode()); - List noteBookList = noteBookMapper.selectList(queryWrapper); - return noteBookConvert.toDto(noteBookList); - } - - /** - * 获取正在运行却没有URL的notebook - * - * @param page - * @return List - */ - @Override - public List getRunNotUrlList(Page page) { - return noteBookMapper.selectRunNotUrlList(page, NoteBookStatusEnum.RUN.getCode()); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java deleted file mode 100644 index 10d38b4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtImageServiceImpl.java +++ /dev/null @@ -1,435 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.async.HarborImagePushAsync; -import org.dubhe.base.DataContext; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.base.ResponseCode; -import org.dubhe.config.NfsConfig; -import org.dubhe.config.TrainHarborConfig; -import org.dubhe.constatnts.UserConstant; -import org.dubhe.dao.PtImageMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.PtImage; -import org.dubhe.domain.vo.PtImageQueryVO; -import org.dubhe.enums.*; -import org.dubhe.exception.BusinessException; -import org.dubhe.harbor.api.HarborApi; -import org.dubhe.service.PtImageService; -import org.dubhe.utils.*; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * @description 镜像服务实现类 - * @date 2020-06-22 - */ -@Service -public class PtImageServiceImpl implements PtImageService { - - @Autowired - private PtImageMapper ptImageMapper; - - @Autowired - private HarborApi harborApi; - - @Autowired - private NfsConfig nfsConfig; - - @Autowired - private HarborImagePushAsync imagePushAsync; - - @Autowired - private TrainHarborConfig trainHarborConfig; - - public final static List FIELD_NAMES; - - static { - FIELD_NAMES = ReflectionUtils.getFieldNames(PtImageQueryVO.class); - } - - /** - * 查询镜像 - * - * @param ptImageQueryDTO 查询镜像条件 - * @return Map 返回镜像分页数据 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Map getImage(PtImageQueryDTO ptImageQueryDTO) { - - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.IMAGE, "The user {} query image list display begins, and the received parameters are :{}", user.getId(), ptImageQueryDTO); - Page page = ptImageQueryDTO.toPage(); - - QueryWrapper query = new QueryWrapper<>(); - - if (ptImageQueryDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { - ptImageQueryDTO.setSort("imageResource"); - DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); - } - if (ptImageQueryDTO.getImageStatus() != null) { - query.eq("image_status", ptImageQueryDTO.getImageStatus()); - } - if (ptImageQueryDTO.getImageResource() != null) { - query.eq("image_resource", ptImageQueryDTO.getImageResource()); - } - query.eq("project_name", resourcetoName(ptImageQueryDTO.getProjectType())); - - if (StringUtils.isNotEmpty(ptImageQueryDTO.getImageNameOrId())) { - query.and(x -> x.eq("id", ptImageQueryDTO.getImageNameOrId()).or().like("image_name", ptImageQueryDTO.getImageNameOrId())); - } - - - //排序 - IPage ptImages; - try { - if (ptImageQueryDTO.getSort() != null && FIELD_NAMES.contains(ptImageQueryDTO.getSort())) { - if (Constant.SORT_ASC.equalsIgnoreCase(ptImageQueryDTO.getOrder())) { - query.orderByAsc(StringUtils.humpToLine(ptImageQueryDTO.getSort())); - } else { - query.orderByDesc(StringUtils.humpToLine(ptImageQueryDTO.getSort())); - } - } else { - query.orderByDesc(Constant.ID); - } - ptImages = ptImageMapper.selectPage(page, query); - } catch (Exception e) { - LogUtil.error(LogEnum.IMAGE, "User {} query mirror list display exception :{}, request information :{}", user.getId(), e, ptImageQueryDTO); - throw new BusinessException("查询镜像列表展示异常"); - } - List ptImageQueryResult = ptImages.getRecords().stream().map(x -> { - PtImageQueryVO ptImageQueryVO = new PtImageQueryVO(); - BeanUtils.copyProperties(x, ptImageQueryVO); - return ptImageQueryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.IMAGE, "User {} query mirror list display ends, the result is {}", user.getId(), ptImageQueryResult); - DataContext.remove(); - return PageUtil.toPage(page, ptImageQueryResult); - } - - /** - * 上传镜像到harbor - * - * @param ptImageUploadDTO 上传条件 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void uploadImage(PtImageUploadDTO ptImageUploadDTO) { - LogUtil.info(LogEnum.IMAGE, "Upload image to harbor to receive parameters :{}", ptImageUploadDTO); - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - - //notebook镜像只能由管理员上传 - if (ptImageUploadDTO.getProjectType().equals(BizEnum.NOTEBOOK.getBizCode()) && UserConstant.ADMIN_USER_ID != currentUser.getId()) { - throw new BusinessException(ResponseCode.UNAUTHORIZED, "该用户不支持上传noteBook镜像!"); - } - - //校验用户自定义镜像不能和预置镜像重名 - List resList = checkUploadImage(ptImageUploadDTO, null, ImageSourceEnum.PRE.getCode()); - if (!CollectionUtils.isEmpty(resList)) { - LogUtil.info(LogEnum.IMAGE, "Preset image [{}:{}] already exists, no repeat allowed!", ptImageUploadDTO.getImageName(), ptImageUploadDTO.getImageTag()); - throw new BusinessException(ResponseCode.BADREQUEST, "不允许和预置镜像信息重复，请重新上传!"); - } - - //同一用户上传镜像的(userId+imageName+imageTag)存在的情况下是不能重复上传的 - List imageList = checkUploadImage(ptImageUploadDTO, currentUser, ImageSourceEnum.MINE.getCode()); - if (!CollectionUtils.isEmpty(imageList) && ImageStateEnum.SUCCESS.getCode().equals(imageList.get(0).getImageStatus())) { - LogUtil.info(LogEnum.IMAGE, "The mirror [id:{}] already exists", imageList.get(0).getId()); - throw new BusinessException(ResponseCode.BADREQUEST, "镜像信息已存在，不允许重复上传!"); - } - - String projectName = ImageTypeEnum.getType(ptImageUploadDTO.getProjectType()); - - String harborImagePath = projectName + StrUtil.SLASH + ptImageUploadDTO.getImageName() + StrUtil.DASHED + currentUser.getId() + - StrUtil.COLON + ptImageUploadDTO.getImageTag(); - //存储镜像信息 - PtImage ptImage = new PtImage(); - ptImage.setImageName(ptImageUploadDTO.getImageName()) - .setProjectName(projectName) - .setImageUrl(harborImagePath) - .setImageResource(ImageSourceEnum.MINE.getCode()) - .setImageStatus(ImageStateEnum.MAKING.getCode()) - .setRemark(ptImageUploadDTO.getRemark()) - .setImageTag(ptImageUploadDTO.getImageTag()) - .setCreateUserId(currentUser.getId()); - if (ptImageUploadDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { - //notebook镜像所有用户都可以查看和使用 - ptImage.setOriginUserId(MagicNumConstant.ZERO_LONG); - } else { - ptImage.setOriginUserId(currentUser.getId()); - } - int count = ptImageMapper.insert(ptImage); - if (count < 1) { - imagePushAsync.updateImageStatus(ptImage, ImageStateEnum.FAIL.getCode()); - LogUtil.info(LogEnum.IMAGE, "User {} failed to store image information!", currentUser.getUsername()); - throw new BusinessException("内部错误!"); - } - //shell脚本上传镜像 - try { - String imagePath = nfsConfig.getRootDir() + nfsConfig.getBucket().substring(1) + ptImageUploadDTO.getImagePath(); - String imageNameAndTag = ptImageUploadDTO.getImageName() + StrUtil.DASHED + currentUser.getId() + StrUtil.COLON + ptImageUploadDTO.getImageTag(); - imagePushAsync.execShell(imagePath, imageNameAndTag, ptImage); - } catch (Exception e) { - LogUtil.error(LogEnum.IMAGE, "Image upload exception :{}", e); - throw new BusinessException("镜像上传失败!"); - } - } - - /** - * 根据镜像获取信息 - * - * @param imageName 镜像名 - * @return List 通过imageName查询所含镜像版本信息 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List searchImages(Integer projectType, String imageName) { - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.eq(PtImage::getProjectName, resourcetoName(projectType)) - .eq(PtImage::getImageName, imageName) - .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); - List ptImages = ptImageMapper.selectList(queryWrapper); - if (CollectionUtils.isEmpty(ptImages)) { - LogUtil.info(LogEnum.IMAGE, "No mirroring information for imageName is :{}", imageName); - throw new BusinessException(ResponseCode.SUCCESS, "未查询到镜像信息!"); - } - List list = new ArrayList<>(); - ptImages = ptImages.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> - new TreeSet<>(Comparator.comparing(PtImage::getImageTag))), ArrayList::new)); - ptImages.stream().forEach(ptImage -> { - ptImage.setImageUrl(trainHarborConfig.getAddress() + StrUtil.SLASH + ptImage.getImageUrl()); - list.add(ptImage); - }); - return list; - } - - - /** - * 删除镜像 - * - * @param imageDeleteDTO 删除镜像条件参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public void deleteTrainImage(PtImageDeleteDTO imageDeleteDTO) { - - UserDTO user = JwtUtils.getCurrentUserDto(); - List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() - .in(PtImage::getId, imageDeleteDTO.getIds())); - - imageList.forEach(image -> { - //禁止删除预置镜像 - if (!BizEnum.NOTEBOOK.getBizCode().equals(image.getProjectName()) && ImageSourceEnum.PRE.getCode().equals(image.getImageResource())) { - throw new BusinessException("禁止删除预置镜像"); - } - if (ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { - String imageUrl = trainHarborConfig.getAddress() + StrUtil.SLASH + image.getImageUrl(); - LogUtil.info(LogEnum.IMAGE, "delete harbor image url:{}", imageUrl); - //同步删除harbor镜像 - harborApi.deleteImageByTag(imageUrl); - } - }); - - - //删除本地镜像 - int deleteSum = ptImageMapper.deleteBatchIds(imageDeleteDTO.getIds()); - if (deleteSum < imageDeleteDTO.getIds().size()) { - LogUtil.error(LogEnum.IMAGE, "The user {} failed to delete image, and the pt_image table deletion operation failed according to the id array {}", user.getId(), imageDeleteDTO.getIds()); - throw new BusinessException("内部错误"); - } - } - - /** - * 修改镜像信息 - * - * @param imageUpdateDTO - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public void updateTrainImage(PtImageUpdateDTO imageUpdateDTO) { - - UserDTO user = JwtUtils.getCurrentUserDto(); - - List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() - .in(PtImage::getId, imageUpdateDTO.getIds())); - - if (CollectionUtils.isEmpty(imageList)) { - LogUtil.error(LogEnum.IMAGE, "The user{} update image failed,inquire condition ids{} not result", user.getId(), imageUpdateDTO.getIds()); - throw new BusinessException("内部错误"); - } - for (PtImage image : imageList) { - //禁止修改预置镜像 - if (ImageSourceEnum.PRE.getCode().equals(image.getImageResource())) { - throw new BusinessException("无法修改预置镜像信息"); - } - image.setRemark(imageUpdateDTO.getRemark()); - LogUtil.info(LogEnum.IMAGE, "The user{}update image,update image info:{}", user.getId(), image); - ptImageMapper.updateById(image); - } - } - - /** - * 获取镜像名称列表 - * @param projectType 镜像项目类型 - * @return Set 镜像列表 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Set getImageNameList(Integer projectType) { - List imageList = ptImageMapper.selectList(new LambdaQueryWrapper() - .eq(PtImage::getProjectName, ImageTypeEnum.getType(projectType)) - .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode())); - Set imageNames = new HashSet<>(); - imageList.forEach(image -> { - imageNames.add(image.getImageName()); - }); - return imageNames; - } - - /** - * 修改镜像来源(notebook定制) - * - * @param id 镜像id - */ - @Override - public void updImageResource(Long id) { - UserDTO userDto = JwtUtils.getCurrentUserDto(); - UpdateWrapper updateWrapper = new UpdateWrapper<>(); - //notebook镜像只能由管理员上传 - if (UserConstant.ADMIN_USER_ID != userDto.getId()) { - throw new BusinessException(ResponseCode.UNAUTHORIZED, "该用户无权限修改镜像状态!"); - } - - //校验id是否存在 - PtImage image = ptImageMapper.selectById(id); - if (image == null || !BizEnum.NOTEBOOK.getBizCode().equals(image.getProjectName())) { - throw new BusinessException(ResponseCode.BADREQUEST, "该镜像不存在或镜像类型不支持!"); - } - - //仅支持[制作成功]状态镜像设置为默认镜像 - if (!ImageStateEnum.SUCCESS.getCode().equals(image.getImageStatus())) { - throw new BusinessException(ResponseCode.BADREQUEST, "仅支持[制作成功]状态镜像设置为默认镜像!"); - } - - //修改该用户的notebook镜像为"我的镜像" - updateWrapper.eq("project_name", BizEnum.NOTEBOOK.getBizCode()); - updateWrapper.eq("image_resource", ImageSourceEnum.PRE.getCode()); - updateWrapper.set("image_resource", ImageSourceEnum.MINE.getCode()); - ptImageMapper.update(null, updateWrapper); - - PtImage ptImage = new PtImage(); - ptImage.setId(id); - ptImage.setImageResource(ImageSourceEnum.PRE.getCode()); - ptImageMapper.updateById(ptImage); - } - - /** - * 获取镜像URL - * - * @param imageQueryUrlDTO 查询镜像路径DTO - * @return 镜像完整路径 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public String getImageUrl(PtImageQueryUrlDTO imageQueryUrlDTO) { - UserDTO userDto = JwtUtils.getCurrentUserDto(); - - if (imageQueryUrlDTO.getProjectType().equals(BizEnum.NOTEBOOK.getCreateResource())) { - DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); - } - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - if (imageQueryUrlDTO.getProjectType() != null && ImageTypeEnum.NOTEBOOK.getType().equals(imageQueryUrlDTO.getProjectType())) { - DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); - } - if (imageQueryUrlDTO.getImageResource() != null) { - queryWrapper.eq(PtImage::getImageResource, imageQueryUrlDTO.getImageResource()); - } - if (StrUtil.isNotEmpty(imageQueryUrlDTO.getImageName())) { - queryWrapper.eq(PtImage::getImageName, imageQueryUrlDTO.getImageName()); - } - if (StrUtil.isNotEmpty(imageQueryUrlDTO.getIamgeTag())) { - queryWrapper.eq(PtImage::getImageTag, imageQueryUrlDTO.getIamgeTag()); - } - queryWrapper.eq(PtImage::getProjectName, resourcetoName(imageQueryUrlDTO.getProjectType())) - .eq(PtImage::getImageStatus, ImageStateEnum.SUCCESS.getCode()); - List imageList = ptImageMapper.selectList(queryWrapper); - - if (CollUtil.isEmpty(imageList)) { - LogUtil.error(LogEnum.IMAGE, "The user{} update image failed,inquire condition {} not result", userDto.getId(), imageQueryUrlDTO); - throw new BusinessException("未查询到镜像信息"); - } - String imageUrl = trainHarborConfig.getAddress() + StrUtil.SLASH + imageList.get(0).getImageUrl(); - DataContext.remove(); - return imageUrl; - } - - /** - * @param ptImageUploadDTO 镜像上传逻辑校验 - * @param user 用户 - * @return List 镜像列表 - **/ - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - private List checkUploadImage(PtImageUploadDTO ptImageUploadDTO, UserDTO user, Integer source) { - - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - - queryWrapper.eq(PtImage::getProjectName, resourcetoName(ptImageUploadDTO.getProjectType())) - .eq(PtImage::getImageName, ptImageUploadDTO.getImageName()) - .eq(PtImage::getImageTag, ptImageUploadDTO.getImageTag()) - .eq(PtImage::getImageResource, source); - - if (user != null) { - queryWrapper.eq(PtImage::getCreateUserId, user.getId()); - } - List imageList = ptImageMapper.selectList(queryWrapper); - return imageList; - } - - /** - * 项目类型转换为项目名称 - * - * @param projectType 项目类型 - * @return String 镜像项目名称 - */ - private static String resourcetoName(Integer projectType) { - String projectName = ImageTypeEnum.getType(projectType); - if (StrUtil.isEmpty(projectName)) { - throw new BusinessException("上传镜像项目类型不支持"); - } - return projectName; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java deleted file mode 100644 index 5089df2..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtMeasureServiceImpl.java +++ /dev/null @@ -1,274 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ -package org.dubhe.service.impl; - -import cn.hutool.core.collection.CollUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.json.JSONUtil; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.config.NfsConfig; -import org.dubhe.config.RecycleConfig; -import org.dubhe.dao.PtMeasureMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.PtMeasure; -import org.dubhe.domain.vo.PtImageQueryVO; -import org.dubhe.domain.vo.PtMeasureQueryVO; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.RecycleModuleEnum; -import org.dubhe.enums.RecycleTypeEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.PtMeasureService; -import org.dubhe.service.RecycleTaskService; -import org.dubhe.utils.*; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @description 度量管理实现类 - * @date 2020-11-16 - */ -@Service -public class PtMeasureServiceImpl implements PtMeasureService { - - - @Autowired - private PtMeasureMapper ptMeasureMapper; - - @Autowired - private NfsUtil nfsUtil; - - @Autowired - private NfsConfig nfsConfig; - - @Autowired - private RecycleTaskService recycleTaskService; - - @Autowired - private RecycleConfig recycleConfig; - - public final static List FIELD_NAMES; - - static { - FIELD_NAMES = ReflectionUtils.getFieldNames(PtImageQueryVO.class); - } - - - /** - * 查询度量信息 - * - * @param ptMeasureQueryDTO 查询条件 - * @return Map 度量列表分页信息 - */ - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - @Override - public Map getMeasure(PtMeasureQueryDTO ptMeasureQueryDTO) { - - //从会话中获取用户信息 - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - Page page = ptMeasureQueryDTO.toPage(); - - QueryWrapper query = new QueryWrapper<>(); - if (StrUtil.isNotEmpty(ptMeasureQueryDTO.getNameOrId())) { - query.and(x -> x.eq("id", ptMeasureQueryDTO.getNameOrId()).or().like("name", ptMeasureQueryDTO.getNameOrId())); - } - - //排序 - IPage ptMeasures; - try { - if (StrUtil.isNotEmpty(ptMeasureQueryDTO.getSort()) && FIELD_NAMES.contains(ptMeasureQueryDTO.getSort())) { - if (Constant.SORT_ASC.equalsIgnoreCase(ptMeasureQueryDTO.getOrder())) { - query.orderByAsc(StringUtils.humpToLine(ptMeasureQueryDTO.getSort())); - } else { - query.orderByDesc(StringUtils.humpToLine(ptMeasureQueryDTO.getSort())); - } - } else { - query.orderByDesc(Constant.ID); - } - ptMeasures = ptMeasureMapper.selectPage(page, query); - } catch (Exception e) { - LogUtil.error(LogEnum.MEASURE, "User {} query measure list display exception :{}, request information :{}", currentUser.getId(), e, ptMeasureQueryDTO); - throw new BusinessException("查询度量列表展示异常"); - } - - List ptMeasureQueryResult = ptMeasures.getRecords().stream().map(x -> { - PtMeasureQueryVO ptMeasureQueryVO = new PtMeasureQueryVO(); - BeanUtils.copyProperties(x, ptMeasureQueryVO); - return ptMeasureQueryVO; - }).collect(Collectors.toList()); - return PageUtil.toPage(page, ptMeasureQueryResult); - } - - /** - * 新建度量 - * - * @param ptMeasureCreateDTO 新建度量入参DTO - */ - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - @Transactional(rollbackFor = Exception.class) - @Override - public void createMeasure(PtMeasureDTO ptMeasureCreateDTO) { - //从会话中获取用户信息 - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - - PtMeasure ptMeasure = new PtMeasure(); - BeanUtils.copyProperties(ptMeasureCreateDTO, ptMeasure); - ptMeasure.setCreateUserId(currentUser.getId()); - try { - ptMeasureMapper.insert(ptMeasure); - } catch (Exception e) { - LogUtil.error(LogEnum.MEASURE, " pt_measure table insert operation failed,Cause of exception is {}", e); - throw new BusinessException("内部错误"); - } - } - - /** - * 修改度量 - * - * @param ptMeasureUpdateDTO 修改度量入参DTO - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void updateMeasure(PtMeasureUpdateDTO ptMeasureUpdateDTO) { - //从会话中获取用户信息 - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - - PtMeasure ptMeasure = new PtMeasure(); - BeanUtils.copyProperties(ptMeasureUpdateDTO, ptMeasure); - ptMeasure.setUpdateUserId(currentUser.getId()); - - try { - PtMeasure measure = ptMeasureMapper.selectById(ptMeasureUpdateDTO.getId()); - if (measure == null) { - LogUtil.error(LogEnum.MEASURE, "The user{} update measure failed,inquire condition id{} no result", currentUser.getId(), ptMeasureUpdateDTO.getId()); - throw new BusinessException("内部错误"); - } - ptMeasureMapper.updateById(ptMeasure); - } catch (BusinessException e) { - LogUtil.error(LogEnum.MEASURE, " pt_measure table update operation failed,Cause of exception is {}", e); - throw new BusinessException("内部错误"); - } - } - - /** - * 根据id删除度量 - * - * @param ptMeasureDeleteDTO 删除度量的条件DTO - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void deleteMeasure(PtMeasureDeleteDTO ptMeasureDeleteDTO) { - //从会话中获取用户信息 - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - try { - - Set idList = ptMeasureDeleteDTO.getIds(); - List measureList = ptMeasureMapper.selectBatchIds(idList); - if (CollUtil.isEmpty(idList)) { - LogUtil.error(LogEnum.MEASURE, "The user:{} delete measure failed,inquire condition id:{} no result", currentUser.getId(), ptMeasureDeleteDTO.getIds()); - throw new BusinessException("您删除的ID不存在或已被删除"); - } - - int count = ptMeasureMapper.deleteBatchIds(idList); - if (count < measureList.size()) { - LogUtil.error(LogEnum.MEASURE, "The user:{} delete measure failed,inquire condition id:{} no result", currentUser.getId(), ptMeasureDeleteDTO.getIds()); - throw new BusinessException("您删除的ID不存在或已被删除"); - } - measureList.forEach(x -> { - delMeasureFile(x.getUrl()); - }); - - } catch (BusinessException e) { - LogUtil.error(LogEnum.MEASURE, " pt_measure table delete operation failed,Cause of exception is {}", e); - throw new BusinessException("内部错误"); - } - } - - /** - * 根据度量名称返回度量文件信息 - * - * @param name 度量名称 - * @return String 度量文件json字符串 - */ - @Override - public String getMeasureByName(String name) { - - List ptMeasureList = ptMeasureMapper.selectList(new LambdaQueryWrapper() - .eq(PtMeasure::getName, name)); - BufferedInputStream bufferedInput = null; - byte[] buffer = new byte[1024]; - StringBuilder sb = new StringBuilder(); - try { - if (CollUtil.isNotEmpty(ptMeasureList)) { - String url = StrUtil.SLASH + nfsConfig.getBucket() + StrUtil.SLASH + ptMeasureList.get(0).getUrl(); - if (!nfsUtil.fileOrDirIsEmpty(url)) { - int bytesRead = 0; - //获取文件输入流 - bufferedInput = nfsUtil.getInputStream(url); - while ((bytesRead = bufferedInput.read(buffer)) != -1) { - //将读取的字节转为字符串对象 - String tmpStr = new String(buffer, 0, bytesRead); - sb.append(tmpStr); - } - } - } - } catch (IOException e) { - LogUtil.error(LogEnum.MEASURE, " getMeasureByName method read jsonFile operation failed,Cause of exception is {}", e); - throw new BusinessException("内部错误"); - } finally { - IOUtil.close(bufferedInput); - } - return JSONUtil.toJsonStr(sb); - } - - /** - * 删除nfs服务器上的度量文件 - * - * @param filePath 度量文件相对路径 - */ - private void delMeasureFile(String filePath) { - String nfsBucket = nfsConfig.getRootDir() + nfsConfig.getBucket() + StrUtil.SLASH; - //度量文件的nfs服务器相对路径 - filePath = filePath.substring(0, filePath.lastIndexOf(StrUtil.SLASH)); - //度量文件的nfs服务器绝对路径 - String recyclePath = nfsUtil.formatPath((nfsBucket + StrUtil.SLASH + filePath)); - - //创建已删除度量的无效文件回收任务 - RecycleTaskCreateDTO recycleTask = new RecycleTaskCreateDTO(); - recycleTask.setRecycleModule(RecycleModuleEnum.BIZ_TRAIN.getValue()) - .setRecycleType(RecycleTypeEnum.FILE.getCode()) - .setRecycleDelayDate(recycleConfig.getTrainValid()) - .setRecycleCondition(recyclePath) - .setRecycleNote("回收已删除度量文件"); - LogUtil.info(LogEnum.MEASURE, "delete measure add recycle task:{}", recycleTask); - recycleTaskService.createRecycleTask(recycleTask); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java deleted file mode 100644 index 8aff34a..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmServiceImpl.java +++ /dev/null @@ -1,455 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.util.RandomUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.map.HashedMap; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.async.TrainAlgorithmUploadAsync; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.base.ResponseCode; -import org.dubhe.config.NfsConfig; -import org.dubhe.config.RecycleConfig; -import org.dubhe.constant.AlgorithmSourceEnum; -import org.dubhe.constant.TrainAlgorithmConstant; -import org.dubhe.dao.NoteBookMapper; -import org.dubhe.dao.PtImageMapper; -import org.dubhe.dao.PtTrainAlgorithmMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.NoteBook; -import org.dubhe.domain.entity.PtImage; -import org.dubhe.domain.entity.PtTrainAlgorithm; -import org.dubhe.domain.vo.PtTrainAlgorithmQueryVO; -import org.dubhe.enums.*; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.NoteBookService; -import org.dubhe.service.PtTrainAlgorithmService; -import org.dubhe.service.RecycleTaskService; -import org.dubhe.utils.*; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - - -/** - * @description 训练算法 服务实现类 - * @date 2020-04-27 - */ -@Service -public class PtTrainAlgorithmServiceImpl implements PtTrainAlgorithmService { - - @Autowired - private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; - - @Autowired - private PtImageMapper ptImageMapper; - - @Autowired - private NfsUtil nfsUtil; - - @Autowired - private LocalFileUtil localFileUtil; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private NfsConfig nfsConfig; - - @Autowired - private TrainAlgorithmConstant trainAlgorithmConstant; - - @Autowired - private NoteBookService noteBookService; - - @Autowired - private NoteBookMapper noteBookMapper; - - @Autowired - private TrainAlgorithmUploadAsync algorithmUpdateAsync; - - @Autowired - private RecycleTaskService recycleTaskService; - - @Autowired - private RecycleConfig recycleConfig; - - public final static List FIELD_NAMES; - - static { - FIELD_NAMES = ReflectionUtils.getFieldNames(PtTrainAlgorithmQueryVO.class); - } - - /** - * 查询数据分页 - * - * @param ptTrainAlgorithmQueryDTO 条件 - * @return Map 返回查询数据 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Map queryAll(PtTrainAlgorithmQueryDTO ptTrainAlgorithmQueryDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The display of user {} query training algorithm list begins, and the parameters received are {}.", user.getId(), ptTrainAlgorithmQueryDTO); - //当算法来源为空时，设置默认算法来源 - if (ptTrainAlgorithmQueryDTO.getAlgorithmSource() == null) { - ptTrainAlgorithmQueryDTO.setAlgorithmSource(trainAlgorithmConstant.getAlgorithmSource()); - } - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("algorithm_source", ptTrainAlgorithmQueryDTO.getAlgorithmSource()); - //判断算法来源 - if (AlgorithmSourceEnum.MINE.getStatus().equals(ptTrainAlgorithmQueryDTO.getAlgorithmSource())) { - wrapper.eq("create_user_id", user.getId()); - } - //根据算法用途筛选 - if (ptTrainAlgorithmQueryDTO.getAlgorithmUsage() != null) { - wrapper.like("algorithm_usage", ptTrainAlgorithmQueryDTO.getAlgorithmUsage()); - } - if (!StringUtils.isEmpty(ptTrainAlgorithmQueryDTO.getAlgorithmName())) { - wrapper.and(qw -> qw.eq("id", ptTrainAlgorithmQueryDTO.getAlgorithmName()).or().like("algorithm_name", - ptTrainAlgorithmQueryDTO.getAlgorithmName())); - } - - Page page = ptTrainAlgorithmQueryDTO.toPage(); - IPage ptTrainAlgorithms; - try { - if (ptTrainAlgorithmQueryDTO.getSort() != null && FIELD_NAMES.contains(ptTrainAlgorithmQueryDTO.getSort())) { - if (Constant.SORT_ASC.equalsIgnoreCase(ptTrainAlgorithmQueryDTO.getOrder())) { - wrapper.orderByAsc(StringUtils.humpToLine(ptTrainAlgorithmQueryDTO.getSort())); - } else { - wrapper.orderByDesc(StringUtils.humpToLine(ptTrainAlgorithmQueryDTO.getSort())); - } - } else { - wrapper.orderByDesc(Constant.ID); - } - ptTrainAlgorithms = ptTrainAlgorithmMapper.selectPage(page, wrapper); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Query training algorithm list display exceptions :{}, request information :{}", e, - ptTrainAlgorithmQueryDTO); - throw new BusinessException("查询训练算法列表展示异常"); - } - List ptTrainAlgorithmQueryResult = ptTrainAlgorithms.getRecords().stream().map(x -> { - PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO = new PtTrainAlgorithmQueryVO(); - BeanUtils.copyProperties(x, ptTrainAlgorithmQueryVO); - //获取镜像名称与版本 - getImageNameAndImageTag(x, ptTrainAlgorithmQueryVO); - return ptTrainAlgorithmQueryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query training algorithm list display ends, the result is {}", user.getUsername(), ptTrainAlgorithmQueryResult); - return PageUtil.toPage(page, ptTrainAlgorithmQueryResult); - } - - /** - * 新增算法 - * - * @param ptTrainAlgorithmCreateDTO 新增算法条件 - * @return idList 返回新增算法 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public Long create(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "Save the new algorithm and receive the parameter {}", ptTrainAlgorithmCreateDTO); - //获取镜像url - if (StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageName()) && StringUtils.isNotBlank(ptTrainAlgorithmCreateDTO.getImageTag())) { - ptTrainAlgorithmCreateDTO.setImageName(getImageUrl(ptTrainAlgorithmCreateDTO, user)); - } - //创建算法校验DTO并设置默认值 - setAlgorithmDtoDefault(ptTrainAlgorithmCreateDTO); - //算法路径 - String path = nfsConfig.getBucket() + ptTrainAlgorithmCreateDTO.getCodeDir(); - if (nfsUtil.fileOrDirIsEmpty(path)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} upload path {} does not exist", user.getUsername(), path); - throw new BusinessException("算法文件或路径不存在"); - } - //保存算法 - PtTrainAlgorithm ptTrainAlgorithm = new PtTrainAlgorithm(); - BeanUtils.copyProperties(ptTrainAlgorithmCreateDTO, ptTrainAlgorithm); - ptTrainAlgorithm.setAlgorithmSource(AlgorithmSourceEnum.MINE.getStatus()) - .setCreateUserId(user.getId()); - - //算法名称校验 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("algorithm_name", ptTrainAlgorithmCreateDTO.getAlgorithmName()); - Integer countResult = ptTrainAlgorithmMapper.selectCount(queryWrapper); - //如果是通过【保存至算法】接口创建算法，名称重复可用随机数生成新算法名，待后续客户自主修改 - if (countResult > 0) { - if (ptTrainAlgorithmCreateDTO.getNoteBookId() != null) { - String randomStr = RandomUtil.randomNumbers(MagicNumConstant.FOUR); - ptTrainAlgorithm.setAlgorithmName(ptTrainAlgorithmCreateDTO.getAlgorithmName() + randomStr); - } else { - LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm name ({}) already exists", ptTrainAlgorithmCreateDTO.getAlgorithmName()); - throw new BusinessException("算法名称已存在，请重新输入"); - } - } - //校验path是否带有压缩文件，如有，则解压至当前文件夹并删除压缩文件 - if (path.toLowerCase().endsWith(Constant.COMPRESS_ZIP)) { - unZip(user, path, ptTrainAlgorithm); - } - try { - //算法未保存成功，抛出异常，并返回失败信息 - ptTrainAlgorithmMapper.insert(ptTrainAlgorithm); - - //上传算法异步处理 - algorithmUpdateAsync.createTrainAlgorithm(user, ptTrainAlgorithm, ptTrainAlgorithmCreateDTO); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} saving algorithm was not successful. Failure reason :{}", user.getUsername(), e.getMessage()); - throw new BusinessException("算法未保存成功"); - } - - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} saves the new algorithm and returns the new algorithm id as {}.", user.getUsername(), ptTrainAlgorithm.getId()); - return ptTrainAlgorithm.getId(); - } - - /** - * 修改算法 - * - * @param ptTrainAlgorithmUpdateDTO 修改算法条件 - * @return PtTrainAlgorithmUpdateVO 返回修改算法 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public Long update(PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the algorithm and receives {} as the parameter", currentUser.getUsername(), ptTrainAlgorithmUpdateDTO); - //权限校验 - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(ptTrainAlgorithmUpdateDTO.getId()); - if (null == ptTrainAlgorithm || ptTrainAlgorithm.getCreateUserId().compareTo(currentUser.getId()) != 0) { - LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for the user {} to modify the algorithm with id {}", currentUser.getUsername(), ptTrainAlgorithmUpdateDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "您修改的算法不存在或已被删除"); - } - PtTrainAlgorithm updatePtAlgorithm = new PtTrainAlgorithm(); - updatePtAlgorithm.setId(ptTrainAlgorithm.getId()).setUpdateUserId(currentUser.getId()); - //判断是否修改算法名称 - if (StringUtils.isNotBlank(ptTrainAlgorithmUpdateDTO.getAlgorithmName())) { - //算法名称校验 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("algorithm_name", ptTrainAlgorithmUpdateDTO.getAlgorithmName()) - .ne("id", ptTrainAlgorithmUpdateDTO.getId()); - Integer countResult = ptTrainAlgorithmMapper.selectCount(queryWrapper); - if (countResult > 0) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm name ({}) already exists", ptTrainAlgorithmUpdateDTO.getAlgorithmName()); - throw new BusinessException("算法名称已存在，请重新输入"); - } - updatePtAlgorithm.setAlgorithmName(ptTrainAlgorithmUpdateDTO.getAlgorithmName()); - } - //判断是否修改算法描述 - if (ptTrainAlgorithmUpdateDTO.getDescription() != null) { - updatePtAlgorithm.setDescription(ptTrainAlgorithmUpdateDTO.getDescription()); - } - //判断是否修改算法用途 - if (ptTrainAlgorithmUpdateDTO.getAlgorithmUsage() != null) { - updatePtAlgorithm.setAlgorithmUsage(ptTrainAlgorithmUpdateDTO.getAlgorithmUsage()); - } - //判断是否修改输出日志 - if (ptTrainAlgorithmUpdateDTO.getIsTrainLog() != null) { - updatePtAlgorithm.setIsTrainLog(ptTrainAlgorithmUpdateDTO.getIsTrainLog()); - } - //判断是否修改可视化日志 - if (ptTrainAlgorithmUpdateDTO.getIsVisualizedLog() != null) { - updatePtAlgorithm.setIsVisualizedLog(ptTrainAlgorithmUpdateDTO.getIsVisualizedLog()); - } - try { - //算法未修改成功，抛出异常，并返回失败信息 - ptTrainAlgorithmMapper.updateById(updatePtAlgorithm); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify the algorithm. Pt_train_algorithm table modification operation failed. Failure reason :{}", currentUser.getUsername(), e.getMessage()); - throw new BusinessException("修改失败"); - } - LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} modification algorithm, return modification algorithm ID={}", currentUser.getUsername(), ptTrainAlgorithm.getId()); - return ptTrainAlgorithm.getId(); - } - - /** - * 解压缩zip压缩包 - * - * @param user 用户 - * @param path 文件路径 - * @param ptTrainAlgorithm 算法参数 - */ - private void unZip(UserDTO user, String path, PtTrainAlgorithm ptTrainAlgorithm) { - //目标路径 - String targetPath = k8sNameTool.getNfsPath(BizNfsEnum.ALGORITHM, user.getId()); - boolean unzip = localFileUtil.unzipLocalPath(path, nfsConfig.getBucket() + targetPath); - if (!unzip) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to unzip", user.getUsername()); - throw new BusinessException("内部错误"); - } - //算法路径 - ptTrainAlgorithm.setCodeDir(targetPath); - } - - /** - * 删除算法 - * - * @param ptTrainAlgorithmDeleteDTO 删除算法条件 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public void deleteAll(PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} delete algorithm, the parameter received is {}", user.getUsername(), ptTrainAlgorithmDeleteDTO.getIds()); - Set idList = ptTrainAlgorithmDeleteDTO.getIds(); - //权限校验 - QueryWrapper query = new QueryWrapper<>(); - query.in("id", idList); - List algorithmList = ptTrainAlgorithmMapper.selectList(query); - if (algorithmList.size() < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} delete algorithm failed, no permission to delete the corresponding data in the algorithm table", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); - } - int deleteCountResult = ptTrainAlgorithmMapper.deleteBatchIds(idList); - if (deleteCountResult < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} deletion algorithm failed, and the algorithm table deletion operation based on the ID array {} failed", user.getUsername(), ptTrainAlgorithmDeleteDTO.getIds()); - throw new BusinessException(ResponseCode.SUCCESS, "删除算法未成功"); - } - //同步更新noteBook表中algorithmId=0 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.in("algorithm_id", idList); - List noteBookList = noteBookMapper.selectList(queryWrapper); - if (!CollectionUtils.isEmpty(noteBookList)) { - noteBookList.forEach(noteBook -> { - noteBookService.updateTrainIdByNoteBookId(noteBook.getId(), null); - }); - } - //定时任务删除相应的算法文件 - for (PtTrainAlgorithm algorithm : algorithmList) { - RecycleTaskCreateDTO recycleTask = new RecycleTaskCreateDTO(); - recycleTask.setRecycleModule(RecycleModuleEnum.BIZ_ALGORITHM.getValue()) - .setRecycleType(RecycleTypeEnum.FILE.getCode()) - .setRecycleDelayDate(recycleConfig.getAlgorithmValid()) - .setRecycleCondition(nfsUtil.formatPath(nfsConfig.getRootDir() + nfsConfig.getBucket() + algorithm.getCodeDir())) - .setRecycleNote("删除算法文件"); - recycleTaskService.createRecycleTask(recycleTask); - } - - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} delete algorithm end, delete algorithm ID array IDS ={}", user.getUsername(), idList); - } - - /** - * 查询算法个数 - * - * @return count 返回个数 - */ - @Override - public Map getAlgorithmCount() { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} queries his algorithm number", user.getUsername()); - QueryWrapper wrapper = new QueryWrapper(); - wrapper.eq("algorithm_source", AlgorithmSourceEnum.MINE.getStatus()); - Integer countResult = ptTrainAlgorithmMapper.selectCount(wrapper); - return new HashedMap() {{ - put("count", countResult); - }}; - } - - /** - * 获取镜像名称与版本 - * - * @param trainAlgorithm 镜像URL - * @param ptTrainAlgorithmQueryVO 镜像名称与版本 - */ - private void getImageNameAndImageTag(PtTrainAlgorithm trainAlgorithm, PtTrainAlgorithmQueryVO ptTrainAlgorithmQueryVO) { - if (StringUtils.isNotBlank(trainAlgorithm.getImageName())) { - String imageNameSuffix = trainAlgorithm.getImageName().substring(trainAlgorithm.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); - String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); - ptTrainAlgorithmQueryVO.setImageName(imageNameSuffixArray[0]); - ptTrainAlgorithmQueryVO.setImageTag(imageNameSuffixArray[1]); - } - } - - - /** - * 创建算法DTO校验并设置默认值 - * - * @param dto 校验DTO - **/ - private void setAlgorithmDtoDefault(PtTrainAlgorithmCreateDTO dto) { - - //设置fork默认值 - if (dto.getFork() == null) { - dto.setFork(trainAlgorithmConstant.getFork()); - } - //设置是否输出训练日志 - if (dto.getIsTrainLog() == null) { - dto.setIsTrainLog(trainAlgorithmConstant.getIsTrainLog()); - } - //设置是否输出训练结果 - if (dto.getIsTrainOut() == null) { - dto.setIsTrainOut(trainAlgorithmConstant.getIsTrainOut()); - } - //设置是否输出可视化日志 - if (dto.getIsVisualizedLog() == null) { - dto.setIsVisualizedLog(trainAlgorithmConstant.getIsVisualizedLog()); - } - } - - /** - * 获取镜像url - * - * @param ptTrainAlgorithmCreateDTO 获取镜像 - * @param user 用户 - * @return String 返回镜像路径 - **/ - private String getImageUrl(PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO, UserDTO user) { - //获取镜像url - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("image_name", ptTrainAlgorithmCreateDTO.getImageName()) - .eq("image_tag", ptTrainAlgorithmCreateDTO.getImageTag()) - .eq("image_status", ImageStateEnum.SUCCESS.getCode()).last(" limit 1 "); - ; - PtImage ptImage = ptImageMapper.selectOne(queryWrapper); - if (ptImage == null || StringUtils.isBlank(ptImage.getImageUrl())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} gets image ,the imageName is {}, the imageTag is {}, and the result of query image table (PT_image) is empty", user.getUsername(), ptTrainAlgorithmCreateDTO.getImageName(), ptTrainAlgorithmCreateDTO.getImageTag()); - throw new BusinessException("镜像不存在"); - } - return ptImage.getImageUrl(); - } - - /** - * - * @param ptModelAlgorithmCreateDTO 模型优化上传算法入参 - * @return PtTrainAlgorithm 新增算法信息 - */ - @Override - public PtTrainAlgorithm modelOptimizationUploadAlgorithm(PtModelAlgorithmCreateDTO ptModelAlgorithmCreateDTO) { - PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); - ptTrainAlgorithmCreateDTO.setAlgorithmName(ptModelAlgorithmCreateDTO.getName()).setCodeDir(ptModelAlgorithmCreateDTO.getPath()).setAlgorithmUsage("模型优化").setIsTrainOut(false).setIsTrainLog(false).setIsVisualizedLog(false); - Long id = create(ptTrainAlgorithmCreateDTO); - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(id); - return ptTrainAlgorithm; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java deleted file mode 100644 index 32a9e75..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainAlgorithmUsageServiceImpl.java +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.aspect.PermissionAspect; -import org.dubhe.base.DataContext; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.base.ResponseCode; -import org.dubhe.dao.PtTrainAlgorithmUsageMapper; -import org.dubhe.domain.dto.*; -import org.dubhe.domain.entity.PtTrainAlgorithmUsage; -import org.dubhe.domain.vo.PtTrainAlgorithmUsageQueryVO; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.PtTrainAlgorithmUsageService; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.WrapperHelp; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @description 用途管理 服务实现类 - * @date 2020-06-23 - */ -@Service -public class PtTrainAlgorithmUsageServiceImpl implements PtTrainAlgorithmUsageService { - - @Autowired - private PtTrainAlgorithmUsageMapper ptTrainAlgorithUsagemMapper; - - /** - * 算法用途 - * - * @param ptTrainAlgorithmUsageQueryDTO 查询算法用途参数 - * @return Map 返回查询算法用途分页 - */ - @Override - public Map queryAll(PtTrainAlgorithmUsageQueryDTO ptTrainAlgorithmUsageQueryDTO) { - - // 从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - QueryWrapper wrapper = WrapperHelp.getWrapper(ptTrainAlgorithmUsageQueryDTO); - - Page page = ptTrainAlgorithmUsageQueryDTO.toPage(); - IPage ptTrainAlgorithms = null; - - if (ptTrainAlgorithmUsageQueryDTO.getIsContainDefault()) { - wrapper.in("origin_user_id", user.getId(), PermissionAspect.PUBLIC_DATA_USER_ID); - } else { - wrapper.eq("origin_user_id", user.getId()); - } - - wrapper.eq("type", ptTrainAlgorithmUsageQueryDTO.getType()); - - DataContext.set(CommonPermissionDataDTO.builder().type(true).build()); - ptTrainAlgorithms = ptTrainAlgorithUsagemMapper.selectPage(page, wrapper); - DataContext.remove(); - - List ptTrainAlgorithmUsageQueryResult = ptTrainAlgorithms.getRecords().stream() - .map(x -> { - PtTrainAlgorithmUsageQueryVO ptTrainAlgorithmUsageQueryVO = new PtTrainAlgorithmUsageQueryVO(); - BeanUtils.copyProperties(x, ptTrainAlgorithmUsageQueryVO); - ptTrainAlgorithmUsageQueryVO.setIsDefault(Objects.equals(x.getOriginUserId(),PermissionAspect.PUBLIC_DATA_USER_ID)); - return ptTrainAlgorithmUsageQueryVO; - }).collect(Collectors.toList()); - return PageUtil.toPage(page, ptTrainAlgorithmUsageQueryResult); - } - - /** - * 新增算法用途 - * - * @param ptTrainAlgorithmUsageCreateDTO 新增算法用途参数 - * @return List 返回新增算法用途 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public List create(PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO) { - UserDTO user = JwtUtils.getCurrentUserDto(); - PtTrainAlgorithmUsage ptTrainAlgorithmUsage = new PtTrainAlgorithmUsage(); - ptTrainAlgorithmUsage.setAuxInfo(ptTrainAlgorithmUsageCreateDTO.getAuxInfo()) - .setType(ptTrainAlgorithmUsageCreateDTO.getType()); - - int insertResult = ptTrainAlgorithUsagemMapper.insert(ptTrainAlgorithmUsage); - - if (insertResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} secondary information was not saved successfully", user.getUsername()); - throw new BusinessException("用户辅助信息未保存成功"); - } - return Collections.singletonList(ptTrainAlgorithmUsage.getId()); - } - - /** - * 删除算法用途 - * - * @param ptTrainAlgorithmUsageDeleteDTO 删除算法用途参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void deleteAll(PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO) { - UserDTO user = JwtUtils.getCurrentUserDto(); - Set idList = Stream.of(ptTrainAlgorithmUsageDeleteDTO.getIds()).collect(Collectors.toSet()); - QueryWrapper query = new QueryWrapper<>(); - query.in("id", idList); - Integer queryCountResult = ptTrainAlgorithUsagemMapper.selectCount(query); - - if (queryCountResult < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete user secondary. No permissions to delete corresponding data in user secondary table", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); - } - int deleteCountResult = ptTrainAlgorithUsagemMapper.deleteBatchIds(idList); - - if (deleteCountResult < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete user assistance information. User service deletion based on id array {} failed", user.getUsername(), ptTrainAlgorithmUsageDeleteDTO.getIds()); - throw new BusinessException(ResponseCode.SUCCESS, "内部错误"); - } - } - - /** - *更新算法用途 - * - * @param ptTrainAlgorithmUsageUpdateDTO 更新算法用途参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public void update(PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO) { - UserDTO user = JwtUtils.getCurrentUserDto(); - - QueryWrapper query = new QueryWrapper<>(); - query.in("id", ptTrainAlgorithmUsageUpdateDTO.getId()); - Integer queryIntResult = ptTrainAlgorithUsagemMapper.selectCount(query); - - if (queryIntResult < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify user auxiliary information and has no permission to modify the corresponding data in the algorithm table", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您修改的ID不存在或已被删除,请重新输入"); - } - PtTrainAlgorithmUsage ptTrainAlgorithmUsage = new PtTrainAlgorithmUsage(); - ptTrainAlgorithmUsage.setId(ptTrainAlgorithmUsageUpdateDTO.getId()); - ptTrainAlgorithmUsage.setAuxInfo(ptTrainAlgorithmUsageUpdateDTO.getAuxInfo()); - - int updateResult = ptTrainAlgorithUsagemMapper.updateById(ptTrainAlgorithmUsage); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to modify user assistance information", user.getUsername()); - throw new BusinessException("内部错误"); - } - - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java deleted file mode 100644 index f6400ea..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobServiceImpl.java +++ /dev/null @@ -1,1494 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.util.StrUtil; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.map.HashedMap; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.async.StopTrainJobAsync; -import org.dubhe.async.TransactionAsyncManager; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.base.ResponseCode; -import org.dubhe.config.NfsConfig; -import org.dubhe.config.RecycleConfig; -import org.dubhe.config.TrainHarborConfig; -import org.dubhe.config.TrainJobConfig; -import org.dubhe.constant.AlgorithmSourceEnum; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.DictDetailMapper; -import org.dubhe.dao.PtJobParamMapper; -import org.dubhe.dao.PtModelBranchMapper; -import org.dubhe.dao.PtModelInfoMapper; -import org.dubhe.dao.PtTrainAlgorithmMapper; -import org.dubhe.dao.PtTrainJobMapper; -import org.dubhe.dao.PtTrainJobSpecsMapper; -import org.dubhe.dao.PtTrainMapper; -import org.dubhe.dao.PtTrainParamMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.PtModelBranch; -import org.dubhe.domain.PtModelInfo; -import org.dubhe.domain.dto.BaseTrainJobDTO; -import org.dubhe.domain.dto.PtTrainDataSourceStatusQueryDTO; -import org.dubhe.domain.dto.PtTrainJobCreateDTO; -import org.dubhe.domain.dto.PtTrainJobDeleteDTO; -import org.dubhe.domain.dto.PtTrainJobDetailQueryDTO; -import org.dubhe.domain.dto.PtTrainJobResumeDTO; -import org.dubhe.domain.dto.PtTrainJobStopDTO; -import org.dubhe.domain.dto.PtTrainJobUpdateDTO; -import org.dubhe.domain.dto.PtTrainJobVersionQueryDTO; -import org.dubhe.domain.dto.PtTrainModelDTO; -import org.dubhe.domain.dto.PtTrainQueryDTO; -import org.dubhe.domain.dto.RecycleTaskCreateDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.DictDetail; -import org.dubhe.domain.entity.PtJobParam; -import org.dubhe.domain.entity.PtTrain; -import org.dubhe.domain.entity.PtTrainAlgorithm; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.domain.entity.PtTrainJobSpecs; -import org.dubhe.domain.entity.PtTrainParam; -import org.dubhe.domain.vo.ModelVO; -import org.dubhe.domain.vo.PtImageAndAlgorithmVO; -import org.dubhe.domain.vo.PtJobMetricsGrafanaVO; -import org.dubhe.domain.vo.PtTrainDataSourceStatusQueryVO; -import org.dubhe.domain.vo.PtTrainJobDeleteVO; -import org.dubhe.domain.vo.PtTrainJobDetailQueryVO; -import org.dubhe.domain.vo.PtTrainJobDetailVO; -import org.dubhe.domain.vo.PtTrainJobModelVO; -import org.dubhe.domain.vo.PtTrainJobStatisticsMineVO; -import org.dubhe.domain.vo.PtTrainJobStopVO; -import org.dubhe.domain.vo.PtTrainVO; -import org.dubhe.enums.AlgorithmStatusEnum; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.ModelResourceEnum; -import org.dubhe.enums.RecycleModuleEnum; -import org.dubhe.enums.RecycleTypeEnum; -import org.dubhe.enums.TrainJobStatusEnum; -import org.dubhe.enums.TrainTypeEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.DistributeTrainApi; -import org.dubhe.k8s.api.PersistentVolumeClaimApi; -import org.dubhe.k8s.api.PodApi; -import org.dubhe.k8s.api.TrainJobApi; -import org.dubhe.k8s.domain.PtBaseResult; -import org.dubhe.k8s.domain.resource.BizPod; -import org.dubhe.k8s.utils.PodUtil; -import org.dubhe.service.PtTrainJobService; -import org.dubhe.service.RecycleTaskService; -import org.dubhe.utils.ImageUtil; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.KeyUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NfsUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.ReflectionUtils; -import org.dubhe.utils.SqlUtil; -import org.dubhe.utils.StringUtils; -import org.dubhe.utils.TrainUtil; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.lang.NonNull; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.io.File; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @description 训练作业job服务实现类 - * @date 2020-04-27 - */ -@Service -public class PtTrainJobServiceImpl implements PtTrainJobService { - - @Autowired - private PtTrainMapper ptTrainMapper; - - @Autowired - private PtTrainJobMapper ptTrainJobMapper; - - @Autowired - private PtJobParamMapper ptJobParamMapper; - - @Autowired - private PtTrainParamMapper ptTrainParamMapper; - - @Autowired - private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; - - @Autowired - private TrainJobApi trainJobApi; - - @Autowired - private PersistentVolumeClaimApi persistentVolumeClaimApi; - - @Autowired - private PtTrainJobSpecsMapper ptTrainJobSpecsMapper; - - @Autowired - private PodApi podApi; - - @Autowired - private TrainJobConfig trainJobConfig; - - @Autowired - private TrainHarborConfig trainHarborConfig; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private ImageUtil imageUtil; - - @Autowired - private TransactionAsyncManager asyncManager; - - @Autowired - private StopTrainJobAsync stopTrainJobAsync; - - @Autowired - private DistributeTrainApi distributeTrainApi; - - @Autowired - private DictDetailMapper dictDetailMapper; - - @Autowired - private NfsConfig nfsConfig; - - @Autowired - private NfsUtil nfsUtil; - - @Autowired - private RecycleConfig recycleConfig; - - @Autowired - private RecycleTaskService recycleTaskService; - - @Value("${k8s.pod.metrics.grafanaUrl}") - private String k8sPodMetricsGrafanaUrl; - - @Autowired - private PtModelInfoMapper ptModelInfoMapper; - - @Autowired - private PtModelBranchMapper ptModelBranchMapper; - - public final static List FIELD_NAMES; - - static { - FIELD_NAMES = ReflectionUtils.getFieldNames(PtTrainVO.class); - } - - /** - * 作业列表展示 - * - * @param ptTrainQueryDTO 查询作业列表参数 - * @return Map 作业列表分页信息 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Map getTrainJob(@NonNull PtTrainQueryDTO ptTrainQueryDTO) { - Page pageTrainResult; - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query job list display, received parameters are {}", currentUser.getUsername(), ptTrainQueryDTO); - - Page page = ptTrainQueryDTO.toPage(); - String order; - String sort; - try { - //排序方式 - order = Constant.SORT_ASC.equalsIgnoreCase(ptTrainQueryDTO.getOrder()) ? Constant.SORT_ASC : Constant.SORT_DESC; - //排序字段 - String sortField = FIELD_NAMES.contains(ptTrainQueryDTO.getSort()) ? ptTrainQueryDTO.getSort() : Constant.ID; - sort = StringUtils.humpToLine(sortField); - pageTrainResult = ptTrainJobMapper.getPageTrain(page, currentUser.getId(), ptTrainQueryDTO.getTrainStatus(), ptTrainQueryDTO.getTrainName(), sort, order); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Query job list shows exception {}, receive parameter {}", e, ptTrainQueryDTO); - throw new BusinessException("查询作业列表展示异常"); - } - List trainResult = pageTrainResult.getRecords(); - if (CollectionUtils.isNotEmpty(trainResult)) { - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} query job list is displayed and the result is as follows {}.", currentUser.getUsername(), trainResult); - } - return PageUtil.toPage(page, trainResult); - - } - - /** - * 作业不同版本job列表展示 - * - * @param ptTrainJobVersionQueryDTO 查询作业不同版本job列表参数 - * @return List 训练详情集合 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List getTrainJobVersion(PtTrainJobVersionQueryDTO ptTrainJobVersionQueryDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, received parameter trainId is {}", currentUser.getUsername(), ptTrainJobVersionQueryDTO.getTrainId()); - - //校验参数 - checkTrainId(ptTrainJobVersionQueryDTO.getTrainId(), currentUser); - String sort = null == ptTrainJobVersionQueryDTO.getSort() ? Constant.ID : ptTrainJobVersionQueryDTO.getSort(); - - QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); - queryTrainJonWrapper.eq("train_id", ptTrainJobVersionQueryDTO.getTrainId()); - //根据训练状态筛选 - if (ptTrainJobVersionQueryDTO.getTrainStatus() != null) { - queryTrainJonWrapper.eq("train_status", ptTrainJobVersionQueryDTO.getTrainStatus()); - } - if (Constant.SORT_ASC.equals(ptTrainJobVersionQueryDTO.getOrder())) { - queryTrainJonWrapper.orderByAsc(StringUtils.humpToLine(sort)); - } else { - queryTrainJonWrapper.orderByDesc(StringUtils.humpToLine(sort)); - } - //按照trainId查找 - List ptTrainJobs = ptTrainJobMapper.selectList(queryTrainJonWrapper); - if (CollectionUtils.isEmpty(ptTrainJobs)) { - LogUtil.info(LogEnum.BIZ_TRAIN, "No training task with trainId of {} and training status of {} was found by user {}", currentUser.getUsername(), ptTrainJobVersionQueryDTO.getTrainId(), ptTrainJobVersionQueryDTO.getTrainStatus()); - return Collections.emptyList(); - } - Set jobIds = ptTrainJobs.stream().map(PtTrainJob::getId).collect(Collectors.toSet()); - QueryWrapper queryJobParamWrapper = new QueryWrapper<>(); - queryJobParamWrapper.in("train_job_id", jobIds); - //找出所有训练参数 - List ptJobParams = ptJobParamMapper.selectList(queryJobParamWrapper); - List algorithmIds = ptJobParams.stream().map(PtJobParam::getAlgorithmId).distinct().collect(Collectors.toList()); - List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectAllBatchIds(algorithmIds); - //获取训练信息 - - PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobVersionQueryDTO.getTrainId()); - //结果集处理 - List list = getTrainJobDetail(ptTrainJobs, ptJobParams, ptTrainAlgorithms, ptTrain); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query different version of job list display completed, return result {}", currentUser.getUsername(), list); - return list; - } - - /** - * 结果集处理 拼装PtTrainJobDetailVO - * - * @param ptTrainJobs 训练任务集合 - * @param ptJobParams 训练参数集合 - * @param ptTrainAlgorithms 训练算法集合 - * @param ptTrain 训练 - * @return List 训练版本查询详情集合 - */ - private List getTrainJobDetail(List ptTrainJobs, List ptJobParams, List ptTrainAlgorithms, PtTrain ptTrain) { - List list = new ArrayList<>(); - Map jobParamMap = new HashedMap<>(); - ptTrainJobs.forEach(x -> { - PtTrainJobDetailVO ptTrainJobDetailVO = new PtTrainJobDetailVO(); - BeanUtil.copyProperties(x, ptTrainJobDetailVO); - list.add(ptTrainJobDetailVO); - jobParamMap.put(x.getId(), list.size()); - }); - - ptJobParams.forEach(x -> { - PtTrainJobDetailVO ptTrainJobDetailVO = list.get(jobParamMap.get(x.getTrainJobId()) - 1); - if (null != ptTrainJobDetailVO) { - - ptTrainJobDetailVO.setAlgorithmId(x.getAlgorithmId()).setRunCommand(x.getRunCommand()).setImageName(x.getImageName()) - .setRunParams(x.getRunParams()) - .setParamF1(x.getParamF1()).setParamCallback(x.getParamCallback()) - .setParamPrecise(x.getParamPrecise()).setParamAccuracy(x.getParamAccuracy()); - long nowTime = System.currentTimeMillis(); - //获取训练延时启动倒计时（分钟） - if (x.getDelayCreateTime() != null && nowTime < x.getDelayCreateTime().getTime() && TrainJobStatusEnum.checkRunStatus(ptTrainJobDetailVO.getTrainStatus())) { - ptTrainJobDetailVO.setDelayCreateCountDown(TrainUtil.getCountDown(x.getDelayCreateTime().getTime())); - } - //获取训练自动停止倒计时（分钟） - if (x.getDelayDeleteTime() != null && nowTime < x.getDelayDeleteTime().getTime() && TrainJobStatusEnum.checkRunStatus(ptTrainJobDetailVO.getTrainStatus())) { - ptTrainJobDetailVO.setDelayDeleteCountDown(TrainUtil.getCountDown(x.getDelayDeleteTime().getTime())); - } - //image信息拼装 - if (StringUtils.isNotBlank(x.getImageName())) { - String imageNameSuffix = x.getImageName().substring(x.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); - String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); - ptTrainJobDetailVO.setImageName(imageNameSuffixArray[0]); - ptTrainJobDetailVO.setImageTag(imageNameSuffixArray[1]); - } - } - }); - - Map algorithmMap = new HashedMap<>(); - ptTrainAlgorithms.forEach(x -> algorithmMap.put(x.getId(), x)); - - for (PtTrainJobDetailVO ptTrainJobDetailVO : list) { - PtTrainAlgorithm ptTrainAlgorithm = algorithmMap.get(ptTrainJobDetailVO.getAlgorithmId()); - if (null != ptTrainAlgorithm) { - ptTrainJobDetailVO.setAlgorithmName(ptTrainAlgorithm.getAlgorithmName()) - .setAlgorithmSource(ptTrainAlgorithm.getAlgorithmSource()) - .setAlgorithmUsage(ptTrainAlgorithm.getAlgorithmUsage()) - .setAccuracy(ptTrainAlgorithm.getAccuracy()) - .setP4InferenceSpeed(ptTrainAlgorithm.getP4InferenceSpeed()); - if (ptTrainAlgorithm.getAlgorithmSource() == MagicNumConstant.ONE) { - ptTrainJobDetailVO.setAlgorithmCodeDir(ptTrainAlgorithm.getCodeDir()); - } - } - } - - list.forEach(x -> x.setTrainName(ptTrain.getTrainName())); - return list; - } - - /** - * 校验请求不同版本job所传参数是否合法 - * - * @param trainId 训练ID - * @param currentUser 当前用户 - */ - private void checkTrainId(Long trainId, UserDTO currentUser) { - if (null == trainId || trainId < 1) { - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, request parameter trainId is {}, it is illegal", currentUser.getUsername(), trainId); - throw new BusinessException("参数不合法"); - } - PtTrain ptTrain = ptTrainMapper.selectById(trainId); - if (null == ptTrain || !currentUser.getId().equals(ptTrain.getCreateUserId())) { - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} queries different versions of job list display, request parameter trainId is {}, it is illegal", currentUser.getUsername(), trainId); - throw new BusinessException("参数不合法"); - } - } - - /** - * 创建训练job - * - * @param ptTrainJobCreateDTO 创建训练job参数 - * @return List id集合 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List createTrainJobVersion(PtTrainJobCreateDTO ptTrainJobCreateDTO) { - - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} creates a training job and receives {} as an argument", currentUser.getUsername(), ptTrainJobCreateDTO); - - // 判断当前trainName是否已经存在 - checkTrainName(ptTrainJobCreateDTO.getTrainName(), currentUser.getId()); - // 校验trainParamName是否存在 - if (ptTrainJobCreateDTO.getSaveParams() != null && ptTrainJobCreateDTO.getSaveParams()) { - checkTrainParamName(ptTrainJobCreateDTO, currentUser.getId()); - // 保存任务参数到数据库 - saveParamToDb(ptTrainJobCreateDTO, currentUser); - } - // 获取镜像和算法目录 - PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId(), - currentUser.getId()); - //使用用户创建训练时提供的镜像与运行命令 - String images = imageUtil.getImageUrl(ptTrainJobCreateDTO, currentUser); - ptImageAndAlgorithmVO.setImageName(trainHarborConfig.getAddress() + StrUtil.SLASH + images).setRunCommand(ptTrainJobCreateDTO.getRunCommand()); - - //jobKey - String trainKey = KeyUtil.generateTrainKey(currentUser.getId()); - - //获取规格 - PtTrainJobSpecs ptTrainJobSpecs = new PtTrainJobSpecs(); - - ptTrainJobSpecs.setResourcesPoolType(ptTrainJobCreateDTO.getResourcesPoolType()); - ptTrainJobSpecs.setSpecsName(ptTrainJobCreateDTO.getTrainJobSpecsName()); - ptTrainJobSpecs.setSpecsInfo(JSONObject.parseObject(ptTrainJobCreateDTO.getTrainJobSpecsInfo())); - - //版本 - String version = trainJobConfig.getVersionLabel() + String.format(TrainUtil.FOUR_DECIMAL, 1); - //生成k8s 的job名称 - String jobName = trainKey + trainJobConfig.getSeparator() + version; - - BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); - BeanUtil.copyProperties(ptTrainJobCreateDTO, baseTrainJobDTO); - baseTrainJobDTO.setJobName(jobName); - - baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); - - //结果集处理 - PtTrainJob ptTrainJob = saveTrainJobTableData(ptTrainJobCreateDTO, currentUser, images, trainKey, baseTrainJobDTO); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} creates training job, returns result {}", currentUser.getUsername(), ptTrainJob.getTrainId()); - - // 提交job - asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); - return Collections.singletonList(ptTrainJob.getTrainId()); - } - - /** - * 保存训练任务数据 - * - * @param ptTrainJobCreateDTO 创建训练任务DTO - * @param currentUser 用户 - * @param imageName 镜像名称 - * @param trainKey 训练key - * @param baseTrainJobDTO 基础训练参数 - * @return PtTrain 训练 - */ - private PtTrainJob saveTrainJobTableData(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser, - String imageName, String trainKey, BaseTrainJobDTO baseTrainJobDTO) { - // 添加train表 - PtTrain ptTrain = new PtTrain(); - ptTrain.setTrainName(ptTrainJobCreateDTO.getTrainName()) - .setTrainKey(trainKey) - .setCreateUserId(currentUser.getId()); - int trainResult = ptTrainMapper.insert(ptTrain); - if (trainResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} creates training job, pt Train table insert data failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - //检查模型是否合法,合法则保存其路径地址 - checkModelAndSavePath(currentUser, baseTrainJobDTO); - - // 添加train_job表 - PtTrainJob ptTrainJob = new PtTrainJob(); - BeanUtil.copyProperties(ptTrainJobCreateDTO, ptTrainJob); - ptTrainJob.setTrainId(ptTrain.getId()) - .setTrainVersion(trainJobConfig.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, TrainUtil.NUMBER_ONE)) - .setJobName(baseTrainJobDTO.getJobName()) - .setCreateUserId(currentUser.getId()); - int jobResult = ptTrainJobMapper.insert(ptTrainJob); - if (jobResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training Job, failed to insert data in train_job table", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - // 保存job参数 - PtJobParam ptJobParam = new PtJobParam(); - ptJobParam.setTrainJobId(ptTrainJob.getId()) - .setAlgorithmId(ptTrainJobCreateDTO.getAlgorithmId()) - .setRunCommand(ptTrainJobCreateDTO.getRunCommand()) - .setImageName(imageName) - .setRunParams(ptTrainJobCreateDTO.getRunParams()) - .setCreateUserId(currentUser.getId()); - //保存训练延时启动时间 - if (ptTrainJobCreateDTO.getDelayCreateTime() != null && ptTrainJobCreateDTO.getDelayCreateTime() > 0) { - ptJobParam.setDelayCreateTime(TrainUtil.getDelayTime(ptTrainJobCreateDTO.getDelayCreateTime())); - } - //保存训练自动停止时间 - if (ptTrainJobCreateDTO.getDelayDeleteTime() != null && ptTrainJobCreateDTO.getDelayDeleteTime() > 0) { - if (ptTrainJobCreateDTO.getDelayCreateTime() != null && ptTrainJobCreateDTO.getDelayCreateTime() > 0) { - ptJobParam.setDelayDeleteTime(TrainUtil.getDelayTime(ptTrainJobCreateDTO.getDelayCreateTime() + ptTrainJobCreateDTO.getDelayDeleteTime())); - } else { - ptJobParam.setDelayDeleteTime(TrainUtil.getDelayTime(ptTrainJobCreateDTO.getDelayDeleteTime())); - } - } - int jobParamResult = ptJobParamMapper.insert(ptJobParam); - if (jobParamResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_job_parAM table insert data failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - return ptTrainJob; - } - - /** - * 检查模型是否合法,合法则保存其路径地址 - * - * @param currentUser 用户 - * @param baseTrainJobDTO 基础训练参数 - */ - private void checkModelAndSavePath(UserDTO currentUser, BaseTrainJobDTO baseTrainJobDTO) { - - Integer modelResource = baseTrainJobDTO.getModelResource(); - if(null == modelResource ) { - if(null == baseTrainJobDTO.getModelId() && - StringUtils.isBlank(baseTrainJobDTO.getStudentModelIds()) && - StringUtils.isBlank(baseTrainJobDTO.getStudentModelIds())) { - return; - } else { - logErrorInfoOnModel(currentUser.getUsername()); - } - } - switch (ModelResourceEnum.getType(modelResource)) { - case MINE: - if(null == baseTrainJobDTO.getModelBranchId() || null == baseTrainJobDTO.getModelId() || - StringUtils.isNotBlank(baseTrainJobDTO.getTeacherModelIds()) || - StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(baseTrainJobDTO.getModelBranchId()); - if(null == ptModelBranch || ptModelBranch.getParentId().compareTo(baseTrainJobDTO.getModelId()) != 0 || - StringUtils.isBlank(ptModelBranch.getModelAddress())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); - if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(baseTrainJobDTO.getModelResource()) != 0){ - logErrorInfoOnModel(currentUser.getUsername()); - } - baseTrainJobDTO.setModelPath(adjustmentUrl(ptModelBranch.getModelAddress())); - break; - case PRESET: - if(null == baseTrainJobDTO.getModelId() || StringUtils.isNotBlank(baseTrainJobDTO.getTeacherModelIds()) || - StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(baseTrainJobDTO.getModelId()); - if(null == ptModelInfoPreset || StringUtils.isBlank(ptModelInfoPreset.getUrl()) || - ptModelInfoPreset.getModelResource().compareTo(baseTrainJobDTO.getModelResource()) != 0) { - logErrorInfoOnModel(currentUser.getUsername()); - } - baseTrainJobDTO.setModelPath(adjustmentUrl(ptModelInfoPreset.getUrl())); - break; - case ATLAS: - if(StringUtils.isBlank(baseTrainJobDTO.getTeacherModelIds()) || null != baseTrainJobDTO.getModelId() ){ - logErrorInfoOnModel(currentUser.getUsername()); - } - Set ids = new HashSet<>(); - Set teacherModelList = new HashSet<>(); - Arrays.stream(baseTrainJobDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> teacherModelList.add(Long.parseLong(id))); - ids.addAll(teacherModelList); - - Set studentModelList = new HashSet<>(); - if(StringUtils.isNotBlank(baseTrainJobDTO.getStudentModelIds())) { - Arrays.stream(baseTrainJobDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> studentModelList.add(Long.parseLong(id))); - ids.addAll(studentModelList); - } - if(ids.isEmpty()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - LambdaQueryWrapper query = new LambdaQueryWrapper<>(); - query.eq(PtModelInfo::getModelResource, baseTrainJobDTO.getModelResource()) - .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); - List modelInfoList = ptModelInfoMapper.selectList(query); - if(null == modelInfoList || modelInfoList.size() < ids.size()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - - //保存炼知教师模型路径地址 - baseTrainJobDTO.setTeacherModelPathList(new ArrayList<>()); - modelInfoList.stream() - .filter(modelInfo -> teacherModelList.contains(modelInfo.getId())) - .forEach(modelInfo -> baseTrainJobDTO.getTeacherModelPathList().add(adjustmentUrl(modelInfo.getUrl()))); - - //保存炼知学生模型路径地址 - if(!studentModelList.isEmpty()) { - baseTrainJobDTO.setStudentModelPathList(new ArrayList<>()); - modelInfoList.stream() - .filter(modelInfo -> studentModelList.contains(modelInfo.getId())) - .forEach(modelInfo -> baseTrainJobDTO.getStudentModelPathList().add(adjustmentUrl(modelInfo.getUrl()))); - } - break; - } - } - - - - /** - * 调整模型地址 - * - * @param modelUrl 模型地址 - * @return - */ - private String adjustmentUrl(String modelUrl) { - if(modelUrl.endsWith(SymbolConstant.SLASH)) { - modelUrl = modelUrl.substring(TrainUtil.NUMBER_ZERO, modelUrl.length() - TrainUtil.NUMBER_ONE); - } - return modelUrl; - } - - /** - * 打印训练任务中模型相关的错误日志 - * - * @param username - */ - private void logErrorInfoOnModel(String username){ - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} operating training job, error on model......", username); - throw new BusinessException("模型参数参数不合法"); - } - - /** - * 保存任务参数到数据库 - * - * @param ptTrainJobCreateDTO 创建训练任务DTO - * @param currentUser 用户 - */ - private void saveParamToDb(PtTrainJobCreateDTO ptTrainJobCreateDTO, UserDTO currentUser) { - PtTrainParam ptTrainParam = new PtTrainParam(); - BeanUtil.copyProperties(ptTrainJobCreateDTO, ptTrainParam); - //获取镜像url - String images = imageUtil.getImageUrl(ptTrainJobCreateDTO, currentUser); - ptTrainParam.setImageName(images); - ptTrainParam.setParamName(ptTrainJobCreateDTO.getTrainParamName()) - .setDescription(ptTrainJobCreateDTO.getTrainParamDesc()) - .setRunParams(ptTrainJobCreateDTO.getRunParams()) - .setCreateUserId(currentUser.getId()); - int trainParamResult = ptTrainParamMapper.insert(ptTrainParam); - if (trainParamResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_param_param table failed to insert data", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - } - - /** - * 获取训练作业规格 - * - * @param trainJobSpecsId 规格类型Id - * @param currentUser 用户 - * @return PtTrainJobSpecs 训练作业规格 - */ - private PtTrainJobSpecs getSpecs(Integer trainJobSpecsId, UserDTO currentUser) { - - PtTrainJobSpecs ptTrainJobSpecs = ptTrainJobSpecsMapper.selectById(trainJobSpecsId); - if (ptTrainJobSpecs == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The PT_train_job_specs table specification type Id for the user {} query does not exist", currentUser.getUsername(), trainJobSpecsId); - throw new BusinessException("内部错误"); - } - return ptTrainJobSpecs; - } - - /** - * 获取镜像和算法目录 - * - * @param algorithmId 算法ID - * @param userId 用户ID - * @return PtImageAndAlgorithmVO 镜像 - */ - private PtImageAndAlgorithmVO getPtImageByAlgorithmId(Long algorithmId, Long userId) { - - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(algorithmId); - if (null == ptTrainAlgorithm || StringUtils.isBlank(ptTrainAlgorithm.getCodeDir())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The record with algorithm training ID {} has no corresponding image or algorithm directory configuration", algorithmId); - throw new BusinessException(ResponseCode.ERROR, "该id的记录没有相应的镜像或者算法目录配置"); - } - - if (!AlgorithmStatusEnum.SUCCESS.getCode().equals(ptTrainAlgorithm.getAlgorithmStatus())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm ID {} algorithmStatus is{} unusual", algorithmId, ptTrainAlgorithm.getAlgorithmStatus()); - throw new BusinessException(ResponseCode.ERROR, "该算法状态异常!"); - } - - if (!(userId.equals(ptTrainAlgorithm.getCreateUserId()) || AlgorithmSourceEnum.PRE.getStatus().equals(ptTrainAlgorithm.getAlgorithmSource()))) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The data {} does not belong to the user {}!", ptTrainAlgorithm, userId); - throw new BusinessException(ResponseCode.ERROR, "该数据不属于该用户!"); - } - - PtImageAndAlgorithmVO ptImageAndAlgorithmVO = new PtImageAndAlgorithmVO(); - BeanUtil.copyProperties(ptTrainAlgorithm, ptImageAndAlgorithmVO); - - return ptImageAndAlgorithmVO; - } - - /** - * 校验trainName - * - * @param trainName 校验trainName - * @param userId 返回用户ID - */ - private void checkTrainName(String trainName, Long userId) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("train_name", trainName) - .eq("create_user_id", userId); - Integer ptTrainCountResult = ptTrainMapper.selectCount(queryWrapper); - if (ptTrainCountResult > 0) { - throw new BusinessException("当前训练名称已经存在"); - } - } - - /** - * 校验trainParamName - * - * @param ptTrainJobCreateDTO 创建训练DTO - * @param userId 用户ID - */ - private void checkTrainParamName(PtTrainJobCreateDTO ptTrainJobCreateDTO, Long userId) { - - String trainParamName = ptTrainJobCreateDTO.getTrainParamName(); - - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("param_name", trainParamName) - .eq("create_user_id", userId); - Integer ptTrainParamCountResult = ptTrainParamMapper.selectCount(queryWrapper); - if (ptTrainParamCountResult > 0) { - throw new BusinessException("当前trainParamName已经存在"); - } - } - - /** - * 生成k8s 的job名称 - * - * @param ptTrain 训练 - * @return String 版本 - */ - private String buildVersion(PtTrain ptTrain) { - return ptTrain.getTrainKey() + trainJobConfig.getSeparator() + trainJobConfig.getVersionLabel() + String.format(TrainUtil.FOUR_DECIMAL, ptTrain.getTotalNum() + 1); - } - - /** - * 修改训练job - * - * @param ptTrainJobUpdateDTO 修改训练job参数 - * @return List id集合 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List updateTrainJob(PtTrainJobUpdateDTO ptTrainJobUpdateDTO) { - - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the training job and receives the parameter {}", ptTrainJobUpdateDTO); - - PtTrainJob existPtTrainJob = ptTrainJobMapper.selectById(ptTrainJobUpdateDTO.getId()); - if (null == existPtTrainJob || !currentUser.getId().equals(existPtTrainJob.getCreateUserId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for a user {} to modify a training job, jobId, to {}", currentUser.getUsername(), ptTrainJobUpdateDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除"); - } - //获取算法 - PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId(), currentUser.getId()); - //使用用户修改训练时提供的镜像与运行命令 - //获取镜像url - String images = imageUtil.getImageUrl(ptTrainJobUpdateDTO, currentUser); - ptImageAndAlgorithmVO.setImageName(trainHarborConfig.getAddress() + StrUtil.SLASH + images).setRunCommand(ptTrainJobUpdateDTO.getRunCommand()); - - //获取规格 - PtTrainJobSpecs ptTrainJobSpecs = new PtTrainJobSpecs(); - ptTrainJobSpecs.setResourcesPoolType(ptTrainJobUpdateDTO.getResourcesPoolType()); - ptTrainJobSpecs.setSpecsName(ptTrainJobUpdateDTO.getTrainJobSpecsName()); - ptTrainJobSpecs.setSpecsInfo(JSONObject.parseObject(ptTrainJobUpdateDTO.getTrainJobSpecsInfo())); - - PtTrain ptTrain = ptTrainMapper.selectById(existPtTrainJob.getTrainId()); - - String jobName = buildVersion(ptTrain); - - BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); - BeanUtil.copyProperties(ptTrainJobUpdateDTO, baseTrainJobDTO); - baseTrainJobDTO.setJobName(jobName); - baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); - //结果集处理 - PtTrainJob ptTrainJob = updateTrainJobTableData(ptTrainJobUpdateDTO, currentUser, existPtTrainJob, images, ptTrain, baseTrainJobDTO); - //提交job - asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); - - return Collections.singletonList(ptTrainJob.getId()); - } - - /** - * 结果集处理 - * - * @param ptTrainJobUpdateDTO 更新训练任务DTO - * @param currentUser 当前用户 - * @param existPtTrainJob 存在的训练任务 - * @param imageName 镜像名称 - * @param ptTrain 训练 - * @param baseTrainJobDTO 基本训练信息 - * @return PtTrainJob 训练任务 - */ - private PtTrainJob updateTrainJobTableData(PtTrainJobUpdateDTO ptTrainJobUpdateDTO, UserDTO - currentUser, PtTrainJob existPtTrainJob, String imageName, PtTrain ptTrain, BaseTrainJobDTO baseTrainJobDTO) { - - //检查模型是否合法,合法则保存其路径地址 - checkModelAndSavePath(currentUser, baseTrainJobDTO); - - //添加train_job表 - PtTrainJob ptTrainJob = new PtTrainJob(); - BeanUtil.copyProperties(ptTrainJobUpdateDTO, ptTrainJob); - ptTrainJob.setTrainId(ptTrain.getId()).setTrainVersion(trainJobConfig.getVersionLabel().toUpperCase() + String.format(TrainUtil.FOUR_DECIMAL, ptTrain.getTotalNum() + 1)) - .setJobName(baseTrainJobDTO.getJobName()).setParentTrainVersion(existPtTrainJob.getTrainVersion()) - .setCreateUserId(currentUser.getId()); - int jobResult = ptTrainJobMapper.insert(ptTrainJob); - if (jobResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training Job, failed to insert data in train_job table", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - //保存job参数 - PtJobParam ptJobParam = new PtJobParam(); - ptJobParam.setTrainJobId(ptTrainJob.getId()) - .setAlgorithmId(ptTrainJobUpdateDTO.getAlgorithmId()) - .setRunCommand(ptTrainJobUpdateDTO.getRunCommand()) - .setImageName(imageName) - .setRunParams(ptTrainJobUpdateDTO.getRunParams()) - .setCreateUserId(currentUser.getId()); - //保存训练延时启动时间 - if (ptTrainJobUpdateDTO.getDelayCreateTime() != null && ptTrainJobUpdateDTO.getDelayCreateTime() > 0) { - ptJobParam.setDelayCreateTime(TrainUtil.getDelayTime(ptTrainJobUpdateDTO.getDelayCreateTime())); - } - //保存训练自动停止时间 - if (ptTrainJobUpdateDTO.getDelayDeleteTime() != null && ptTrainJobUpdateDTO.getDelayDeleteTime() > 0) { - if (ptTrainJobUpdateDTO.getDelayCreateTime() != null && ptTrainJobUpdateDTO.getDelayCreateTime() > 0) { - ptJobParam.setDelayDeleteTime(TrainUtil.getDelayTime(ptTrainJobUpdateDTO.getDelayCreateTime() + ptTrainJobUpdateDTO.getDelayDeleteTime())); - } else { - ptJobParam.setDelayDeleteTime(TrainUtil.getDelayTime(ptTrainJobUpdateDTO.getDelayDeleteTime())); - } - } - int jobParamResult = ptJobParamMapper.insert(ptJobParam); - if (jobParamResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_job_parAM table insert data failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - //更新pt_train - PtTrain updatePtTrain = new PtTrain(); - updatePtTrain.setId(ptTrain.getId()).setVersionNum(ptTrain.getVersionNum() + 1) - .setTotalNum(ptTrain.getTotalNum() + 1).setUpdateUserId(currentUser.getId()); - int updateResult = ptTrainMapper.updateById(updatePtTrain); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} created training job, pT_train table failed to update version number", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - return ptTrainJob; - } - - /** - * 删除训练job - * - * @param ptTrainJobDeleteDTO 删除训练job参数 - * @return PtTrainJobDeleteVO 删除训练任务结果 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public PtTrainJobDeleteVO deleteTrainJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} deletes the training job and receives the parameter {}", currentUser.getUsername(), ptTrainJobDeleteDTO); - - List jobList = new ArrayList<>(); - PtTrain ptTrain = checkAndReturnPtTrain(ptTrainJobDeleteDTO, currentUser, jobList); - - Collection jobIdList = new ArrayList<>(); - if (null != ptTrainJobDeleteDTO.getId()) { - - //要删除的训练任务 - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobDeleteDTO.getId()); - if (ptTrainJob == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_train_job table failed to delete data", currentUser.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); - } - //删除job - deleteJobs(currentUser, jobList); - ptTrainJobMapper.deleteById(ptTrainJobDeleteDTO.getId()); - - PtTrain updatePtTrain = new PtTrain(); - updatePtTrain.setVersionNum(ptTrain.getVersionNum() - 1); - UpdateWrapper updateWrapper = new UpdateWrapper<>(); - updateWrapper.eq("id", ptTrain.getId()).eq("version_num", ptTrain.getVersionNum()); - int updateResult = ptTrainMapper.update(updatePtTrain, updateWrapper); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted the training job and updated the version_num in the PT_train table failed", currentUser.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); - } - - if (ptTrain.getVersionNum() == 1) { - int trainResult = ptTrainMapper.deleteById(ptTrain.getId()); - if (trainResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pt Train table deleted data failed", currentUser.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); - } - } - jobIdList.add(ptTrainJobDeleteDTO.getId()); - - //回收已删除训练任务无效文件 - String recyclePath = nfsUtil.formatPath(nfsConfig.getRootDir() + nfsConfig.getBucket() + trainJobConfig.getManage() + - File.separator + ptTrainJob.getCreateUserId() + File.separator + ptTrainJob.getJobName()); - recycleTaskWithTrain(recyclePath); - - } else { - deleteTrainAndJob(ptTrainJobDeleteDTO, currentUser, jobList, ptTrain, jobIdList); - - } - - //删除pt_job_param表中相关数据 - UpdateWrapper updateJobParamWrapper = new UpdateWrapper<>(); - updateJobParamWrapper.in("train_job_id", jobIdList); - int jobParamResult = ptJobParamMapper.delete(updateJobParamWrapper); - if (jobParamResult < jobIdList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_job_param table failed to delete data", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - PtTrainJobDeleteVO ptTrainJobDeleteVO = new PtTrainJobDeleteVO(); - BeanUtil.copyProperties(ptTrainJobDeleteDTO, ptTrainJobDeleteVO); - LogUtil.info(LogEnum.BIZ_TRAIN, "Deleting the training job completes and the result returned is {}", ptTrainJobDeleteVO); - return ptTrainJobDeleteVO; - } - - /** - * 删除训练和任务 - * - * @param ptTrainJobDeleteDTO 删除训练任务DTO - * @param currentUser 用户 - * @param jobList 任务集合 - * @param ptTrain 训练 - * @param jobIdList 任务ID集合 - */ - private void deleteTrainAndJob(PtTrainJobDeleteDTO ptTrainJobDeleteDTO, UserDTO - currentUser, List jobList, PtTrain ptTrain, Collection jobIdList) { - QueryWrapper query = new QueryWrapper<>(); - query.eq("train_id", ptTrainJobDeleteDTO.getTrainId()); - List ptTrainJobs = ptTrainJobMapper.selectList(query); - if (ptTrainJobs.size() < 1) { - throw new BusinessException(ResponseCode.SUCCESS, "没有待删除的训练任务"); - } - ptTrainJobs.forEach(x -> { - jobList.add(x); - jobIdList.add(x.getId()); - }); - - //删除job - deleteJobs(currentUser, jobList); - - PtTrain updatePtTrain = new PtTrain(); - updatePtTrain.setVersionNum(0); - UpdateWrapper updateTrainWrapper = new UpdateWrapper<>(); - updateTrainWrapper.eq("id", ptTrain.getId()).eq("version_num", ptTrain.getVersionNum()); - int updateResult = ptTrainMapper.update(updatePtTrain, updateTrainWrapper); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted the training job and updated the version_num in the PT_train table failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - int trainResult = ptTrainMapper.deleteById(ptTrain.getId()); - if (trainResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pt Train table deleted data failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - UpdateWrapper updateJobWrapper = new UpdateWrapper<>(); - updateJobWrapper.eq("train_id", ptTrain.getId()); - int jobResult = ptTrainJobMapper.delete(updateJobWrapper); - if (jobResult < jobIdList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, pT_train_job table failed to delete data", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - - //回收已删除训练任务无效文件 - for (PtTrainJob trainJob : ptTrainJobs) { - String recyclePath = nfsUtil.formatPath(nfsConfig.getRootDir() + nfsConfig.getBucket() + trainJobConfig.getManage() + - StrUtil.SLASH + trainJob.getCreateUserId() + StrUtil.SLASH + trainJob.getJobName()); - recycleTaskWithTrain(recyclePath); - } - } - - /** - * 校验并返回PtTrain - * - * @param ptTrainJobDeleteDTO 删除训练任务DTO - * @param currentUser 用户 - * @param jobList 任务集合 - * @return PtTrain 训练 - */ - private PtTrain checkAndReturnPtTrain(PtTrainJobDeleteDTO ptTrainJobDeleteDTO, UserDTO currentUser, List jobList) { - PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobDeleteDTO.getTrainId()); - if (null == ptTrain || !ptTrain.getCreateUserId().equals(currentUser.getId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete training job, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobDeleteDTO); - throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); - } - - if (null != ptTrainJobDeleteDTO.getId()) { - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobDeleteDTO.getId()); - if (null == ptTrainJob || !ptTrainJob.getTrainId().equals(ptTrainJobDeleteDTO.getTrainId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete training job, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobDeleteDTO); - throw new BusinessException(ResponseCode.SUCCESS, "训练任务已删除或参数不合法"); - } - jobList.add(ptTrainJob); - } - return ptTrain; - } - - /** - * 检测停止训练任务 - * - * @param ptTrainJobStopDTO 停止训练DTO - * @param currentUser 用户 - * @param jobList 任务集合 - */ - private void checkAndReturnPtTrain(PtTrainJobStopDTO ptTrainJobStopDTO, UserDTO currentUser, List jobList) { - PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJobStopDTO.getTrainId()); - if (null == ptTrain || !ptTrain.getCreateUserId().equals(currentUser.getId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stopped the training job failed, the parameter is illegal, the training does not exist, as follows {}", currentUser.getUsername(), ptTrainJobStopDTO); - throw new BusinessException(ResponseCode.SUCCESS, "参数不合法，该训练不存在"); - } - - if (null != ptTrainJobStopDTO.getId()) { - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobStopDTO.getId()); - if (null == ptTrainJob || !ptTrainJob.getTrainId().equals(ptTrainJobStopDTO.getTrainId()) || - TrainJobStatusEnum.checkStopStatus(ptTrainJob.getTrainStatus())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} stopped training job failed, invalid parameter, as follows {}", currentUser.getUsername(), ptTrainJobStopDTO); - throw new BusinessException(ResponseCode.SUCCESS, "参数不合法，该训练不存在"); - } - jobList.add(ptTrainJob); - } - } - - /** - * 删除任务 - * - * @param currentUser 用户 - * @param jobList 任务集合 - */ - private void deleteJobs(UserDTO currentUser, List jobList) { - String namespace = k8sNameTool.generateNamespace(currentUser.getId()); - try { - for (PtTrainJob job : jobList) { - if (TrainJobStatusEnum.STOP.getStatus().equals(job.getTrainStatus())) { - boolean bool = TrainTypeEnum.isDistributeTrain(job.getTrainType()) ? - distributeTrainApi.deleteByResourceName(namespace, job.getJobName()).isSuccess() : - trainJobApi.delete(namespace, job.getJobName()); - if (!bool) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deletes the training job and K8s fails to execute the delete() method, namespace为{}, resourceName为{}", - currentUser.getUsername(), namespace, job.getJobName()); - } - } - PtBaseResult ptBaseResult = persistentVolumeClaimApi.recycle(namespace, job.getJobName()); - if (null == ptBaseResult || !ptBaseResult.isSuccess()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} deleted training job, k8s failed to implement the recycle() method, namespace为{}, resourceName为{}", - currentUser.getUsername(), namespace, job.getJobName()); - } - } - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} delete training job, k8s delete failed,exception:{}", e); - throw new BusinessException("内部错误"); - } - } - - /** - * 停止训练job - * - * @param ptTrainJobStopDTO 停止训练job参数 - * @return PtTrainJobStopVO 停止训练任务结果 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public PtTrainJobStopVO stopTrainJob(PtTrainJobStopDTO ptTrainJobStopDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} stops training Job and receives the parameter {}", currentUser.getUsername(), ptTrainJobStopDTO); - //训练job名称集合 用于停止k8s - List jobList = new ArrayList<>(); - checkAndReturnPtTrain(ptTrainJobStopDTO, currentUser, jobList); - - if (null != ptTrainJobStopDTO.getId()) { - //停止job - stopTrainJobAsync.stopJobs(currentUser, jobList); - - } else if (null != ptTrainJobStopDTO.getTrainId()) { - QueryWrapper queryTrainJonWrapper = new QueryWrapper<>(); - queryTrainJonWrapper.eq("train_id", ptTrainJobStopDTO.getTrainId()); - List trainJobs = ptTrainJobMapper.selectList(queryTrainJonWrapper); - - for (PtTrainJob trainJob : trainJobs) { - if (!TrainJobStatusEnum.checkStopStatus(trainJob.getTrainStatus())) { - jobList.add(trainJob); - } - } - - if (jobList.size() < 1) { - throw new BusinessException(ResponseCode.SUCCESS, "没有待停止的job"); - } - - //停止job - stopTrainJobAsync.stopJobs(currentUser, jobList); - } - - PtTrainJobStopVO ptTrainJobStopVO = new PtTrainJobStopVO(); - ptTrainJobStopVO.setTrainId(ptTrainJobStopDTO.getTrainId()); - ptTrainJobStopVO.setId(ptTrainJobStopDTO.getId()); - return ptTrainJobStopVO; - } - - /** - * 任务统计 - * - * @return PtTrainJobStatisticsMineVO 我的训练任务统计结果 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public PtTrainJobStatisticsMineVO statisticsMine() { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - // 获取运行中的任务 - Integer runCount = ptTrainJobMapper.selectCountByStatus(userDTO.getId(), - SqlUtil.integerlistToString(new Integer[]{TrainJobStatusEnum.RUNNING.getStatus()})); - - // 已经完成的任务 - Integer finishCount = ptTrainJobMapper.selectCountByStatus(userDTO.getId(), - SqlUtil.integerlistToString( - new Integer[]{TrainJobStatusEnum.FAILED.getStatus(), TrainJobStatusEnum.STOP.getStatus(), - TrainJobStatusEnum.SUCCEEDED.getStatus(), TrainJobStatusEnum.UNKNOWN.getStatus()})); - - PtTrainJobStatisticsMineVO vo = new PtTrainJobStatisticsMineVO(); - vo.setRunJobCount(runCount); - vo.setFinishJobCount(finishCount); - return vo; - } - - /** - * 查询训练作业job状态 - * - * @param ptTrainDataSourceStatusQueryDTO 查询训练作业job状态参数 - * @return HashedMap 数据集路径-是否可以删除 的map集合 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Map getTrainDataSourceStatus(PtTrainDataSourceStatusQueryDTO ptTrainDataSourceStatusQueryDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} queries the state of the dataset starting with the received parameter {}", currentUser.getUsername(), ptTrainDataSourceStatusQueryDTO); - if (CollectionUtils.isEmpty(ptTrainDataSourceStatusQueryDTO.getDataSourcePath())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The dataset set {} is empty", ptTrainDataSourceStatusQueryDTO.getDataSourcePath()); - throw new BusinessException("传入参数为空，请重新输入"); - } - //去重 - List dataSourceList = ptTrainDataSourceStatusQueryDTO.getDataSourcePath().stream().distinct().collect(Collectors.toList()); - QueryWrapper query = new QueryWrapper<>(); - query.in("data_source_path", dataSourceList); - List ptTrainJobs = ptTrainJobMapper.selectList(query); - //结果集处理 - List ptTrainDataSourceStatusQueryList = ptTrainJobs.stream().map(x -> { - PtTrainDataSourceStatusQueryVO ptTrainDataSourceStatusQuery = new PtTrainDataSourceStatusQueryVO(); - ptTrainDataSourceStatusQuery.setDataSourcePath(x.getDataSourcePath()); - ptTrainDataSourceStatusQuery.setStatus(x.getTrainStatus() >= TrainJobStatusEnum.SUCCEEDED.getStatus()); - return ptTrainDataSourceStatusQuery; - }).distinct().collect(Collectors.toList()); - //结果去重 - HashMap map = new HashMap<>(); - for (PtTrainDataSourceStatusQueryVO ptTrainDataSourceStatusQuery : ptTrainDataSourceStatusQueryList) { - if (map.containsKey(ptTrainDataSourceStatusQuery.getDataSourcePath())) { - if (!ptTrainDataSourceStatusQuery.getStatus()) { - map.put(ptTrainDataSourceStatusQuery.getDataSourcePath(), ptTrainDataSourceStatusQuery.getStatus()); - } - } else { - map.put(ptTrainDataSourceStatusQuery.getDataSourcePath(), ptTrainDataSourceStatusQuery.getStatus()); - } - } - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query data set state ends, return result is {}", currentUser.getUsername(), map); - return map; - } - - /** - * 根据jobId查询训练任务详情查询 - * - * @param ptTrainJobDetailQueryDTO 根据jobId查询训练任务详情查询条件 - * @return PtTrainQueryJobDetailVO 根据jobId查询训练任务详情返回结果 - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public PtTrainJobDetailQueryVO getTrainJobDetail(PtTrainJobDetailQueryDTO ptTrainJobDetailQueryDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} starts by querying jobId={} for training task details", user.getUsername(), ptTrainJobDetailQueryDTO.getId()); - //获取训练job参数 - QueryWrapper trainJobQuery = new QueryWrapper<>(); - trainJobQuery.eq("create_user_id", user.getId()); - trainJobQuery.eq("id", ptTrainJobDetailQueryDTO.getId()); - PtTrainJob ptTrainJob = ptTrainJobMapper.selectOne(trainJobQuery); - if (ptTrainJob == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The jobId for the user {} query does not exist", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您查询的id不存在或已被删除"); - } - //获取训练参数 - PtTrain ptTrain = ptTrainMapper.selectById(ptTrainJob.getTrainId()); - //获取训练任务参数 - QueryWrapper jobParamQuery = new QueryWrapper<>(); - jobParamQuery.eq("train_job_id", ptTrainJob.getId()); - PtJobParam ptJobParam = ptJobParamMapper.selectOne(jobParamQuery); - if (ptJobParam == null || ptJobParam.getAlgorithmId() < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm ID corresponding to the jobId={} query by the user {} does not exist", user.getUsername(), ptTrainJobDetailQueryDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "您查询的jobId对应的算法id不存在或已被删除"); - } - //获取算法参数 - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectAllById(ptJobParam.getAlgorithmId()); - //结果集处理 - PtTrainJobDetailQueryVO ptTrainJobDetailQueryVO = new PtTrainJobDetailQueryVO(); - BeanUtils.copyProperties(ptTrainJob, ptTrainJobDetailQueryVO); - ptTrainJobDetailQueryVO.setTrainName(ptTrain.getTrainName()).setAlgorithmId(ptJobParam.getAlgorithmId()).setRunCommand(ptJobParam.getRunCommand()) - .setRunParams(ptJobParam.getRunParams()).setParamF1(ptJobParam.getParamF1()).setParamCallback(ptJobParam.getParamCallback()) - .setParamPrecise(ptJobParam.getParamPrecise()).setParamAccuracy(ptJobParam.getParamAccuracy()); - long nowTime = System.currentTimeMillis(); - //获取训练延时启动倒计时（分钟） - if (ptJobParam.getDelayCreateTime() != null && nowTime < ptJobParam.getDelayCreateTime().getTime() && TrainJobStatusEnum.checkRunStatus(ptTrainJob.getTrainStatus())) { - ptTrainJobDetailQueryVO.setDelayCreateCountDown(TrainUtil.getCountDown(ptJobParam.getDelayCreateTime().getTime())); - } - //获取训练自动停止倒计时（分钟） - if (ptJobParam.getDelayDeleteTime() != null && nowTime < ptJobParam.getDelayDeleteTime().getTime() && TrainJobStatusEnum.checkRunStatus(ptTrainJob.getTrainStatus())) { - ptTrainJobDetailQueryVO.setDelayDeleteCountDown(TrainUtil.getCountDown(ptJobParam.getDelayDeleteTime().getTime())); - } - //拼装镜像信息 - if (StringUtils.isNotBlank(ptJobParam.getImageName())) { - String imageNameSuffix = ptJobParam.getImageName().substring(ptJobParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); - String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); - ptTrainJobDetailQueryVO.setImageName(imageNameSuffixArray[0]); - ptTrainJobDetailQueryVO.setImageTag(imageNameSuffixArray[1]); - } - //拼装算法信息 - if (ptTrainAlgorithm != null) { - ptTrainJobDetailQueryVO.setAlgorithmName(ptTrainAlgorithm.getAlgorithmName()) - .setAlgorithmSource(ptTrainAlgorithm.getAlgorithmSource()) - .setAlgorithmUsage(ptTrainAlgorithm.getAlgorithmUsage()) - .setAccuracy(ptTrainAlgorithm.getAccuracy()) - .setP4InferenceSpeed(ptTrainAlgorithm.getP4InferenceSpeed()); - if (ptTrainAlgorithm.getAlgorithmSource() == MagicNumConstant.ONE) { - ptTrainJobDetailQueryVO.setAlgorithmCodeDir(ptTrainAlgorithm.getCodeDir()); - } - } - - return ptTrainJobDetailQueryVO; - } - - /** - * 恢复训练 - * - * @param ptTrainJobResumeDTO 恢复训练请求参数 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public void resumeTrainJob(PtTrainJobResumeDTO ptTrainJobResumeDTO) { - //从会话中获取用户信息 - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "user {} according to jobId={} started to check the training task details", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); - // 检查jobId是否合法 - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainJobResumeDTO.getId()); - if (null == ptTrainJob || !currentUser.getId().equals(ptTrainJob.getCreateUserId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for user {} to resume training job and jobId to be {}", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除，请重新输入"); - } - // 获取算法id和运行参数 - QueryWrapper jobParamQuery = new QueryWrapper<>(); - jobParamQuery.eq("train_job_id", ptTrainJob.getId()); - PtJobParam ptJobParam = ptJobParamMapper.selectOne(jobParamQuery); - if (ptJobParam == null || ptJobParam.getAlgorithmId() < MagicNumConstant.ONE) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The algorithm ID corresponding to the jobId={} query by the user {} does not exist", currentUser.getUsername(), ptTrainJobResumeDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "您查询的jobId对应的算法id不存在"); - } - //获取镜像 - PtImageAndAlgorithmVO ptImageAndAlgorithmVO = getPtImageByAlgorithmId(ptJobParam.getAlgorithmId(), currentUser.getId()); - //使用用户训练时提供的镜像与运行命令 - ptImageAndAlgorithmVO.setImageName(trainHarborConfig.getAddress() + StrUtil.SLASH + ptJobParam.getImageName()).setRunCommand(ptJobParam.getRunCommand()); - String[] codeDirResult = ptImageAndAlgorithmVO.getCodeDir().split(StrUtil.SLASH); - String codeDirName = codeDirResult[codeDirResult.length - 1]; - - //处理目录问题 - String noEnvPath = StrUtil.SLASH + trainJobConfig.getManage() + StrUtil.SLASH + currentUser.getId() + StrUtil.SLASH - + ptTrainJob.getJobName(); - String commonPath = nfsConfig.getBucket() + noEnvPath.substring(1); - String outPath = commonPath + StrUtil.SLASH + trainJobConfig.getOutPath(); - String loadPath = commonPath + StrUtil.SLASH + trainJobConfig.getLoadPath(); - String codePath = commonPath + StrUtil.SLASH + codeDirName; - String noEnvOut = noEnvPath + StrUtil.SLASH + trainJobConfig.getOutPath(); - String path = ptTrainJobResumeDTO.getPath(); - if (!path.startsWith(noEnvOut)) { - LogUtil.error(LogEnum.BIZ_TRAIN, "path: {}", path); - throw new BusinessException("内部错误"); - } - String modelLoadDir = path.substring(noEnvOut.length()); - FileUtil.del(nfsConfig.getRootDir() + loadPath); - FileUtil.del(nfsConfig.getRootDir() + codePath); - FileUtil.rename(new File(nfsConfig.getRootDir() + outPath), nfsConfig.getRootDir() + loadPath, false, true); - - //获取训练规格信息 - PtTrainJobSpecs ptTrainJobSpecs = new PtTrainJobSpecs(); - List dictDetails = dictDetailMapper.selectList(new LambdaQueryWrapper().eq(DictDetail::getLabel, ptTrainJob.getTrainJobSpecsName())); - ptTrainJobSpecs.setResourcesPoolType(ptTrainJob.getResourcesPoolType()); - ptTrainJobSpecs.setSpecsName(ptTrainJob.getTrainJobSpecsName()); - ptTrainJobSpecs.setSpecsInfo(JSONObject.parseObject(dictDetails.get(0).getValue())); - - // 拼load路径 - JSONObject runParams = ptJobParam.getRunParams(); - runParams.put(trainJobConfig.getLoadKey(), trainJobConfig.getDockerTrainPath() + StrUtil.SLASH + - trainJobConfig.getLoadPath() + modelLoadDir); - BaseTrainJobDTO baseTrainJobDTO = new BaseTrainJobDTO(); - BeanUtil.copyProperties(ptTrainJob, baseTrainJobDTO); - baseTrainJobDTO.setPtTrainJobSpecs(ptTrainJobSpecs); - baseTrainJobDTO.setRunParams(runParams); - - // 初始化训练时间和状态 - PtTrainJob updatePtTrainJob = new PtTrainJob(); - updatePtTrainJob.setId(ptTrainJob.getId()).setRuntime(TrainUtil.INIT_RUNTIME) - .setTrainStatus(TrainJobStatusEnum.PENDING.getStatus()) - .setUpdateTime(new Timestamp(System.currentTimeMillis())); - int updateResult = ptTrainJobMapper.updateById(updatePtTrainJob); - if (updateResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} resumed training job, pt train Job table update failed", currentUser.getUsername()); - throw new BusinessException("内部错误"); - } - // 此处将ptTrainJob的trainStatus和runTime设为null以避免doJob中再次调用updateById错误更新状态和时间 - ptTrainJob.setTrainStatus(null).setRuntime(null).setCreateTime(null); - // 提交job - asyncManager.execute(baseTrainJobDTO, currentUser, ptImageAndAlgorithmVO, ptTrainJob); - } - - /** - * 获取job在grafana监控的地址 - * - * @param jobId 任务ID - * @return List grafana监控的地址信息 - */ - @Override - public List getGrafanaUrl(Long jobId) { - - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} gets grafanaUrl of Job and receives parameter [jobId] {}", currentUser.getUsername(), jobId); - - //通过jobId获取job相关信息 - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(jobId); - if (null == ptTrainJob || !currentUser.getId().equals(ptTrainJob.getCreateUserId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for user {} to get grafanaUrl on Job, jobId for {}", currentUser.getUsername(), jobId); - throw new BusinessException(ResponseCode.SUCCESS, "您输入的id不存在或已被删除，请重新输入"); - } - - List list = new ArrayList<>(); - try { - List bizPodList = podApi.getListByResourceName(k8sNameTool.generateNamespace(currentUser.getId()), ptTrainJob.getJobName()); - bizPodList.stream() - .filter(bizPod -> bizPod.getPhase().equalsIgnoreCase(TrainJobStatusEnum.RUNNING.getMessage())) - .forEach(bizPod -> { - String podName = bizPod.getName(); - PtJobMetricsGrafanaVO ptJobMetricsGrafanaVO = new PtJobMetricsGrafanaVO(); - ptJobMetricsGrafanaVO.setJobMetricsGrafanaUrl(k8sPodMetricsGrafanaUrl.concat(podName)); - ptJobMetricsGrafanaVO.setJobPodName(podName); - if (ptTrainJob.getTrainType() == 1 && PodUtil.isMaster(podName)) { - list.add(0, ptJobMetricsGrafanaVO); - } else { - list.add(ptJobMetricsGrafanaVO); - } - }); - } catch (Exception e) { - LogUtil.info(LogEnum.BIZ_K8S, "Failed to obtain grafanaUrl of Pod, params:[namespace]={}, [resourceName]={}, error:{}", - k8sNameTool.generateNamespace(currentUser.getId()), ptTrainJob.getJobName(), e); - } - - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} completes getting grafanaUrl on job, receives {} parameter [jobId], returns {} result", - currentUser.getUsername(), jobId, JSONObject.toJSONString(list)); - return list; - } - - /** - * 获取训练使用的模型信息 - * - * @param ptTrainModelDTO - * @return PtTrainJobModelVO - */ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public PtTrainJobModelVO getTrainJobModel(PtTrainModelDTO ptTrainModelDTO) { - - PtTrainJobModelVO ptTrainJobModelVO = new PtTrainJobModelVO(); - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - Integer modelResource = ptTrainModelDTO.getModelResource(); - switch (ModelResourceEnum.getType(modelResource)) { - case MINE: - if(null == ptTrainModelDTO.getModelBranchId() || null == ptTrainModelDTO.getModelId()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(ptTrainModelDTO.getModelBranchId()); - if(null == ptModelBranch) { - break; - } - if(ptModelBranch.getParentId().compareTo(ptTrainModelDTO.getModelId()) != 0 || - StringUtils.isBlank(ptModelBranch.getModelAddress())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); - if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(ptTrainModelDTO.getModelResource()) != 0){ - logErrorInfoOnModel(currentUser.getUsername()); - } - ptTrainJobModelVO.setModelList(new ArrayList<>()); - ptTrainJobModelVO.getModelList() - .add(new ModelVO(ptModelInfo.getName(), ptModelBranch.getVersionNum(), adjustmentUrl(ptModelBranch.getModelAddress()))); - break; - case PRESET: - if(null == ptTrainModelDTO.getModelId()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(ptTrainModelDTO.getModelId()); - if(null == ptModelInfoPreset) { - break; - } - if(StringUtils.isBlank(ptModelInfoPreset.getUrl()) || - ptModelInfoPreset.getModelResource().compareTo(ptTrainModelDTO.getModelResource()) != 0) { - logErrorInfoOnModel(currentUser.getUsername()); - } - ptTrainJobModelVO.setModelList(new ArrayList<>()); - ptTrainJobModelVO.getModelList() - .add(new ModelVO(ptModelInfoPreset.getName(), ptModelInfoPreset.getVersionNum(), adjustmentUrl(ptModelInfoPreset.getUrl()))); - break; - case ATLAS: - if(StringUtils.isBlank(ptTrainModelDTO.getTeacherModelIds())){ - logErrorInfoOnModel(currentUser.getUsername()); - } - Set ids = new HashSet<>(); - Set teacherModelList = new HashSet<>(); - Arrays.stream(ptTrainModelDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> teacherModelList.add(Long.parseLong(id))); - ids.addAll(teacherModelList); - - Set studentModelList = new HashSet<>(); - if(StringUtils.isNotBlank(ptTrainModelDTO.getStudentModelIds())) { - Arrays.stream(ptTrainModelDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> studentModelList.add(Long.parseLong(id))); - ids.addAll(studentModelList); - } - if(ids.isEmpty()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - LambdaQueryWrapper query = new LambdaQueryWrapper<>(); - query.eq(PtModelInfo::getModelResource, ptTrainModelDTO.getModelResource()) - .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); - List modelInfoList = ptModelInfoMapper.selectList(query); - if(null == modelInfoList || modelInfoList.isEmpty()) { - break; - } - - //保存炼知教师模型信息 - ptTrainJobModelVO.setTeacherModelList(new ArrayList<>()); - List teacherModelVOS = ptTrainJobModelVO.getTeacherModelList(); - modelInfoList.stream() - .filter(modelInfo -> teacherModelList.contains(modelInfo.getId())) - .forEach(modelInfo -> teacherModelVOS - .add(new ModelVO(modelInfo.getName(), modelInfo.getVersionNum(), adjustmentUrl(modelInfo.getUrl())))); - - //保存炼知学生模型信息 - if(!studentModelList.isEmpty()) { - ptTrainJobModelVO.setStudentModelList(new ArrayList<>()); - List studentModelVOS = ptTrainJobModelVO.getStudentModelList(); - modelInfoList.stream() - .filter(modelInfo -> studentModelList.contains(modelInfo.getId())) - .forEach(modelInfo -> studentModelVOS - .add(new ModelVO(modelInfo.getName(), modelInfo.getVersionNum(), adjustmentUrl(modelInfo.getUrl())))); - } - break; - } - - return ptTrainJobModelVO; - } - - public void recycleTaskWithTrain(String recyclePath) { - //创建已删除训练任务的无效文件回收任务 - RecycleTaskCreateDTO recycleTask = new RecycleTaskCreateDTO(); - recycleTask.setRecycleModule(RecycleModuleEnum.BIZ_TRAIN.getValue()) - .setRecycleType(RecycleTypeEnum.FILE.getCode()) - .setRecycleDelayDate(recycleConfig.getTrainValid()) - .setRecycleCondition(recyclePath) - .setRecycleNote("回收已删除训练任务文件"); - LogUtil.info(LogEnum.BIZ_TRAIN, "delete train job add recycle task:{}", recycleTask); - recycleTaskService.createRecycleTask(recycleTask); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java deleted file mode 100644 index 32b4623..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainJobSpecsServiceImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import org.dubhe.dao.PtTrainJobSpecsMapper; -import org.dubhe.domain.dto.PtTrainJobSpecsQueryDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainJobSpecs; -import org.dubhe.domain.vo.PtTrainJobSpecsQueryVO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.PtTrainJobSpecsService; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.LogUtil; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * @description 训练作业规格服务实现类 - * @date 2020-05-06 - */ -@Service -public class PtTrainJobSpecsServiceImpl implements PtTrainJobSpecsService { - - @Autowired - private PtTrainJobSpecsMapper ptTrainJobSpecsMapper; - - /** - * 规格查询 - * - * @param ptTrainJobSpecsQueryDTO 查询规格表参数 - * @return List 返回规格查询结果 - **/ - @Override - public List getTrainJobSpecs(PtTrainJobSpecsQueryDTO ptTrainJobSpecsQueryDTO) { - UserDTO currentUser = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "user{}Query job list display, the received parameters are{}", currentUser.getUsername(), ptTrainJobSpecsQueryDTO); - QueryWrapper query = new QueryWrapper<>(); - if (ptTrainJobSpecsQueryDTO.getResourcesPoolType() != null) { - query.eq("resources_pool_type", ptTrainJobSpecsQueryDTO.getResourcesPoolType()); - } - List ptTrainJobSpecs = ptTrainJobSpecsMapper.selectList(query); - List ptTrainJobSpecsQueryList = ptTrainJobSpecs.stream().map(x -> { - PtTrainJobSpecsQueryVO ptTrainJobSpecsQueryVO = new PtTrainJobSpecsQueryVO(); - BeanUtils.copyProperties(x, ptTrainJobSpecsQueryVO); - return ptTrainJobSpecsQueryVO; - }).collect(Collectors.toList()); - return ptTrainJobSpecsQueryList; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java deleted file mode 100644 index f5c5f41..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainLogServiceImpl.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.util.StrUtil; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.PtTrainJobMapper; -import org.dubhe.domain.dto.PtTrainLogQueryDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainJob; -import org.dubhe.domain.vo.PtTrainLogQueryVO; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.LogMonitoringApi; -import org.dubhe.k8s.domain.bo.LogMonitoringBO; -import org.dubhe.k8s.domain.dto.PodQueryDTO; -import org.dubhe.k8s.domain.vo.LogMonitoringVO; -import org.dubhe.k8s.domain.vo.PodVO; -import org.dubhe.k8s.service.PodService; -import org.dubhe.service.PtTrainLogService; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.LogUtil; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.util.Collections; -import java.util.List; - - -/** - * @description 训练日志 服务实现类 - * @date 2020-05-08 - */ - -@Service -public class PtTrainLogServiceImpl implements PtTrainLogService { - - @Autowired - private LogMonitoringApi logMonitoringApi; - - @Autowired - private PtTrainJobMapper ptTrainJobMapper; - - @Autowired - private K8sNameTool k8sNameTool; - - @Autowired - private PodService podService; - - /** - * 查询训练任务运行日志 - * - * @param ptTrainLogQueryDTO 训练日志查询 - * @return PtTrainLogQueryVO 返回训练日志查询 - **/ - @Override - public PtTrainLogQueryVO queryTrainLog(PtTrainLogQueryDTO ptTrainLogQueryDTO) { - LogUtil.info(LogEnum.BIZ_TRAIN, "Start viewing log parameters, received as {}", ptTrainLogQueryDTO); - - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - String namespace = k8sNameTool.generateNamespace(user.getId()); - - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(ptTrainLogQueryDTO.getJobId()); - if (null == ptTrainJob || !user.getId().equals(ptTrainJob.getCreateUserId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "It is illegal for the user {} to look up the training job log with jobId as {}", user.getId(), ptTrainLogQueryDTO.getJobId()); - throw new BusinessException("内部错误"); - } - - /** 查询日志起始行 **/ - Integer startLine = null == ptTrainLogQueryDTO.getStartLine() ? NumberConstant.NUMBER_1 : ptTrainLogQueryDTO.getStartLine(); - /** 查询日志总行数 **/ - Integer lines = null == ptTrainLogQueryDTO.getLines() ? NumberConstant.NUMBER_50 : ptTrainLogQueryDTO.getLines(); - /** 拼接请求es的参数 **/ - LogMonitoringBO logMonitoringBo = new LogMonitoringBO(); - logMonitoringBo.setNamespace(namespace) - .setResourceName(ptTrainJob.getJobName()); - - PtTrainLogQueryVO ptTrainLogQueryVO = new PtTrainLogQueryVO(); - - LogMonitoringVO result = logMonitoringApi.searchLogByResName(startLine, lines, logMonitoringBo); - List list = result.getLogs(); - - if (CollectionUtils.isEmpty(list)) { - ptTrainLogQueryVO.setContent(list); - ptTrainLogQueryVO.setStartLine(startLine); - ptTrainLogQueryVO.setEndLine(startLine - 1); - ptTrainLogQueryVO.setLines(0); - return ptTrainLogQueryVO; - } - - ptTrainLogQueryVO.setContent(list).setStartLine(startLine) - .setEndLine(Long.valueOf(startLine + result.getTotalLogs()).intValue() - 1) - .setLines(Long.valueOf(result.getTotalLogs()).intValue()).setJobName(ptTrainJob.getJobName()); - LogUtil.info(LogEnum.BIZ_TRAIN, "Query log results are returned==>{}", ptTrainLogQueryVO); - return ptTrainLogQueryVO; - } - - /** - * 字符串换行 - * - * @param content 内容 - * @return strContent 返回字符串内容 - */ - @Override - public String getTrainLogString(List content) { - if (content == null) { - return SymbolConstant.BLANK; - } - return StringUtils.join(content,StrUtil.CRLF); - } - - /** - * 获取训练任务的Pod - * - * @param id 训练作业job表 id - * @return 训练节点信息 - */ - @Override - public List getPods(Long id) { - PtTrainJob ptTrainJob = ptTrainJobMapper.selectById(id); - if (ptTrainJob == null){ - return Collections.emptyList(); - } - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - String nameSpace = k8sNameTool.generateNamespace(user.getId()); - return podService.getPods(new PodQueryDTO(nameSpace,ptTrainJob.getJobName())); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java deleted file mode 100644 index 36b6b65..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/PtTrainParamServiceImpl.java +++ /dev/null @@ -1,447 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.base.MagicNumConstant; -import org.dubhe.base.ResponseCode; -import org.dubhe.config.TrainJobConfig; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.PtModelBranchMapper; -import org.dubhe.dao.PtModelInfoMapper; -import org.dubhe.dao.PtTrainAlgorithmMapper; -import org.dubhe.dao.PtTrainParamMapper; -import org.dubhe.data.constant.Constant; -import org.dubhe.domain.PtModelBranch; -import org.dubhe.domain.PtModelInfo; -import org.dubhe.domain.dto.BaseTrainParamDTO; -import org.dubhe.domain.dto.PtTrainParamCreateDTO; -import org.dubhe.domain.dto.PtTrainParamDeleteDTO; -import org.dubhe.domain.dto.PtTrainParamQueryDTO; -import org.dubhe.domain.dto.PtTrainParamUpdateDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtTrainAlgorithm; -import org.dubhe.domain.entity.PtTrainParam; -import org.dubhe.domain.vo.PtTrainParamQueryVO; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.ModelResourceEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.service.PtTrainParamService; -import org.dubhe.utils.ImageUtil; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.StringUtils; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -/** - * @description 任务参数服务实现类 - * @date 2020-04-27 - */ -@Service -public class PtTrainParamServiceImpl implements PtTrainParamService { - - @Autowired - private PtTrainParamMapper ptTrainParamMapper; - - @Autowired - private PtTrainAlgorithmMapper ptTrainAlgorithmMapper; - - @Autowired - private ImageUtil imageUtil; - - @Autowired - private PtModelInfoMapper ptModelInfoMapper; - - @Autowired - private PtModelBranchMapper ptModelBranchMapper; - - /** - * 参数列表展示 - * - * @param ptTrainParamQueryDTO 任务参数列表展示条件 - * @return Map 任务参数列表分页数据 - **/ - @Override - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public Map getTrainParam(PtTrainParamQueryDTO ptTrainParamQueryDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The display of user {} query task parameter list begins. The parameters received are {}.", user.getUsername(), ptTrainParamQueryDTO); - Page page = ptTrainParamQueryDTO.toPage(); - //查询任务参数列表 - QueryWrapper query = new QueryWrapper<>(); - //根据任务参数名称模糊搜索 - if (ptTrainParamQueryDTO.getParamName() != null) { - query.like("param_name", ptTrainParamQueryDTO.getParamName()); - } - //根据类型筛选 - if (ptTrainParamQueryDTO.getResourcesPoolType() != null) { - query.eq("resources_pool_type", ptTrainParamQueryDTO.getResourcesPoolType()); - } - IPage ptTrainParams; - try { - if (ptTrainParamQueryDTO.getSort() == null || ptTrainParamQueryDTO.getSort().equalsIgnoreCase(TrainJobConfig.ALGORITHM_NAME)) { - query.orderByDesc(Constant.ID); - } else { - if (Constant.SORT_ASC.equalsIgnoreCase(ptTrainParamQueryDTO.getOrder())) { - query.orderByAsc(StringUtils.humpToLine(ptTrainParamQueryDTO.getSort())); - } else { - query.orderByDesc(StringUtils.humpToLine(ptTrainParamQueryDTO.getSort())); - } - } - ptTrainParams = ptTrainParamMapper.selectPage(page, query); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Query task parameter list shows exception {}", e); - throw new BusinessException("内部错误"); - } - //结果集的处理 - //获取算法id - List algorithmIds = ptTrainParams.getRecords().stream().map(PtTrainParam::getAlgorithmId).collect(Collectors.toList()); - if (algorithmIds.size() < 1) { - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query task parameter list is empty, the result is {}", user.getUsername(), ptTrainParams); - List ptTrainParamQueryResult = new ArrayList<>(); - return PageUtil.toPage(page, ptTrainParamQueryResult); - } - List ptTrainAlgorithms = ptTrainAlgorithmMapper.selectAllBatchIds(algorithmIds); - //获取算法id对应的算法名称并封装至map集合中 - Map ptTrainAlgorithmMap = ptTrainAlgorithms.stream().collect(Collectors.toMap(PtTrainAlgorithm::getId, PtTrainAlgorithm::getAlgorithmName, (o, n) -> n)); - List ptTrainParamQueryResult = ptTrainParams.getRecords().stream().map(x -> { - PtTrainParamQueryVO ptTrainParamQueryVO = new PtTrainParamQueryVO(); - BeanUtils.copyProperties(x, ptTrainParamQueryVO); - ptTrainParamQueryVO.setAlgorithmName(ptTrainAlgorithmMap.get(x.getAlgorithmId())); - //获取镜像名称与版本 - getImageNameAndImageTag(x, ptTrainParamQueryVO); - return ptTrainParamQueryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} query task parameter list display ends, the result is {}", user.getUsername(), ptTrainParamQueryResult); - return PageUtil.toPage(page, ptTrainParamQueryResult); - } - - /** - * 保存任务参数 - * - * @param ptTrainParamCreateDTO 保存任务参数条件 - * @return List 保存任务参数id集合 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List createTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "The user {} saves the task parameter and receives the parameter {}.", user.getUsername(), ptTrainParamCreateDTO); - //参数校验 - PtTrainAlgorithm ptTrainAlgorithm = checkCreateTrainParam(ptTrainParamCreateDTO, user); - //获取算法来源（1为我的算法，2为预置算法） - Integer algorithmSource = ptTrainAlgorithm.getAlgorithmSource(); - //保存任务参数 - PtTrainParam ptTrainParam = new PtTrainParam(); - //模型检测 - BaseTrainParamDTO baseTrainParamDTO = new BaseTrainParamDTO(); - BeanUtil.copyProperties(ptTrainParamCreateDTO, baseTrainParamDTO); - checkModel(user, baseTrainParamDTO); - - BeanUtils.copyProperties(ptTrainParamCreateDTO, ptTrainParam); - //获取镜像 - String images = imageUtil.getImageUrl(ptTrainParamCreateDTO, user); - ptTrainParam.setImageName(images).setAlgorithmSource(algorithmSource).setCreateUserId(user.getId()); - int insertResult = ptTrainParamMapper.insert(ptTrainParam); - //任务参数未保存成功，抛出异常，并返回失败信息 - if (insertResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} saved the task parameters successfully, and the pt_train_param table insert operation failed", user.getUsername()); - throw new BusinessException("内部错误"); - } - //返回新增任务参数id - LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} saving task parameters, return new task parameter ID ={}", user.getUsername(), ptTrainParam.getId()); - return Collections.singletonList(ptTrainParam.getId()); - } - - /** - * 检查模型是否合法 - * - * @param currentUser 用户 - * @param baseTrainParamDTO 基础训练模板参数 - */ - private void checkModel(UserDTO currentUser, BaseTrainParamDTO baseTrainParamDTO) { - - Integer modelResource = baseTrainParamDTO.getModelResource(); - if(null == modelResource ) { - if(null == baseTrainParamDTO.getModelId() && - StringUtils.isBlank(baseTrainParamDTO.getStudentModelIds()) && - StringUtils.isBlank(baseTrainParamDTO.getStudentModelIds())) { - return; - } else { - logErrorInfoOnModel(currentUser.getUsername()); - } - } - switch (ModelResourceEnum.getType(modelResource)) { - case MINE: - if(null == baseTrainParamDTO.getModelBranchId() || null == baseTrainParamDTO.getModelId() || - StringUtils.isNotBlank(baseTrainParamDTO.getTeacherModelIds()) || - StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelBranch ptModelBranch = ptModelBranchMapper.selectById(baseTrainParamDTO.getModelBranchId()); - if(null == ptModelBranch || ptModelBranch.getParentId().compareTo(baseTrainParamDTO.getModelId()) != 0 || - StringUtils.isBlank(ptModelBranch.getModelAddress())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(ptModelBranch.getParentId()); - if(null == ptModelInfo || ptModelInfo.getModelResource().compareTo(baseTrainParamDTO.getModelResource()) != 0){ - logErrorInfoOnModel(currentUser.getUsername()); - } - break; - case PRESET: - if(null == baseTrainParamDTO.getModelId() || StringUtils.isNotBlank(baseTrainParamDTO.getTeacherModelIds()) || - StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { - logErrorInfoOnModel(currentUser.getUsername()); - } - PtModelInfo ptModelInfoPreset = ptModelInfoMapper.selectById(baseTrainParamDTO.getModelId()); - if(null == ptModelInfoPreset || StringUtils.isBlank(ptModelInfoPreset.getUrl()) || - ptModelInfoPreset.getModelResource().compareTo(baseTrainParamDTO.getModelResource()) != 0) { - logErrorInfoOnModel(currentUser.getUsername()); - } - break; - case ATLAS: - if(StringUtils.isBlank(baseTrainParamDTO.getTeacherModelIds()) || null != baseTrainParamDTO.getModelId()){ - logErrorInfoOnModel(currentUser.getUsername()); - } - Set ids = new HashSet<>(); - Set teacherModelList = new HashSet<>(); - Arrays.stream(baseTrainParamDTO.getTeacherModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> teacherModelList.add(Long.parseLong(id))); - ids.addAll(teacherModelList); - - Set studentModelList = new HashSet<>(); - if(StringUtils.isNotBlank(baseTrainParamDTO.getStudentModelIds())) { - Arrays.stream(baseTrainParamDTO.getStudentModelIds().trim().split(SymbolConstant.COMMA)) - .forEach(id -> studentModelList.add(Long.parseLong(id))); - ids.addAll(studentModelList); - } - if(ids.isEmpty()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - LambdaQueryWrapper query = new LambdaQueryWrapper<>(); - query.eq(PtModelInfo::getModelResource, baseTrainParamDTO.getModelResource()) - .in(PtModelInfo::getId, ids).isNotNull(PtModelInfo::getUrl).ne(PtModelInfo::getUrl, SymbolConstant.BLANK); - List modelInfoList = ptModelInfoMapper.selectList(query); - if(null == modelInfoList || modelInfoList.size() < ids.size()) { - logErrorInfoOnModel(currentUser.getUsername()); - } - break; - } - } - - /** - * 打印训练模板中模型相关的错误日志 - * - * @param username - */ - private void logErrorInfoOnModel(String username){ - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} operating training param, error on model......", username); - throw new BusinessException("模型参数不合法"); - } - - /** - * 修改任务参数 - * - * @param ptTrainParamUpdateDTO 修改任务参数条件 - * @return List 修改任务参数id集合 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public List updateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} modifies the task parameters and receives the parameters as {}.", user.getUsername(), ptTrainParamUpdateDTO); - //参数校验 - checkUpdateTrainParam(ptTrainParamUpdateDTO, user); - //修改任务参数 - PtTrainParam ptTrainParam = new PtTrainParam(); - //模型检测 - BaseTrainParamDTO baseTrainParamDTO = new BaseTrainParamDTO(); - BeanUtil.copyProperties(ptTrainParamUpdateDTO, baseTrainParamDTO); - checkModel(user, baseTrainParamDTO); - - BeanUtils.copyProperties(ptTrainParamUpdateDTO, ptTrainParam); - ptTrainParam.setUpdateUserId(user.getId()); - //获取镜像url - String images = imageUtil.getImageUrl(ptTrainParamUpdateDTO, user); - //添加镜像url - ptTrainParam.setImageName(images); - try { - //任务参数未修改成功，抛出异常，并返回失败信息 - ptTrainParamMapper.updateById(ptTrainParam); - } catch (Exception e) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to modify the task parameters. The modification operation of pt_train_param table failed. The failure reason :{}", user.getUsername(), e.getMessage()); - throw new BusinessException("内部错误"); - } - - //返回修改任务参数id - LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} saving task parameters, return modify task parameter ID ={}", user.getUsername(), ptTrainParam.getId()); - return Collections.singletonList(ptTrainParam.getId()); - } - - /** - * 删除任务参数 - * - * @param ptTrainParamDeleteDTO 删除任务参数条件 - **/ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public void deleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - LogUtil.info(LogEnum.BIZ_TRAIN, "User {} deletes the task parameter and receives the parameter {}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); - Set idList = ptTrainParamDeleteDTO.getIds(); - //参数校验 - checkDeleteTrainParam(ptTrainParamDeleteDTO, user, idList); - //删除任务参数 - int deleteCountResult = ptTrainParamMapper.deleteBatchIds(idList); - //任务参数未删除成功,抛出异常，并返回失败信息 - if (deleteCountResult < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to delete the task parameter, and the pt_train_param table deletion operation failed according to the id array {}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); - throw new BusinessException("内部错误"); - } - //返回删除任务参数id数组 - LogUtil.info(LogEnum.BIZ_TRAIN, "End of user {} deleting task parameters,the array of delete task parameters IDS ={}", user.getUsername(), ptTrainParamDeleteDTO.getIds()); - } - - /** - * 获取镜像名称与版本 - * - * @param trainParam 镜像URL - * @param ptTrainParamQueryVO 镜像名称与版本 - */ - private void getImageNameAndImageTag(PtTrainParam trainParam, PtTrainParamQueryVO ptTrainParamQueryVO) { - if (StringUtils.isNotBlank(trainParam.getImageName())) { - String imageNameSuffix = trainParam.getImageName().substring(trainParam.getImageName().lastIndexOf(StrUtil.SLASH) + MagicNumConstant.ONE); - String[] imageNameSuffixArray = imageNameSuffix.split(StrUtil.COLON); - ptTrainParamQueryVO.setImageName(imageNameSuffixArray[0]); - ptTrainParamQueryVO.setImageTag(imageNameSuffixArray[1]); - } - } - - /** - * 创建训练参数模板参数校验 - * - * @param ptTrainParamCreateDTO 任务参数创建条件 - * @param user 用户 - * @return PtTrainAlgorithm 算法 - */ - private PtTrainAlgorithm checkCreateTrainParam(PtTrainParamCreateDTO ptTrainParamCreateDTO, UserDTO user) { - //算法id校验 - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectAllById(ptTrainParamCreateDTO.getAlgorithmId()); - if (ptTrainAlgorithm == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Algorithm ID ={} does not exist", ptTrainParamCreateDTO.getAlgorithmId()); - throw new BusinessException("算法不存在或已被删除"); - } - //任务参数名称校验 - QueryWrapper query = new QueryWrapper<>(); - query.eq("param_name", ptTrainParamCreateDTO.getParamName()); - Integer trainParamCountResult = ptTrainParamMapper.selectCount(query); - if (trainParamCountResult > 0) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter name ({}) already exists", ptTrainParamCreateDTO.getParamName()); - throw new BusinessException("任务参数名称已存在"); - } - return ptTrainAlgorithm; - } - - /** - * 修改训练参数模板参数校验 - * - * @param ptTrainParamUpdateDTO 训练参数模板参数 - * @param user 用户 - */ - private void checkUpdateTrainParam(PtTrainParamUpdateDTO ptTrainParamUpdateDTO, UserDTO user) { - //任务参数id校验 - if (ptTrainParamMapper.selectById(ptTrainParamUpdateDTO.getId()) == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter ID ={} does not exist", ptTrainParamUpdateDTO.getId()); - throw new BusinessException(ResponseCode.SUCCESS, "任务参数id不存在"); - } - //算法id校验 - PtTrainAlgorithm ptTrainAlgorithm = ptTrainAlgorithmMapper.selectById(ptTrainParamUpdateDTO.getAlgorithmId()); - if (ptTrainAlgorithm == null) { - LogUtil.error(LogEnum.BIZ_TRAIN, "Algorithm ID ={} does not exist", ptTrainParamUpdateDTO.getAlgorithmId()); - throw new BusinessException(ResponseCode.SUCCESS, "算法id不存在"); - } - //权限校验 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("id", ptTrainParamUpdateDTO.getId()); - Integer countResult = ptTrainParamMapper.selectCount(queryWrapper); - if (countResult < 1) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The user {} failed to modify the task parameters and has no permission to modify the corresponding data in the pt_train_param table", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您修改的ID不存在或已被删除"); - } - //任务参数名称校验 - QueryWrapper query = new QueryWrapper<>(); - query.eq("param_name", ptTrainParamUpdateDTO.getParamName()); - PtTrainParam trainParam = ptTrainParamMapper.selectOne(query); - if (trainParam != null && !ptTrainParamUpdateDTO.getId().equals(trainParam.getId())) { - LogUtil.error(LogEnum.BIZ_TRAIN, "The task parameter name ({}) already exists", ptTrainParamUpdateDTO.getParamName()); - throw new BusinessException(ResponseCode.SUCCESS, "任务参数名称已存在"); - } - } - - /** - * 删除训练参数模板参数校验 - * - * @param ptTrainParamDeleteDTO 训练参数模板参数 - * @param user 用户 - * @param idList 训练参数id集合 - **/ - private void checkDeleteTrainParam(PtTrainParamDeleteDTO ptTrainParamDeleteDTO, UserDTO user, Set idList) { - //id校验 - List ptTrainParams = ptTrainParamMapper.selectBatchIds(idList); - if (ptTrainParams.size() == 0 || ptTrainParams.size() != idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete the task parameters, request parameters IDS ={} cannot query the corresponding data in pt_train_param table, the parameters are illegal", user.getUsername(), ptTrainParamDeleteDTO.getIds()); - throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); - } - //权限校验 - QueryWrapper query = new QueryWrapper<>(); - query.in("id", idList); - Integer queryCountResult = ptTrainParamMapper.selectCount(query); - if (queryCountResult < idList.size()) { - LogUtil.error(LogEnum.BIZ_TRAIN, "User {} failed to delete the task parameters and has no permission to delete the corresponding data in the pt_train_param table", user.getUsername()); - throw new BusinessException(ResponseCode.SUCCESS, "您删除的ID不存在或已被删除"); - } - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java deleted file mode 100644 index 2060c48..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingAsyncServiceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import org.dubhe.dto.callback.BaseK8sDeploymentCallbackCreateDTO; -import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; -import org.dubhe.enums.LogEnum; -import org.dubhe.service.DeploymentCallbackAsyncService; -import org.dubhe.service.ServingService; -import org.dubhe.service.abstracts.AbstractDeploymentCallback; -import org.dubhe.utils.LogUtil; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; - -/** - * @description 云端Serving回调 - * @date 2020-09-16 - */ -@Service(value = "servingAsyncService") -public class ServingAsyncServiceImpl extends AbstractDeploymentCallback implements DeploymentCallbackAsyncService { - - @Resource - private ServingService servingService; - - @Override - public boolean doCallback(int times, R k8sDeploymentCallbackCreateDTO) { - // 强制转型 - ServingK8sDeploymentCallbackCreateDTO req = (ServingK8sDeploymentCallbackCreateDTO) k8sDeploymentCallbackCreateDTO; - LogUtil.info(LogEnum.SERVING, "Thread {} try {} time.Request: {}", Thread.currentThread(), times, req.toString()); - //在线服务回调 - return servingService.servingCallback(times, req); - } - - @Override - public void callbackFailed(int retryTimes, R k8sDeploymentCallbackCreateDTO) { - ServingK8sDeploymentCallbackCreateDTO req = (ServingK8sDeploymentCallbackCreateDTO) k8sDeploymentCallbackCreateDTO; - LogUtil.info(LogEnum.SERVING, "Thread {} try {} times FAILED! if you want to storage or send failed msg,please impl this.. Request: {}", Thread.currentThread(), retryTimes, req.toString()); - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java deleted file mode 100644 index 7078a2a..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/service/impl/ServingServiceImpl.java +++ /dev/null @@ -1,951 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.service.impl; - -import cn.hutool.core.bean.BeanUtil; -import com.alibaba.fastjson.JSONObject; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import io.grpc.ManagedChannel; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections4.CollectionUtils; -import org.dubhe.annotation.DataPermissionMethod; -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.ServingConstant; -import org.dubhe.constant.StringConstant; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.dao.PtModelBranchMapper; -import org.dubhe.dao.PtModelInfoMapper; -import org.dubhe.dao.ServingInfoMapper; -import org.dubhe.dao.ServingModelConfigMapper; -import org.dubhe.domain.PtModelBranch; -import org.dubhe.domain.PtModelInfo; -import org.dubhe.domain.dto.PredictParamDTO; -import org.dubhe.domain.dto.ServingInfoCreateDTO; -import org.dubhe.domain.dto.ServingInfoDeleteDTO; -import org.dubhe.domain.dto.ServingInfoDetailDTO; -import org.dubhe.domain.dto.ServingInfoQueryDTO; -import org.dubhe.domain.dto.ServingInfoUpdateDTO; -import org.dubhe.domain.dto.ServingModelConfigDTO; -import org.dubhe.domain.dto.ServingStartDTO; -import org.dubhe.domain.dto.ServingStopDTO; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.ImageInfo; -import org.dubhe.domain.entity.ServingInfo; -import org.dubhe.domain.entity.ServingModelConfig; -import org.dubhe.domain.vo.PredictParamVO; -import org.dubhe.domain.vo.ServingConfigMetricsVO; -import org.dubhe.domain.vo.ServingInfoCreateVO; -import org.dubhe.domain.vo.ServingInfoDeleteVO; -import org.dubhe.domain.vo.ServingInfoDetailVO; -import org.dubhe.domain.vo.ServingInfoQueryVO; -import org.dubhe.domain.vo.ServingInfoUpdateVO; -import org.dubhe.domain.vo.ServingMetricsVO; -import org.dubhe.domain.vo.ServingModelConfigVO; -import org.dubhe.domain.vo.ServingPodMetricsVO; -import org.dubhe.domain.vo.ServingStartVO; -import org.dubhe.domain.vo.ServingStopVO; -import org.dubhe.dto.callback.ServingK8sDeploymentCallbackCreateDTO; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.DatasetTypeEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.ServingErrorEnum; -import org.dubhe.enums.ServingRouteEventEnum; -import org.dubhe.enums.ServingStatusEnum; -import org.dubhe.enums.ServingTypeEnum; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.MetricsApi; -import org.dubhe.k8s.domain.dto.PodQueryDTO; -import org.dubhe.k8s.domain.vo.PodVO; -import org.dubhe.k8s.domain.vo.PtPodsVO; -import org.dubhe.k8s.service.PodService; -import org.dubhe.service.ServingLuaScriptService; -import org.dubhe.service.ServingModelConfigService; -import org.dubhe.service.ServingService; -import org.dubhe.task.DeployServingAsyncTask; -import org.dubhe.utils.DateUtil; -import org.dubhe.utils.GrpcClient; -import org.dubhe.utils.JwtUtils; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.K8sUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.NfsUtil; -import org.dubhe.utils.PageUtil; -import org.dubhe.utils.ReflectionUtils; -import org.dubhe.utils.StringUtils; -import org.dubhe.utils.WrapperHelp; -import org.springframework.amqp.rabbit.connection.CorrelationData; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.redis.connection.stream.StreamRecords; -import org.springframework.data.redis.connection.stream.StringRecord; -import org.springframework.data.redis.core.StringRedisTemplate; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.multipart.MultipartFile; - -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; - - -/** - * @description 在线服务管理 - * @date 2020-08-25 - */ -@Service -public class ServingServiceImpl implements ServingService { - - @Value("${serving.gateway-uri-postfix}") - private String GATEWAY_URI_POSTFIX; - @Value("${k8s.pod.metrics.grafanaUrl}") - private String k8sPodMetricsGrafanaUrl; - @Value("${serving.group}") - private String servingGroup; - @Resource - private ServingInfoMapper servingInfoMapper; - @Resource - private ServingModelConfigService servingModelConfigService; - @Resource - private ServingModelConfigMapper servingModelConfigMapper; - @Resource - private PtModelInfoMapper ptModelInfoMapper; - @Resource - private PtModelBranchMapper ptModelBranchMapper; - @Resource - private DeployServingAsyncTask deployServingAsyncTask; - @Resource - private K8sUtil k8sUtil; - @Resource - private ServingLuaScriptService servingLuaScriptService; - @Resource - private PodService podService; - @Resource - private MetricsApi metricsApi; - @Resource - private NfsUtil nfsUtil; - @Resource - private K8sNameTool k8sNameTool; - @Resource - private StringRedisTemplate stringRedisTemplate; - - public final static List FILE_NAMES; - - static { - FILE_NAMES = ReflectionUtils.getFieldNames(ServingInfoQueryVO.class); - } - - /** - * 查询分页数据 - * - * @param servingInfoQueryDTO 服务查询参数 - * @return Map云端服务分页对象 - */ - @Override - public Map query(ServingInfoQueryDTO servingInfoQueryDTO) { - String name = servingInfoQueryDTO.getName(); - //服务名称或id条件非空 - if (StringUtils.isNotBlank(name)) { - //整数匹配 - if (StringConstant.PATTERN_NUM.matcher(name).matches()) { - servingInfoQueryDTO.setId(Long.parseLong(name)); - servingInfoQueryDTO.setName(null); - Map map = queryServing(servingInfoQueryDTO); - if (((List) map.get(StringConstant.RESULT)).size() > NumberConstant.NUMBER_0) { - return map; - } else { - servingInfoQueryDTO.setId(null); - servingInfoQueryDTO.setName(name); - } - } - } - return queryServing(servingInfoQueryDTO); - } - - /** - * 分页查询在线服务 - * - * @param servingInfoQueryDTO 查询条件 - * @return Map 分页结果 - */ - public Map queryServing(ServingInfoQueryDTO servingInfoQueryDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - LogUtil.info(LogEnum.SERVING, "User {} queried online service list, with the query {}", userDTO.getUsername(), JSONObject.toJSONString(servingInfoQueryDTO)); - QueryWrapper wrapper = WrapperHelp.getWrapper(servingInfoQueryDTO); - wrapper.eq("create_user_id", userDTO.getId()); - if (StringUtils.isNotBlank(servingInfoQueryDTO.getName())) { - //按名称搜索时不区分大小写 - wrapper.like("lower(name)", servingInfoQueryDTO.getName().toLowerCase()); - } - Page page = new Page(null == servingInfoQueryDTO.getCurrent() ? NumberConstant.NUMBER_1 : servingInfoQueryDTO.getCurrent(), - null == servingInfoQueryDTO.getSize() ? NumberConstant.NUMBER_10 : servingInfoQueryDTO.getSize()); - try { - //排序字段，默认按更新时间降序，否则将驼峰转换为下划线 - String column = servingInfoQueryDTO.getSort() != null && FILE_NAMES.contains(servingInfoQueryDTO.getSort()) - ? StringUtils.humpToLine(servingInfoQueryDTO.getSort()) : "update_time"; - //排序方式 - boolean isAsc = !StringUtils.isBlank(servingInfoQueryDTO.getOrder()) && !StringUtils.equals(servingInfoQueryDTO.getOrder(), StringConstant.SORT_DESC); - wrapper.orderBy(true, isAsc, column); - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "User queried online service with an exception, request info: {}，exception info: {}", JSONObject.toJSONString(servingInfoQueryDTO), e); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - IPage servingInfos = servingInfoMapper.selectPage(page, wrapper); - List queryVOList = servingInfos.getRecords().stream().map(servingInfo -> { - ServingInfoQueryVO servingInfoQueryVO = new ServingInfoQueryVO(); - BeanUtils.copyProperties(servingInfo, servingInfoQueryVO); - servingInfoQueryVO.setUrl(servingInfo.getUuid() + GATEWAY_URI_POSTFIX); - Map statistics = servingLuaScriptService.countCallsByServingInfoId(servingInfo.getId()); - servingInfoQueryVO.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); - servingInfoQueryVO.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); - return servingInfoQueryVO; - }).collect(Collectors.toList()); - LogUtil.info(LogEnum.SERVING, "User {} queried online service list, online service count = {}", userDTO.getUsername(), queryVOList.size()); - return PageUtil.toPage(page, queryVOList); - } - - /** - * 创建服务 - * - * @param servingInfoCreateDTO 服务创建参数 - * @return ServingInfoCreateVO 服务创建返回对象 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public ServingInfoCreateVO create(ServingInfoCreateDTO servingInfoCreateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - //参数校验 - assert Objects.nonNull(userDTO); - checkNameExist(servingInfoCreateDTO.getName(), userDTO.getId()); - //gRPC不支持灰度发布 - if (NumberConstant.NUMBER_1 == servingInfoCreateDTO.getType() && servingInfoCreateDTO.getModelConfigList().size() > NumberConstant.NUMBER_1) { - throw new BusinessException(ServingErrorEnum.GRPC_PROTOCOL_NOT_SUPPORTED); - } - ServingInfo servingInfo = buildServingInfo(servingInfoCreateDTO); - List modelConfigList = insertServing(servingInfoCreateDTO, userDTO, servingInfo); - //异步部署容器 - deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); - return new ServingInfoCreateVO(servingInfo.getId(), servingInfo.getStatus()); - } - - /** - * 创建服务并保存数据 - * - * @param servingInfoCreateDTO 在线服务信息创建对象 - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息对象 - * @return List 在线服务模型配置信息列表 - */ - @Transactional(rollbackFor = Exception.class) - public List insertServing(ServingInfoCreateDTO servingInfoCreateDTO, UserDTO userDTO, ServingInfo servingInfo) { - int result = servingInfoMapper.insert(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed saving service into into the database. Service name: {}", userDTO.getUsername(), servingInfoCreateDTO.getName()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - return saveModelConfig(servingInfoCreateDTO.getModelConfigList(), servingInfo, userDTO.getUsername()); - } - - - /** - * 校验名称是否存在 - * - * @param name 服务名称 - * @param userId 用户ID - */ - public void checkNameExist(String name, Long userId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ServingInfo::getName, name); - wrapper.eq(ServingInfo::getCreateUserId, userId); - int count = servingInfoMapper.selectCount(wrapper); - if (count > NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.SERVING_NAME_EXIST); - } - } - - /** - * 构建服务信息 - * - * @param servingInfoCreateDTO 服务创建参数 - * @return ServingInfo 服务信息 - */ - public ServingInfo buildServingInfo(ServingInfoCreateDTO servingInfoCreateDTO) { - ServingInfo servingInfo = new ServingInfo(); - BeanUtils.copyProperties(servingInfoCreateDTO, servingInfo); - servingInfo.setUuid(StringUtils.getUUID()); - servingInfo.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - int totalNode = servingInfoCreateDTO.getModelConfigList().stream().mapToInt(ServingModelConfigDTO::getResourcesPoolNode).sum(); - servingInfo.setTotalNode(totalNode); - return servingInfo; - } - - /** - * 校验资源类型 - * - * @param servingModelConfig 模型配置 - */ - public void checkResourceType(ServingModelConfig servingModelConfig) { - // oneflow 暂不支持cpu - if (servingModelConfig.getFrameType() == NumberConstant.NUMBER_1 && servingModelConfig.getResourcesPoolType() == NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.CPU_NOT_SUPPORTED_BY_ONEFLOW); - } - } - - /** - * 保存模型配置对象列表 - * - * @param modelConfigDTOList 模型配置列表 - * @param servingInfo 服务信息 - * @return List 模型配置对象列表 - */ - List saveModelConfig(List modelConfigDTOList, ServingInfo servingInfo, String userName) { - List list = new ArrayList<>(); - //生成4位随机字符串，作为部署id - String deployId = StringUtils.getTimestamp(); - for (ServingModelConfigDTO servingModelConfigDTO : modelConfigDTOList) { - ServingModelConfig servingModelConfig = new ServingModelConfig(); - BeanUtils.copyProperties(servingModelConfigDTO, servingModelConfig); - servingModelConfig.setServingId(servingInfo.getId()); - servingModelConfig.setDeployId(deployId); - PtModelInfo ptModelInfo = ptModelInfoMapper.selectById(servingModelConfig.getModelId()); - //校验框架 - Integer frameType = getFrameType(ptModelInfo, servingModelConfigDTO.getModelAddress()); - servingModelConfig.setFrameType(frameType); - checkResourceType(servingModelConfig); - LambdaQueryWrapper wrapper = new LambdaQueryWrapper(); - wrapper.eq(PtModelBranch::getParentId, servingModelConfigDTO.getModelId()); - wrapper.eq(PtModelBranch::getModelAddress, servingModelConfigDTO.getModelAddress()); - PtModelBranch ptModelBranch = ptModelBranchMapper.selectOne(wrapper); - if (ptModelBranch != null) { - servingModelConfig.setModelVersion(ptModelBranch.getVersionNum()); - } - servingModelConfig.setModelName(ptModelInfo.getName()); - String path = k8sUtil.getAbsoluteNfsPath(servingModelConfig.getModelAddress()); - if (nfsUtil.fileOrDirIsEmpty(path)) { - throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); - } - if (servingModelConfigMapper.insert(servingModelConfig) < NumberConstant.NUMBER_1) { - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - list.add(servingModelConfig); - } - return list; - } - - /** - * 获取模型框架 - * - * @param ptModelInfo 模型信息 - * @param modelAddress 模型路径 - * @return 模型框架 - */ - public Integer getFrameType(PtModelInfo ptModelInfo, String modelAddress) { - String path = k8sUtil.getAbsoluteNfsPath(modelAddress); - if (ptModelInfo == null || nfsUtil.fileOrDirIsEmpty(path)) { - throw new BusinessException(ServingErrorEnum.MODEL_FILE_NOT_EXIST); - } - if (ptModelInfo.getFrameType() > NumberConstant.NUMBER_4) { - throw new BusinessException(ServingErrorEnum.MODEL_FRAME_TYPE_NOT_SUPPORTED); - } - return ptModelInfo.getFrameType(); - } - - /** - * 在线服务修改 - * - * @param servingInfoUpdateDTO 服务对象修改 - * @return ServingInfoUpdateVO 返回修改后的状态 - */ - @Override - @Transactional(rollbackFor = Exception.class) - @DataPermissionMethod(dataType = DatasetTypeEnum.PUBLIC) - public ServingInfoUpdateVO update(ServingInfoUpdateDTO servingInfoUpdateDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(servingInfoUpdateDTO.getId(), userDTO.getId()); - checkRunningNode(servingInfo.getRunningNode()); - //如果修改了任务名，校验新的任务名是否存在 - if (!servingInfo.getName().equals(servingInfoUpdateDTO.getName())) { - checkNameExist(servingInfoUpdateDTO.getName(), userDTO.getId()); - } - servingInfo.setName(servingInfoUpdateDTO.getName()) - .setDescription(servingInfoUpdateDTO.getDescription()) - .setType(servingInfoUpdateDTO.getType()) - .setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()) - .setUpdateTime(DateUtil.getCurrentTimestamp()); - servingInfo.setUuid(StringUtils.getUUID()); - int totalNode = servingInfoUpdateDTO.getModelConfigList().stream().mapToInt(ServingModelConfigDTO::getResourcesPoolNode).sum(); - servingInfo.setTotalNode(totalNode); - Set oldIds = servingModelConfigService.getIdsByServingId(servingInfoUpdateDTO.getId()); - //删除去掉的模型配置信息 - if (CollectionUtils.isNotEmpty(oldIds)) { - if (!servingModelConfigService.removeByIds(oldIds)) { - LogUtil.error(LogEnum.SERVING, "User {} modified online service model config but failed deleting online service model config. Model config ids={}", userDTO.getUsername(), oldIds); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - List oldModelConfigList = servingModelConfigService.listByIds(oldIds); - deployServingAsyncTask.deleteServing(userDTO, servingInfo, oldModelConfigList); - //删除路由信息 - if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - this.notifyUpdateServingRoute(Collections.emptyList(), oldModelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); - } - } - List modelConfigList = updateServing(servingInfoUpdateDTO, userDTO, servingInfo); - //异步部署容器 - deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); - return new ServingInfoUpdateVO(servingInfo.getId(), servingInfo.getStatus()); - } - - /** - * 修改服务并保存数据 - * - * @param servingInfoUpdateDTO 服务对象修改 - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - * @return List 模型配置对象列表 - */ - @Transactional(rollbackFor = Exception.class) - public List updateServing(ServingInfoUpdateDTO servingInfoUpdateDTO, UserDTO userDTO, ServingInfo servingInfo) { - int result = servingInfoMapper.updateById(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed deleting online service from the database, service id={}", userDTO.getUsername(), servingInfo.getId()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - return saveModelConfig(servingInfoUpdateDTO.getModelConfigList(), servingInfo, userDTO.getUsername()); - } - - /** - * 校验是否有节点在运行 - * - * @param runningNode 运行节点数 - */ - void checkRunningNode(Integer runningNode) { - if (runningNode != null && runningNode > NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.OPERATION_NOT_ALLOWED); - } - } - - /** - * 校验服务信息是否存在 - * - * @param id 服务id - * @param userId 用户id - * @return ServingInfo 服务信息对象 - */ - ServingInfo checkServingInfoExist(Long id, Long userId) { - ServingInfo servingInfo = servingInfoMapper.selectById(id); - //校验服务是否存在，或是否为当前用户创建 - if (servingInfo == null || (!userId.equals(servingInfo.getCreateUserId()))) { - throw new BusinessException(ServingErrorEnum.SERVING_INFO_ABSENT); - } - return servingInfo; - } - - /** - * 在线服务删除 - * - * @param servingInfoDeleteDTO 服务对象删除 - * @return ServingInfoDeleteVO 返回删除对象的id - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ServingInfoDeleteVO delete(ServingInfoDeleteDTO servingInfoDeleteDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(servingInfoDeleteDTO.getId(), userDTO.getId()); - List modelConfigList = getModelConfigByServingId(servingInfo.getId()); - deleteServing(servingInfoDeleteDTO, userDTO, servingInfo); - Map map = new HashMap<>(NumberConstant.NUMBER_2); - map.put("serving_id", servingInfo.getId()); - if (!servingModelConfigService.removeByMap(map)) { - LogUtil.error(LogEnum.SERVING, "User {} failed update online service in the database, service id={}, service name:{}", - userDTO.getUsername(), servingInfoDeleteDTO.getId(), servingInfo.getName()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - deployServingAsyncTask.deleteServing(userDTO, servingInfo, modelConfigList); - //删除路由信息 - if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - this.notifyUpdateServingRoute(Collections.emptyList(), modelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); - } - return new ServingInfoDeleteVO(servingInfo.getId()); - } - - /** - * 删除服务并保存数据 - * - * @param servingInfoDeleteDTO 服务信息删除对象 - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - */ - @Transactional(rollbackFor = Exception.class) - public void deleteServing(ServingInfoDeleteDTO servingInfoDeleteDTO, UserDTO userDTO, ServingInfo servingInfo) { - int result = servingInfoMapper.deleteById(servingInfo.getId()); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed deleting online service from the database, service id={}, service name:{}", - userDTO.getUsername(), servingInfoDeleteDTO.getId(), servingInfo.getName()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 在线服务获取详情 - * - * @param servingInfoDetailDTO 获取服务详情参数 - * @return ServingInfoDetailVO 返回详情对象 - */ - @Override - public ServingInfoDetailVO getDetail(ServingInfoDetailDTO servingInfoDetailDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(servingInfoDetailDTO.getId(), userDTO.getId()); - Map map = new HashMap<>(NumberConstant.NUMBER_2); - map.put("serving_id", servingInfo.getId()); - List servingModelConfigList = servingModelConfigService.listByMap(map); - ServingInfoDetailVO servingInfoDetailVO = new ServingInfoDetailVO(); - BeanUtils.copyProperties(servingInfo, servingInfoDetailVO); - List servingModelConfigVOList = new ArrayList<>(); - List configIdList = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(servingModelConfigList)) { - for (ServingModelConfig servingModelConfig : servingModelConfigList) { - ServingModelConfigVO vo = new ServingModelConfigVO(); - BeanUtils.copyProperties(servingModelConfig, vo); - servingModelConfigVOList.add(vo); - configIdList.add(servingModelConfig.getId()); - } - } - servingInfoDetailVO.setModelConfigList(servingModelConfigVOList); - // 获取调用次数 - Map countCalls = servingLuaScriptService.countCalls(configIdList); - servingInfoDetailVO.setTotalNum(countCalls.getOrDefault("callCount", SymbolConstant.ZERO)); - servingInfoDetailVO.setFailNum(countCalls.getOrDefault("failedCount", SymbolConstant.ZERO)); - return servingInfoDetailVO; - } - - /** - * 启动在线服务 - * - * @param servingStartDTO 启动服务参数 - * @return ServingStartVO 返回启动后对象 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ServingStartVO start(ServingStartDTO servingStartDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(servingStartDTO.getId(), userDTO.getId()); - checkRunningNode(servingInfo.getRunningNode()); - List modelConfigList = getModelConfigByServingId(servingInfo.getId()); - servingInfo.setUuid(StringUtils.getUUID()); - servingInfo.setStatus(ServingStatusEnum.IN_DEPLOYMENT.getStatus()); - updateServingStart(userDTO, servingInfo); - //异步部署容器 - deployServingAsyncTask.deployServing(userDTO, servingInfo, modelConfigList); - return new ServingStartVO(servingInfo.getId(), servingInfo.getStatus()); - } - - /** - * 启动服务并保存数据 - * - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - */ - @Transactional(rollbackFor = Exception.class) - public void updateServingStart(UserDTO userDTO, ServingInfo servingInfo) { - int result = servingInfoMapper.updateById(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed update online service in the database, service id={}", userDTO.getUsername(), servingInfo.getId()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 在线服务推理 - * - * @param id 预测服务ID - * @param url 预测地址 - * @param files 需要预测的图片文件 - * @return String 返回推理结果 - */ - @Override - public String predict(Long id, String url, MultipartFile[] files) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(id, userDTO.getId()); - //校验服务状态是否为运行中 - if (servingInfo.getRunningNode() == NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.SERVING_NOT_WORKING); - } - if (StringUtils.isBlank(url)) { - throw new BusinessException(ServingErrorEnum.SERVING_NOT_WORKING); - } - //校验推理图片 - if (files == null || files.length == NumberConstant.NUMBER_0) { - throw new BusinessException(ServingErrorEnum.PREDICT_IMAGE_EMPTY); - } - List imageInfoList = new ArrayList<>(); - for (MultipartFile file : files) { - String fileName = file.getOriginalFilename(); - //获取文件后缀名 - assert Objects.nonNull(fileName); - int begin = fileName.lastIndexOf(SymbolConstant.DOT); - String suffix = fileName.substring(begin).toLowerCase(); - if (!ServingConstant.IMAGE_FORMAT.contains(suffix)) { - throw new BusinessException(ServingErrorEnum.IMAGE_FORMAT_ERROR); - } - String base64File; - try { - base64File = Base64.encodeBase64String(file.getBytes()); - } catch (Exception e) { - throw new BusinessException(ServingErrorEnum.IMAGE_CONVERT_BASE64_FAIL); - } - if (StringUtils.isNotBlank(base64File)) { - ImageInfo imageInfo = new ImageInfo(); - imageInfo.setImageName(fileName); - imageInfo.setImageFile(base64File); - imageInfoList.add(imageInfo); - } - } - if (imageInfoList.isEmpty()) { - throw new BusinessException(ServingErrorEnum.IMAGE_FORMAT_ERROR); - } - ManagedChannel channel = GrpcClient.getChannel(id, url); - return GrpcClient.getResult(channel, imageInfoList).getJsonResult(); - } - - /** - * 在线服务停止 - * - * @param servingStopDTO 停止服务参数 - * @return ServingStopVO 返回停止后对象 - */ - @Override - @Transactional(rollbackFor = Exception.class) - public ServingStopVO stop(ServingStopDTO servingStopDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(servingStopDTO.getId(), userDTO.getId()); - //修改状态及可用节点数 - servingInfo.setStatus(ServingStatusEnum.STOP.getStatus()); - servingInfo.setRunningNode(NumberConstant.NUMBER_0); - List modelConfigList = getModelConfigByServingId(servingInfo.getId()); - updateServingStop(userDTO, servingInfo, modelConfigList); - deployServingAsyncTask.deleteServing(userDTO, servingInfo, modelConfigList); - //删除路由信息 - if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - this.notifyUpdateServingRoute(Collections.emptyList(), modelConfigList.stream().map(ServingModelConfig::getId).collect(Collectors.toList())); - } - return new ServingStopVO(servingInfo.getId(), servingInfo.getStatus()); - } - - /** - * 停止服务并保存数据 - * - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - * @param modelConfigList 模型配置列表 - */ - @Transactional(rollbackFor = Exception.class) - public void updateServingStop(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { - int result = servingInfoMapper.updateById(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed stopping the online service and failed to update the service in the database. Service id={}, service name:{}, running node number:{}", - userDTO.getUsername(), servingInfo.getId(), servingInfo.getName(), servingInfo.getRunningNode()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - modelConfigList.forEach(servingModelConfig -> { - servingModelConfig.setReadyReplicas(NumberConstant.NUMBER_0); - servingModelConfigMapper.updateById(servingModelConfig); - }); - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - GrpcClient.shutdownChannel(servingInfo.getId(), userDTO); - } - } - - /** - * 获取推理参数 - * - * @param predictParamDTO 获取预测参数服务 - * @return PredictParamVO 返回推理参数对象 - */ - @Override - public PredictParamVO getPredictParam(PredictParamDTO predictParamDTO) { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(userDTO); - ServingInfo servingInfo = checkServingInfoExist(predictParamDTO.getId(), userDTO.getId()); - PredictParamVO predictParamVO = new PredictParamVO(); - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ServingModelConfig::getServingId, predictParamDTO.getId()); - List servingModelConfigList = servingModelConfigService.list(wrapper); - if (CollectionUtils.isEmpty(servingModelConfigList)) { - return predictParamVO; - } - //grpc协议 - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - predictParamVO.setRequestMethod("gRPC"); - String url = servingModelConfigList.get(0).getUrl(); - predictParamVO.setUrl(url); - Map inputs = new HashMap<>(); - inputs.put("DataRequest", "List"); - predictParamVO.setInputs(inputs); - Map> other = new HashMap<>(); - Map image = new HashMap<>(); - image.put("image_file", "String"); - image.put("image_name", "String"); - other.put("Image", image); - Map outputs = new HashMap<>(); - outputs.put("DataResponse", "String"); - predictParamVO.setOutputs(outputs); - predictParamVO.setOther(other); - } else if (ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - String url = "http://" + servingInfo.getUuid() + GATEWAY_URI_POSTFIX + ServingConstant.INFERENCE_INTERFACE_NAME; - predictParamVO.setUrl(url); - Map inputs = new HashMap<>(); - inputs.put("image_files", "File"); - predictParamVO.setInputs(inputs); - - Map outputs = new HashMap<>(); - outputs.put("label", "String"); - outputs.put("probability", "Float"); - predictParamVO.setOutputs(outputs); - predictParamVO.setRequestMethod(SymbolConstant.POST); - } else { - throw new BusinessException(ServingErrorEnum.PROTOCOL_NOT_SUPPORTED); - } - return predictParamVO; - } - - /** - * 根据servingId获取模型配置列表 - * - * @param servingId 在线服务id - * @return List 模型配置列表 - */ - public List getModelConfigByServingId(Long servingId) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(ServingModelConfig::getServingId, servingId); - return servingModelConfigService.list(wrapper); - } - - /** - * 获取容器监控信息 - * - * @param id 模型部署信息id - * @return ServingMetricsVO 返回监控信息对象 - */ - @Override - public ServingMetricsVO getMetricsDetail(Long id) { - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - ServingInfoDetailDTO queryDTO = new ServingInfoDetailDTO(); - queryDTO.setId(id); - ServingInfoDetailVO detail = this.getDetail(queryDTO); - List configMetricsVOS = new ArrayList<>(); - for (ServingModelConfigVO servingModelConfigVO : detail.getModelConfigList()) { - // 获取id、模型等信息 - ServingConfigMetricsVO metricsTemp = new ServingConfigMetricsVO(); - BeanUtil.copyProperties(servingModelConfigVO, metricsTemp); - // 获取实时监控数据 - assert Objects.nonNull(user); - List podsVOS = metricsApi.getPodMetricsRealTime(k8sUtil.getNamespace(user), k8sUtil.getResourceName(servingModelConfigVO.getResourceInfo())); - if (CollectionUtils.isNotEmpty(podsVOS)) { - List podMetricsVOS = podsVOS.stream().map(podsVO -> { - ServingPodMetricsVO servingPodMetricsVO = new ServingPodMetricsVO(); - BeanUtil.copyProperties(podsVO, servingPodMetricsVO); - servingPodMetricsVO.setGrafanaUrl(k8sPodMetricsGrafanaUrl.concat(podsVO.getPodName())); - return servingPodMetricsVO; - }).collect(Collectors.toList()); - metricsTemp.setPodList(podMetricsVOS); - } else { - metricsTemp.setPodList(Collections.emptyList()); - } - // 接口调用信息 - Map statistics = servingLuaScriptService.countCallsByServingConfigId(servingModelConfigVO.getId()); - metricsTemp.setTotalNum(statistics.getOrDefault("callCount", SymbolConstant.ZERO)); - metricsTemp.setFailNum(statistics.getOrDefault("failedCount", SymbolConstant.ZERO)); - configMetricsVOS.add(metricsTemp); - } - return new ServingMetricsVO().setId(id).setServingConfigList(configMetricsVOS); - } - - /** - * 新增和删除在线服务路由通知 - * - * @param saveIdList 新增的路由ID列表 - * @param deleteIdList 删除的路由ID列表 - */ - @Override - public void notifyUpdateServingRoute(List saveIdList, List deleteIdList) { - String message = StringUtils.EMPTY; - // 处理新增路由消息 - if (CollectionUtils.isNotEmpty(saveIdList)) { - String idString = StringUtils.join(saveIdList.toArray(), SymbolConstant.COMMA); - message = ServingRouteEventEnum.SAVE.getCode() + SymbolConstant.COLON + idString + SymbolConstant.EVENT_SEPARATOR; - LogUtil.info(LogEnum.SERVING, "Serving start success and seed notify {}", message); - } - // 处理删除路由消息 - if (CollectionUtils.isNotEmpty(deleteIdList)) { - String idString = StringUtils.join(deleteIdList.toArray(), SymbolConstant.COMMA); - message = message + ServingRouteEventEnum.DELETE.getCode() + SymbolConstant.COLON + idString; - LogUtil.info(LogEnum.SERVING, "Serving stop success and seed notify {}", message); - } - if (StringUtils.isNotBlank(message)) { - LogUtil.info(LogEnum.SERVING, "Start send message to stream with notify {}", message); - StringRecord stringRecord = StreamRecords.string(Collections.singletonMap(servingGroup, message)).withStreamKey(ServingConstant.SERVING_STREAM); - stringRedisTemplate.opsForStream().add(stringRecord); - } - } - - /** - * 获取在线服务所有POD - * - * @param id 服务配置id - * @return List 返回POD列表 - */ - @Override - public List getPods(Long id) { - ServingModelConfig modelConfig = servingModelConfigService.getById(id); - if (modelConfig == null) { - return Collections.emptyList(); - } - //从会话中获取用户信息 - UserDTO user = JwtUtils.getCurrentUserDto(); - assert Objects.nonNull(user); - String nameSpace = k8sUtil.getNamespace(user); - List list = podService.getPods(new PodQueryDTO(nameSpace, k8sUtil.getResourceName(modelConfig.getResourceInfo()))); - list.sort((pod1, pod2) -> { - Integer pod1Index = Integer.parseInt(pod1.getDisplayName().substring(NumberConstant.NUMBER_3)); - Integer pod2Index = Integer.parseInt(pod2.getDisplayName().substring(NumberConstant.NUMBER_3)); - return pod1Index - pod2Index; - }); - return list; - } - - /** - * 在线服务回调 - * - * @param times 回调请求次数 - * @param req 回调请求对象 - * @return boolean 返回是否回调成功 - */ - @Override - public boolean servingCallback(int times, ServingK8sDeploymentCallbackCreateDTO req) { - // 根据namespace和podName找到模型配置 - String resourceInfo = k8sNameTool.getResourceInfoFromResourceName(BizEnum.SERVING, req.getResourceName()); - if (StringUtils.isBlank(resourceInfo)) { - LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig ID! Request: {}", Thread.currentThread(), times, req.toString()); - return false; - } - String idStr = resourceInfo.substring(NumberConstant.NUMBER_4); - ServingModelConfig servingModelConfig = servingModelConfigService.getById(Long.parseLong(idStr)); - if (Objects.isNull(servingModelConfig)) { - LogUtil.warn(LogEnum.SERVING, "Cannot find modelConfig! Request: {}", Thread.currentThread(), times, req.toString()); - return false; - } - ServingInfo servingInfo = servingInfoMapper.selectById(servingModelConfig.getServingId()); - if (Objects.isNull(servingInfo)) { - LogUtil.warn(LogEnum.SERVING, "Cannot find servingInfo! Request: {}", Thread.currentThread(), times, req.toString()); - return false; - } - //不处理已停止服务的回调 - if (ServingStatusEnum.STOP.getStatus().equals(servingInfo.getStatus())) { - return false; - } - //更新信息 - if (updateByCallback(req, servingModelConfig, servingInfo)) { - return false; - } - //增加发送路由信息 - if (req.getReadyReplicas() > NumberConstant.NUMBER_0 && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - this.notifyUpdateServingRoute(Collections.singletonList(servingModelConfig.getId()), Collections.emptyList()); - } - //增加删除路由信息 - if (req.getReadyReplicas() == NumberConstant.NUMBER_0 && ServingTypeEnum.HTTP.getType().equals(servingInfo.getType())) { - this.notifyUpdateServingRoute(Collections.emptyList(), Collections.singletonList(servingModelConfig.getId())); - } - return true; - } - - @Transactional(rollbackFor = Exception.class) - public boolean updateByCallback(ServingK8sDeploymentCallbackCreateDTO req, ServingModelConfig servingModelConfig, ServingInfo servingInfo) { - //更新当前模型配置有效节点数 - servingModelConfig.setReadyReplicas(req.getReadyReplicas()); - int result = servingModelConfigMapper.updateById(servingModelConfig); - if (result < NumberConstant.NUMBER_1) { - return true; - } - //更新当前服务有效总节点数 - ServingModelConfig another = servingModelConfigMapper.selectAnother(servingInfo.getId(), servingModelConfig.getId()); - if (Objects.nonNull(another)) { - servingInfo.setRunningNode(req.getReadyReplicas() + another.getReadyReplicas()); - } else { - servingInfo.setRunningNode(req.getReadyReplicas()); - } - //运行中的服务回调运行节点数为0则说明服务运行失败 - if (ServingStatusEnum.WORKING.getStatus().equals(servingInfo.getStatus()) && servingInfo.getRunningNode() == NumberConstant.NUMBER_0) { - servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - } - if (servingInfo.getRunningNode() > NumberConstant.NUMBER_0) { - servingInfo.setStatus(ServingStatusEnum.WORKING.getStatus()); - } - servingInfo.setUpdateTime(DateUtil.getCurrentTimestamp()); - return servingInfoMapper.updateById(servingInfo) < NumberConstant.NUMBER_1; - } - - /** - * 获取在线服务回滚信息列表 - * - * @param servingId 在线服务id - * @return Map> 返回回滚信息列表 - */ - @Override - public Map> getRollbackList(Long servingId) { - List servingModelConfigList = servingModelConfigMapper.getRollbackList(servingId); - Map> map = new HashMap<>(); - if (CollectionUtils.isEmpty(servingModelConfigList)) { - return map; - } - servingModelConfigList.forEach(servingModelConfig -> { - if (!map.containsKey(servingModelConfig.getDeployId())) { - ServingModelConfigVO servingModelConfigVO = new ServingModelConfigVO(); - BeanUtils.copyProperties(servingModelConfig, servingModelConfigVO); - List list = new ArrayList<>(); - list.add(servingModelConfigVO); - map.put(servingModelConfig.getDeployId(), list); - } else { - ServingModelConfigVO servingModelConfigVO = new ServingModelConfigVO(); - BeanUtils.copyProperties(servingModelConfig, servingModelConfigVO); - List list = map.get(servingModelConfig.getDeployId()); - list.add(servingModelConfigVO); - } - }); - return map; - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java deleted file mode 100644 index 78ffbc4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/task/DeployServingAsyncTask.java +++ /dev/null @@ -1,391 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.task; - -import org.dubhe.constant.NumberConstant; -import org.dubhe.constant.ServingConstant; -import org.dubhe.dao.BatchServingMapper; -import org.dubhe.dao.ServingInfoMapper; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.BatchServing; -import org.dubhe.domain.entity.ServingInfo; -import org.dubhe.domain.entity.ServingModelConfig; -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.enums.ResourcesPoolTypeEnum; -import org.dubhe.enums.ServingErrorEnum; -import org.dubhe.enums.ServingFrameTypeEnum; -import org.dubhe.enums.ServingStatusEnum; -import org.dubhe.enums.ServingTypeEnum; -import org.dubhe.exception.BaseErrorCode; -import org.dubhe.exception.BusinessException; -import org.dubhe.k8s.api.DistributeTrainApi; -import org.dubhe.k8s.api.ModelServingApi; -import org.dubhe.k8s.api.TrainJobApi; -import org.dubhe.k8s.domain.PtBaseResult; -import org.dubhe.k8s.domain.bo.DistributeTrainBO; -import org.dubhe.k8s.domain.bo.ModelServingBO; -import org.dubhe.k8s.domain.bo.PtJupyterJobBO; -import org.dubhe.k8s.domain.bo.PtMountDirBO; -import org.dubhe.k8s.domain.resource.BizDistributeTrain; -import org.dubhe.k8s.domain.resource.BizIngressRule; -import org.dubhe.k8s.domain.vo.ModelServingVO; -import org.dubhe.k8s.domain.vo.PtJupyterJobVO; -import org.dubhe.service.MailService; -import org.dubhe.service.ServingModelConfigService; -import org.dubhe.utils.GrpcClient; -import org.dubhe.utils.K8sNameTool; -import org.dubhe.utils.K8sUtil; -import org.dubhe.utils.LogUtil; -import org.dubhe.utils.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; - -import javax.annotation.Resource; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -/** - * @description 部署服务异步处理 - * @date 2020-09-02 - */ -@Component -public class DeployServingAsyncTask { - - @Resource - private ServingInfoMapper servingInfoMapper; - - @Resource - private BatchServingMapper batchServingMapper; - - @Resource - private ServingModelConfigService servingModelConfigService; - - @Resource - private ModelServingApi modelServingApi; - - @Resource - private K8sUtil k8sUtil; - - @Resource - private K8sNameTool k8sNameTool; - - @Resource - private DistributeTrainApi distributeTrainApi; - - @Resource - private TrainJobApi trainJobApi; - - @Resource - private MailService mailService; - - /** - * Service源码路径 - */ - @Value("${serving.sourcePath}") - private String sourcePath; - - @Value("${serving.gpu-image}") - private String gpuImage; - - @Value("${serving.cpu-image}") - private String cpuImage; - - private final static String TRUE = "True"; - - private final static String FALSE = "False"; - - /** - * 创建pod，部署模型 - * - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - * @param modelConfigList 在线服务模型部署信息集合 - */ - @Async("servingExecutor") - @Transactional(rollbackFor = Exception.class) - public void deployServing(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { - boolean flag = false; - for (ServingModelConfig servingModelConfig : modelConfigList) { - try { - ModelServingBO bo = buildModelServingBO(userDTO, servingInfo, servingModelConfig); - //创建pod - ModelServingVO modelServingVO = modelServingApi.create(bo); - if (ServingConstant.SUCCESS_CODE.equals(modelServingVO.getCode())) { - // 获取pod对应的url，并修改模型部署状态 - List rules = modelServingVO.getBizIngress().getRules(); - //取第一个url - String url = rules.get(NumberConstant.NUMBER_0).getHost(); - if (StringUtils.isNotBlank(url)) { - servingModelConfig.setUrl(url); - if (servingModelConfigService.updateById(servingModelConfig)) { - flag = true; - } else { - LogUtil.error(LogEnum.SERVING, "User {} failed saving online service model config. Database update FAILED, service id={}", userDTO.getUsername(), servingModelConfig.getId()); - } - } - } - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "User {} create serving failed.The name of serving is {}", userDTO.getUsername(), servingInfo.getName(), e); - } - } - - //修改服务状态 - if (!flag) { - servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - } - //grpc协议，创建对应的通道 - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - String url = modelConfigList.get(0).getUrl(); - if (StringUtils.isNotBlank(url)) { - GrpcClient.createChannel(servingInfo.getId(), url, userDTO); - } - } - int result = servingInfoMapper.updateById(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} failed to update the servingInfo table.The name of serving is {}", userDTO.getUsername(), servingInfo.getName()); - } - } - - /** - * 构建模型部署参数 - * - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - * @param servingModelConfig 在线服务模型部署信息 - * @return ModelServingBO 返回构建后对象 - */ - private ModelServingBO buildModelServingBO(UserDTO userDTO, ServingInfo servingInfo, ServingModelConfig servingModelConfig) { - ModelServingBO bo = new ModelServingBO(); - //容器端口 - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - bo.setGrpcPort(ServingConstant.POD_GRPC_PORT); - } else { - bo.setHttpPort(ServingConstant.POD_HTTP_PORT); - } - //推理脚本名称，不同协议采用不同的推理脚本 - String scriptName = ServingConstant.HTTP_SCRIPT; - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - scriptName = ServingConstant.GRPC_SCRIPT; - } - - // 构建参数 - String command = String.format(ServingConstant.SERVING_COMMAND, ServingConstant.TS_SERVING_PATH, scriptName, ServingFrameTypeEnum.getFrameName(servingModelConfig.getFrameType()), - ServingConstant.MODEL_PATH, ResourcesPoolTypeEnum.isGpuCode(servingModelConfig.getResourcesPoolType()) ? TRUE : FALSE) + servingModelConfig.getDeployParam(); - String resourceInfo = StringUtils.getRandomString() + servingModelConfig.getId(); - servingModelConfig.setResourceInfo(resourceInfo); - bo.setNamespace(k8sUtil.getNamespace(userDTO)) - .setResourceName(k8sUtil.getResourceName(resourceInfo)) - .setReplicas(servingModelConfig.getResourcesPoolNode()) - .setGpuNum(servingModelConfig.getGpuNum()) - .setCpuNum(servingModelConfig.getCpuNum()) - .setMemNum(servingModelConfig.getMemNum()) - .setImage(ResourcesPoolTypeEnum.isGpuCode(servingModelConfig.getResourcesPoolType()) ? gpuImage : cpuImage) - .setCmdLines(Arrays.asList("-c", command)); - bo.setNfsMounts(new HashMap(NumberConstant.NUMBER_4) {{ - put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(servingModelConfig.getModelAddress()))); - put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); - }}); - bo.setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.SERVING)); - return bo; - } - - /** - * 删除pod - * - * @param userDTO 用户信息 - * @param servingInfo 在线服务信息 - * @param modelConfigList 在线服务模型部署信息集合 - */ - @Async("servingExecutor") - @Transactional(rollbackFor = Exception.class) - public void deleteServing(UserDTO userDTO, ServingInfo servingInfo, List modelConfigList) { - boolean flag = true; - for (ServingModelConfig servingModelConfig : modelConfigList) { - String namespace = k8sUtil.getNamespace(userDTO); - String resourceName = k8sUtil.getResourceName(servingModelConfig.getResourceInfo()); - PtBaseResult ptBaseResult = modelServingApi.delete(namespace, resourceName); - if (!ServingConstant.SUCCESS_CODE.equals(ptBaseResult.getCode())) { - flag = false; - } - } - if (!flag) { - servingInfo.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - LogUtil.error(LogEnum.SERVING, "An Exception occurred when user {} stopping the service, service name:{}", userDTO.getUsername(), servingInfo.getName()); - } - LogUtil.info(LogEnum.SERVING, "User {} stopped the service with SUCCESS, service name:{}", userDTO.getUsername(), servingInfo.getName()); - //grpc协议关闭对应通道 - if (ServingTypeEnum.GRPC.getType().equals(servingInfo.getType())) { - GrpcClient.shutdownChannel(servingInfo.getId(), userDTO); - } - int result = servingInfoMapper.updateById(servingInfo); - if (result < NumberConstant.NUMBER_1) { - LogUtil.error(LogEnum.SERVING, "User {} FAILED stopping the online service. Database update FAILED. Service id={}, service name:{}，service status:{}", userDTO.getUsername(), servingInfo.getId(), servingInfo.getName(), servingInfo.getStatus()); - throw new BusinessException(ServingErrorEnum.INTERNAL_SERVER_ERROR); - } - } - - /** - * 异步部署批量服务 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - */ - @Async("servingExecutor") - @Transactional(rollbackFor = Exception.class) - public void deployBatchServing(UserDTO userDTO, BatchServing batchServing) { - if (batchServing.getResourcesPoolNode() == NumberConstant.NUMBER_1) { - //单节点 - PtJupyterJobBO ptJupyterJobBO = buildJobBo(userDTO, batchServing); - if (ptJupyterJobBO != null) { - PtJupyterJobVO vo = trainJobApi.create(ptJupyterJobBO); - if (vo.isSuccess()) { - LogUtil.info(LogEnum.SERVING, "User {} deployed batching service with SUCCESS. Service name：{}", userDTO.getUsername(), batchServing.getName()); - batchServingMapper.updateById(batchServing); - return; - } - } - - } else { - //多节点分布式 - DistributeTrainBO distributeTrainBO = buildDistributeTrainBO(userDTO, batchServing); - if (distributeTrainBO != null) { - BizDistributeTrain distribute = distributeTrainApi.create(distributeTrainBO); - if (distribute.isSuccess()) { - LogUtil.info(LogEnum.SERVING, "User {} deployed batching service with SUCCESS. Service name：{}", userDTO.getUsername(), batchServing.getName()); - batchServingMapper.updateById(batchServing); - return; - } - } - } - batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - batchServingMapper.updateById(batchServing); - LogUtil.error(LogEnum.SERVING, "User {} FAILED deployed batching service. Service name：{}", userDTO.getUsername(), batchServing.getName()); - } - - /** - * 异步删除批量服务 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - */ - @Async("servingExecutor") - @Transactional(rollbackFor = Exception.class) - public void deleteBatchServing(UserDTO userDTO, BatchServing batchServing) { - String namespace = k8sUtil.getNamespace(userDTO); - String resourceName = k8sUtil.getBatchResourceName(batchServing.getResourceInfo()); - if (batchServing.getResourcesPoolNode() == NumberConstant.NUMBER_1) { - if (trainJobApi.delete(namespace, resourceName)) { - return; - } - } else { - PtBaseResult result = distributeTrainApi.deleteByResourceName(namespace, resourceName); - if (ServingConstant.SUCCESS_CODE.equals(result.getCode())) { - return; - } - } - batchServing.setStatus(ServingStatusEnum.EXCEPTION.getStatus()); - batchServingMapper.updateById(batchServing); - LogUtil.error(LogEnum.SERVING, "User {} FAILED deployed batching service. Service name：{}", userDTO.getUsername(), batchServing.getName()); - } - - /** - * 构建单节点推理参数 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - * @return PtJupyterJobBO 返回构建后对象 - */ - public PtJupyterJobBO buildJobBo(UserDTO userDTO, BatchServing batchServing) { - String command = String.format(ServingConstant.BATCH_COMMAND, ServingConstant.TS_SERVING_PATH, ServingFrameTypeEnum.getFrameName(batchServing.getFrameType()), - ServingConstant.MODEL_PATH, ServingConstant.INPUT_PATH, ServingConstant.OUTPUT_PATH, FALSE, ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? TRUE : FALSE) + batchServing.getDeployParam(); - String resourceInfo = StringUtils.getRandomString() + batchServing.getId(); - batchServing.setResourceInfo(resourceInfo); - PtJupyterJobBO bo = new PtJupyterJobBO() - .setNamespace(k8sUtil.getNamespace(userDTO)) - .setName(k8sUtil.getBatchResourceName(resourceInfo)) - .setCpuNum(batchServing.getCpuNum()) - .setGpuNum(batchServing.getGpuNum()) - .setUseGpu(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType())) - .setMemNum(batchServing.getMemNum()) - .setCmdLines(Arrays.asList("-c", command)) - .setNfsMounts(new HashMap(NumberConstant.NUMBER_6) {{ - put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getModelAddress()))); - put(ServingConstant.INPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getInputPath()))); - put(ServingConstant.OUTPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getOutputPath()))); - put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); - }}) - .setImage(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? gpuImage : cpuImage) - .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.BATCH_SERVING)); - return bo; - } - - /** - * 构建分布式推理参数 - * - * @param userDTO 用户信息 - * @param batchServing 批量服务信息 - * @return DistributeTrainBO 返回构建后对象 - */ - public DistributeTrainBO buildDistributeTrainBO(UserDTO userDTO, BatchServing batchServing) { - String command = String.format(ServingConstant.BATCH_COMMAND, ServingConstant.TS_SERVING_PATH, ServingFrameTypeEnum.getFrameName(batchServing.getFrameType()), - ServingConstant.MODEL_PATH, ServingConstant.INPUT_PATH, ServingConstant.OUTPUT_PATH, TRUE, ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? TRUE : FALSE) + batchServing.getDeployParam(); - String resourceInfo = StringUtils.getRandomString() + batchServing.getId(); - batchServing.setResourceInfo(resourceInfo); - DistributeTrainBO bo = new DistributeTrainBO() - .setName(k8sUtil.getBatchResourceName(resourceInfo)) - .setNamespace(k8sUtil.getNamespace(userDTO)) - .setSize(batchServing.getResourcesPoolNode()) - .setImage(ResourcesPoolTypeEnum.isGpuCode(batchServing.getResourcesPoolType()) ? gpuImage : cpuImage) - .setMemNum(batchServing.getMemNum()) - .setCpuNum(batchServing.getCpuNum()) - .setGpuNum(batchServing.getGpuNum()) - .setMasterCmd(command) - .setSlaveCmd(command) - .setBusinessLabel(k8sNameTool.getPodLabel(BizEnum.BATCH_SERVING)); - bo.setNfsMounts(new HashMap(NumberConstant.NUMBER_6) {{ - put(ServingConstant.MODEL_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getModelAddress()))); - put(ServingConstant.INPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getInputPath()))); - put(ServingConstant.OUTPUT_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(batchServing.getOutputPath()))); - put(ServingConstant.TS_SERVING_PATH, new PtMountDirBO(k8sUtil.getAbsoluteNfsPath(sourcePath))); - }}); - return bo; - } - - /** - * 批量服务推理成功异步发送邮件通知 - * - * @param receiverMailAddress 邮件地址 - * @param id 批量服务ID - */ - @Async("servingExecutor") - public void asyncSendServingMail(String receiverMailAddress, Long id) { - try { - final StringBuffer sb = new StringBuffer(); - sb.append("

" + "亲爱的").append(receiverMailAddress).append("您好！

"); - sb.append("

ID为：" + id + "的批量服务推理成功，请及时查看！

"); - mailService.sendHtmlMail(receiverMailAddress, "推理成功通知", sb.toString()); - } catch (Exception e) { - LogUtil.error(LogEnum.SERVING, "UserServiceImpl sendMail error , param:{} error:{}", receiverMailAddress, e); - throw new BusinessException(BaseErrorCode.ERROR_SYSTEM.getCode(), BaseErrorCode.ERROR_SYSTEM.getMsg()); - } - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java deleted file mode 100644 index 4d90455..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/ImageUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import org.dubhe.base.BaseImageDTO; -import org.dubhe.dao.PtImageMapper; -import org.dubhe.domain.dto.UserDTO; -import org.dubhe.domain.entity.PtImage; -import org.dubhe.enums.ImageStateEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.exception.BusinessException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * @description 镜像 - * @date 2020-06-22 - */ -@Component -public class ImageUtil { - - @Autowired - private PtImageMapper ptImageMapper; - - /** - * 获取镜像url - * - * @param baseImageDTO 镜像参数 - * @return BaseImageDTO 镜像url - **/ - public String getImageUrl(BaseImageDTO baseImageDTO, UserDTO user) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("image_name", baseImageDTO.getImageName()).eq("image_tag", baseImageDTO.getImageTag()) - .eq("image_status", ImageStateEnum.SUCCESS.getCode()).last(" limit 1 "); - PtImage ptImage = ptImageMapper.selectOne(queryWrapper); - if (ptImage == null || StringUtils.isBlank(ptImage.getImageUrl())) { - LogUtil.error(LogEnum.BIZ_TRAIN, " User {} gets image ,the imageName is {}, the imageTag is {}, and the result of query image table (PT_image) is empty", user.getUsername(), baseImageDTO.getImageName(), baseImageDTO.getImageTag()); - throw new BusinessException("镜像不存在"); - } - return ptImage.getImageUrl(); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java deleted file mode 100644 index 6b14dd4..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/NotebookUtil.java +++ /dev/null @@ -1,244 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import cn.hutool.core.date.DateField; -import cn.hutool.core.date.DatePattern; -import cn.hutool.core.date.DateUtil; -import cn.hutool.core.util.RandomUtil; -import org.dubhe.constant.SymbolConstant; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.enums.LogEnum; -import org.dubhe.k8s.domain.PtBaseResult; -import org.apache.shiro.UnavailableSecurityManagerException; -import org.dubhe.domain.dto.UserDTO; - -import java.util.Date; - -/** - * @description Notebook 工具类 - * @date 2020-04-27 - */ -public class NotebookUtil { - - private static final char SEPARATOR = '-'; - private static final String NOTEBOOK = "notebook"; - private static final String NAMESPACE = "namespace"; - private static final String RESOURCE_NAME = "resource-name"; - private static final String IMAGE_NAME = "10.5.24.118:5000/notebook-tf-of-pytorch"; - private static final String K8S_MOUNT_PATH = "/tf"; - private static final String K8S_FILE_SEPARATOR = "/"; - private static final int TRUNCATION_INDEX = 250; - private static final int NOTEBOOK_TIMEOUT_SECOND = 180; - private static final String JUPYTER_TOKEN_KEY = "?token="; - - public static final String K8S_REGEX = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"; - public static final String K8S_NOTEBOOK_REGEX = "^[\\u4e00-\\u9fa5_a-zA-Z0-9\\-]+$"; - public static final String NOTEBOOK_NOT_EXISTS = "Notebook不存在"; - public static final Long ANONYMITY_USER_ID = 0L; - - public static final int CPU_MIN_NUMBER = 1; - public static final int CPU_MAX_NUMBER = 8; - public static final int GPU_MAX_NUMBER = 4; - public static final int GPU_MIN_NUMBER = 0; - public static final int MEMORY_MIN_NUMBER = 1; - public static final int MEMORY_MAX_NUMBER = 8; - public static final int DISK_MEMORY_MIN_NUMBER = 1; - public static final int DISK_MEMORY_MAX_NUMBER = 1024; - - public static final String FAILED = "失败!"; - - private NotebookUtil() { - - } - - /** - * 验证是否包含失败关键字 - * - * @param info - * @return boolean - */ - public static boolean validateFailedInfo(String info) { - return info != null && info.contains(FAILED); - } - - /** - * 获取当前登录人ID - * 如果当前没登录人，则返回匿名用户0 - * - * @return long - */ - public static long getCurUserId() { - try { - UserDTO userDTO = JwtUtils.getCurrentUserDto(); - return userDTO == null ? ANONYMITY_USER_ID : userDTO.getId(); - } catch (UnavailableSecurityManagerException e) { - LogUtil.error(LogEnum.NOTE_BOOK, "未找到登录用户"); - return ANONYMITY_USER_ID; - } - } - - /** - * @param userId 当前登录人 - * @return namespace - * @deprecated use K8sNameTool.generateNameSpace(..) instead - * 根据当前用户 生成 Notebook的NameSpace - */ - @Deprecated - public static String generateNameSpace(long userId) { - return NOTEBOOK + SEPARATOR + NAMESPACE + SEPARATOR + userId; - } - - /** - * @param notebookName notebook名称 - * @return resourceName - * @deprecated use K8sNameTool.generateResourceName(..) instead - * 根据当前nodebook 生成 ResourceName - */ - @Deprecated - public static String generateResourceName(String notebookName) { - return NOTEBOOK + SEPARATOR + RESOURCE_NAME + SEPARATOR + notebookName; - } - - /** - * @return jupyter镜像名称 - * @deprecated use Harbor image instead - */ - @Deprecated - public static String getImageName() { - return IMAGE_NAME; - } - - /** - * @return String 容器内映射到宿主机的路径 - */ - public static String getK8sMountPath() { - return K8S_MOUNT_PATH; - } - - /** - * @param userId 当前登录人 - * @return 根据规则生成随机PVC路径 - * @deprecated use K8sNameTool.getNfsPath(..) instead - * 根据当前用户生成PVC路径（notebook重新启动得用原先路径） - */ - @Deprecated - public static String generatePvcPath(long userId) { - return K8S_FILE_SEPARATOR - + "nfs" - + K8S_FILE_SEPARATOR - + NOTEBOOK - + K8S_FILE_SEPARATOR - + userId - + K8S_FILE_SEPARATOR - + DateUtil.format(new Date(), DatePattern.PURE_DATETIME_FORMAT) + RandomUtil.randomString(4); - } - - /** - * 获取k8s响应结果的tMessage信息 - * 如果超长(250)，则截断 - * - * @param info - * @return String - */ - public static String getK8sStatusInfo(String info) { - if (info == null) { - return SymbolConstant.BLANK; - } - return StringUtils.truncationString(info, TRUNCATION_INDEX); - } - - - /** - * 获取k8s响应结果的tMessage信息 - * 如果超长(250)，则截断 - * - * @param ptBaseResult - * @return String - */ - public static String getK8sStatusInfo(PtBaseResult ptBaseResult) { - if (ptBaseResult == null) { - return SymbolConstant.BLANK; - } - return getK8sStatusInfo(ptBaseResult.getMessage()); - } - - /** - * 获取k8s异常结果的tMessage信息 - * 如果超长(250)，则截断 - * - * @param e 异常 - * @return String - */ - public static String getK8sStatusInfo(Exception e) { - if (e == null) { - return SymbolConstant.BLANK; - } else if (e instanceof NullPointerException) { - return "K8s NullPointerException"; - } - return getK8sStatusInfo(e.getMessage()); - } - - /** - * 验证 jupyter URL 是否包含token - * - * @param jupyterUrl - * @return boolean - */ - public static boolean checkUrlContainsToken(String jupyterUrl) { - if (StringUtils.isBlank(jupyterUrl)) { - return false; - } - return jupyterUrl.contains(JUPYTER_TOKEN_KEY); - } - - /** - * 判断notebook是否启动超时 - * - * @param lastStartDate - * @return boolean - */ - public static boolean notebookStartTimeout(Date lastStartDate) { - if (lastStartDate == null) { - return true; - } - Date timeOutPoint = DateUtil.offset(new Date(), DateField.SECOND, -NOTEBOOK_TIMEOUT_SECOND); - return timeOutPoint.after(lastStartDate); - } - - /** - * 获取超时时间秒 - * - * @return Long - */ - public static Long getTimeoutSecondLong() { - return K8sCallBackTool.getTimeoutSecondLong(NOTEBOOK_TIMEOUT_SECOND); - } - - - /** - * 生成第三方notebook名称 - * - * @param bizNfsEnum 业务源 - * @param sourceId 第三方源主键 - * @return String - */ - public static String generateName(BizNfsEnum bizNfsEnum, long sourceId) { - return bizNfsEnum.getBizCode() + SEPARATOR + sourceId; - } -} diff --git a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java b/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java deleted file mode 100644 index 20dc4c7..0000000 --- a/dubhe-server/dubhe-admin/src/main/java/org/dubhe/utils/TrainUtil.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.utils; - -import org.dubhe.base.MagicNumConstant; -import org.dubhe.constant.SymbolConstant; - -import java.sql.Timestamp; - -/** - * @description 训练任务工具类 - * @date 2020-07-14 - */ -public class TrainUtil { - - - private TrainUtil() { - - } - - public static final String REGEXP = "^[a-zA-Z0-9\\-\\_\\u4e00-\\u9fa5]+$"; - public static final String REGEXP_NAME = "^[a-zA-Z0-9\\-\\_]+$"; - public static final String REGEXP_TAG = "^[a-zA-Z0-9\\-\\_\\.]+$"; - - public static final String RUNTIME = "%02d:%02d:%02d"; - public static final String FOUR_DECIMAL = "%04d"; - public static final String FOUR_TWO = "%.2f"; - - public static final int NUMBER_ZERO = 0; - public static final int NUMBER_ONE = 1; - public static final int NUMBER_TWO = 2; - public static final int NUMBER_SEVEN = 7; - public static final int NUMBER_EIGHT = 8; - public static final int NUMBER_TWENTY = 20; - public static final int NUMBER_THIRTY_TWO = 32; - public static final int NUMBER_SIXTY_FOUR = 64; - public static final int NUMBER_ONE_HUNDRED_AND_TWENTY_SEVEN = 127; - public static final int NUMBER_ONE_HUNDRED_AND_TWENTY_EIGHT = 128; - public static final int NUMBER_ONE_HUNDRED_AND_SIXTY_EIGHT = 168; - public static final int NUMBER_TWO_HUNDRED_AND_FIFTY_FIVE = 255; - public static final int NUMBER_ONE_THOUSAND = 1000; - public static final int NUMBER_ONE_THOUSAND_AND_TWENTY_FOUR = 1024; - - // 初始化训练时间 - public static final String INIT_RUNTIME = SymbolConstant.BLANK; - - /** - * 获取延时时间 - * @param delayTime 延时时间（单位为小时） - * @return 延时时间 - */ - public static Timestamp getDelayTime(Integer delayTime) { - return new Timestamp(System.currentTimeMillis() + delayTime * MagicNumConstant.SIXTY * MagicNumConstant.SIXTY * MagicNumConstant.ONE_THOUSAND); - } - - /** - * 获取倒计时 - * @param delayTime 延时时间(单位为毫秒) - * @return 倒计时（单位为分钟） - */ - public static Integer getCountDown(Long delayTime) { - return (int) ((delayTime - System.currentTimeMillis()) / (MagicNumConstant.SIXTY * MagicNumConstant.ONE_THOUSAND)); - } - -} diff --git a/dubhe-server/dubhe-admin/src/main/resources/banner.txt b/dubhe-server/dubhe-admin/src/main/resources/banner.txt deleted file mode 100644 index f961ed5..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/banner.txt +++ /dev/null @@ -1,2 +0,0 @@ - - :: Spring Boot :: (v2.2.5.RELEASE) diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml deleted file mode 100644 index 7e70595..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/config/application-prod.yml +++ /dev/null @@ -1,204 +0,0 @@ -server: - port: 8000 - # rest API 版本号 - rest-version: v1 - -#配置数据源 -spring: - redis: - #数据库索引 - database: 0 - host: 127.0.0.1 - port: 6379 - password: - #连接超时时间 - timeout: 5000 - datasource: - druid: - db-type: com.alibaba.druid.pool.DruidDataSource - driverClassName: net.sf.log4jdbc.sql.jdbcapi.DriverSpy - url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true - username: test - password: test - - # 初始化配置 - initial-size: 3 - # 最小连接数 - min-idle: 3 - # 最大连接数 - max-active: 15 - # 获取连接超时时间 - max-wait: 5000 - # 连接有效性检测时间 - time-between-eviction-runs-millis: 90000 - # 最大空闲时间 - min-evictable-idle-time-millis: 1800000 - test-while-idle: true - test-on-borrow: false - test-on-return: false - - validation-query: select 1 - # 配置监控统计拦截的filters - filters: stat - stat-view-servlet: - url-pattern: /druid/* - reset-enable: false - - web-stat-filter: - url-pattern: /* - exclusions: "*.js,*.gif,*.jpg,*.bmp,*.png,*.css,*.ico,/druid/*" - - - # 配置允许后面的Bean覆盖前面名称重复的Bean - main: - allow-bean-definition-overriding: true - - # 配置Sharding-JDBC数据源名称，可配置多个 - shardingsphere: - datasource: - name: master - master: - type: com.alibaba.druid.pool.DruidDataSource - driver-class-name: net.sf.log4jdbc.sql.jdbcapi.DriverSpy - url: jdbc:log4jdbc:mysql://127.0.0.1:3306/dubhe-dev?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&allowMultiQueries=true&useInformationSchema=true - username: test - password: test - - #指定data_dataset表 主键id 生成策略 - sharding: - tables: - data_file: - actual-data-nodes: master.data_file_$->{1..100000} - table-strategy: - standard: - sharding-column: dataset_id - precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm - - data_dataset_version_file: - actual-data-nodes: master.data_dataset_version_file_$->{1..100000} - table-strategy: - standard: - sharding-column: dataset_id - precise-algorithm-class-name: org.dubhe.utils.MyPreciseShardingAlgorithm - props: - sql: - show: true - -k8s: - # k8s集群配置文件 - kubeconfig: kubeconfig - # nfs服务暴露的IP地址 如需测试需修改为合适的地址 - nfs: 127.0.0.1 - #nfs服务端 共享目录 - nfs-root-path: /nfs/ - nfs-root-windows-path: "Z:" - # 命名空间关键字 - namespace: namespace - # k8s ingress域名 如需测试需修改为合适的域名 - host: notebook.test.com - # k8s ingress-controller 对外port，获取方式：部署 ingress-controller 后，在k8s master节点执行 kubectl get svc -A | grep 'ingress-nginx-controller' 获取80对应的外部端口 - port: 33334 - # k8s ingress-controller 对外grpc port - https-port: 31365 - # k8s 模型部署配置 - serving: - # k8s 模型部署域名 如需测试需修改为合适的域名 - host: serving.test.com - # tls 证书 crt - tls-crt: - # tls 证书 key - tls-key: - # elasticsearch暴露的服务地址，获取方式 部署 管理集群日志 后，在k8s msater节点执行 kubectl get svc -A | grep 'elasticsearch' 获取9200对应的外部端口 - elasticsearch: - hostlist: ${eshostlist:127.0.0.1:33333} - # 日志采集配置信息 - log: - type: _doc - # 过滤源字段 - source_field: message,@timestamp,kubernetes.pod_name - # 异步回调 - callback: - # boot 单机部署即回调本机实例 - url: localhost:${server.port} - token: - # 秘钥 - secret-key: 1qaz2wsx - # 过期时间（秒） - expire-seconds: 300 - # 展示Pod的CPU使用率,Memory使用量,GPU使用率的grafana地址 - pod: - metrics: - grafanaUrl: http://127.0.0.1:30006/d/job/monitor?orgId=1&refresh=5s&kiosk&var-pod= - prometheus: - query-url: http://127.0.0.1:30003/api/v1/query - gpu-query-param: sum(container_accelerator_duty_cycle{pod="{}"})by(pod,acc_id) - nfs-storage-class-name: zjlab-nfs-storage -#配置harbor -harbor: - # habor服务域名，需要将用户持有域名解析到 harbor服务所在服务器ip - address: harbor.test.com - username: admin - password: Harbor12345 - model-name: train - -# data模块配置 -data: - annotation: - task: - # 自动标注任务分割的文件split size - splitSize: 16 - server: - # 文件存储服务器用户名 - userName: root - # 数据集训练配置 - ptversion: localhost:${server.port}/ - -# minio配置 -minio: - url: http://127.0.0.1:9000/ - # 部署 minio 时指定的 MINIO_ACCESS_KEY - accessKey: admin - # 部署 minio 时指定的 MINIO_SECRET_KEY - secretKey: 123@abc.com - bucketName: dubhe-dev - presignedUrlExpiryTime: 300 - annotation: /annotation/ - -# 数据处理医学影像数据集dcm服务器配置，查看影像功能需要使用该服务 -dcm: - host: 10.5.29.100 - port: 11112 - -#镜像脚本路径 -image: - shell-path: /data/prod/dubhe-prod/ - -#logback -logging.config: - classpath:logback-spring-dev.xml -#是否开启 swagger-ui -swagger: - enabled: false - -docker: - host: 127.0.0.1 - port: 2375 - -# 配置转发 -dubhe-proxy: - visual: - keyword: visual - server: localhost - port: 9898 - refine: - keyword: refine - server: localhost - port: 9797 - -# serving模块配置 -serving: - sourcePath: "/serving/TS_Serving" - gateway-uri-postfix: .api.dubhe.ai - rootPath: "/serving/deployment/" - batchRootPath: "/serving/batch/" - group: serving_prod_group diff --git a/dubhe-server/dubhe-admin/src/main/resources/config/application.yml b/dubhe-server/dubhe-admin/src/main/resources/config/application.yml deleted file mode 100644 index b69f95a..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/config/application.yml +++ /dev/null @@ -1,169 +0,0 @@ -spring: - profiles: - active: prod - jackson: - time-zone: GMT+8 - date-format: yyyy-MM-dd HH:mm:ss - data: - redis: - repositories: - enabled: false - - # 邮箱配置，用于发送用户注册邮箱验证码 - mail: - host: smtp.163.com - # 邮件的发送者 163邮箱(开发测试时使用 发送延时 20秒，每天发送量限制 50) - username: test@163.com - # SMTP授权密码 - password: AAAAAAAAAAAAAAAA - protocol: smtp - properties.mail.smtp.auth: true - properties.mail.smtp.port: 465 #465或者994 - properties.mail.smtp.starttls.enable: true - properties.mail.smtp.starttls.required: true - properties.mail.smtp.ssl.enable: true - default-encoding: utf-8 - -#登录图形验证码有效时间/分钟 -loginCode: - expiration: 2 - codeKey: "validate_code" - width: 108 - height: 28 - length: 4 - -#密码加密传输，前端公钥加密，后端私钥解密，与前端src/settings.js publicKey搭配使用；可使用genKeyPair方法进行生成 -rsa: - private_key: MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEA0vfvyTdGJkdbHkB8mp0f3FE0GYP3AYPaJF7jUd1M0XxFSE2ceK3k2kw20YvQ09NJKk+OMjWQl9WitG9pB6tSCQIDAQABAkA2SimBrWC2/wvauBuYqjCFwLvYiRYqZKThUS3MZlebXJiLB+Ue/gUifAAKIg1avttUZsHBHrop4qfJCwAI0+YRAiEA+W3NK/RaXtnRqmoUUkb59zsZUBLpvZgQPfj1MhyHDz0CIQDYhsAhPJ3mgS64NbUZmGWuuNKp5coY2GIj/zYDMJp6vQIgUueLFXv/eZ1ekgz2Oi67MNCk5jeTF2BurZqNLR3MSmUCIFT3Q6uHMtsB9Eha4u7hS31tj1UWE+D+ADzp59MGnoftAiBeHT7gDMuqeJHPL4b+kC+gzV4FGTfhR9q3tTbklZkD2A== - public_key: MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANL378k3RiZHWx5AfJqdH9xRNBmD9wGD2iRe41HdTNF8RUhNnHit5NpMNtGL0NPTSSpPjjI1kJfVorRvaQerUgkCAwEAAQ== - -#jwt -jwt: - header: Authorization - # 令牌前缀 - token-start-with: Bearer - # 必须使用最少88位的Base64对该令牌进行编码 - base64-secret: ZmQ0ZGI5NjQ0MDQwY2I4MjMxY2Y3ZmI3MjdhN2ZmMjNhODViOTg1ZGE0NTBjMGM4NDA5NzYxMjdjOWMwYWRmZTBlZjlhNGY3ZTg4Y2U3YTE1ODVkZDU5Y2Y3OGYwZWE1NzUzNWQ2YjFjZDc0NGMxZWU2MmQ3MjY1NzJmMTIzNDU= - # 令牌过期时间 此处单位/毫秒 ，默认24小时，可在此网站生成 https://www.convertworld.com/zh-hans/time/milliseconds.html - token-validity-in-seconds: 86400000 - # 在线用户key - online-key: online-token - # 验证码 - code-key: code-key - -#是否开启 swagger-ui -swagger: - enabled: true - -# 后台添加用户的初始密码 -initial_password: "123456" - -train-job: - namespace: "namespace-" - separator: "-" - version-label: "v" - pod-name: "platform/resource-name" - python-format: " --" - manage: "train-manage" - out-path: "out" - log-path: "log" - load-path: "load" - load-key: "model_load_dir" - load-val-dataset-key: "val_data_url" - visualized-log-path: "visualizedlog" - docker-dataset-path: "/dataset" - docker-train-path: "/workspace" - docker-model-path: "/modeldir" - docker-val-dataset-path: "/valdataset" - docker-out-path: "train_out=${train-job.docker-train-path}/${train-job.out-path}" - docker-log-path: "train_log=${train-job.docker-train-path}/${train-job.log-path}" - docker-visualized-log-path: "train_visualized_log=${train-job.docker-train-path}/${train-job.visualized-log-path}" - docker-dataset: "data_url=${train-job.docker-dataset-path}" - docker-teacher-model-key: "teacher_path_list" - docker-teacher-model-path: "/model/teacher" - docker-student-model-key: "student_path_list" - docker-student-model-path: "/model/student" - atlas-anaconda: "cd /root/anaconda3/bin && source activate pytorch_1_3" - atlas-pythonioencoding: "PYTHONIOENCODING=utf-8 " - eight: "8" - plus-eight: "+8" - node-ips: "node_ips" - node-num: "num_nodes" - gpu-num-per-node: "gpu_num_per_node" - - - -minioweb: - GetToken: - url: minio/webrpc - param: - id: 2 - jsonrpc: 2.0 - method: Web.Login - zip: - url: minio/zip?token= - - -train-algorithm: - #是否输出训练结果 - is-train-out: true - #是否输出训练日志 - is-train-log: true - #是否输出可视化日志 - is-visualized-log: false - #设置默认算法来源(1为我的算法，2为预置算法) - algorithm-source: 1 - #设置fork默认值 - fork: false - - -# 配置sql打印日志 -logging: - level: - com: - lawt: - repository: - mapper: debug - -# 配置转发 -dubhe-proxy: - visual: - keyword: visual - # 可视化服务后端 ip - server: 127.0.0.1 - # 可视化服务后端 端口 - port: 9898 - refine: - keyword: refine - server: localhost - port: 9797 - -# 延时全局配置 -delay: - notebook: - #模型开发延时关闭时间 - delete: 240 - -# 模型优化容器镜像配置 -optimize: - image: harbor.dubhe.ai/optimize/oneflow-gpu:base - -# # serving模块容器镜像配置 -serving: - gpu-image: harbor.dubhe.ai/serving/serving-gpu:base - cpu-image: harbor.dubhe.ai/serving/serving-cpu:base - -#垃圾回收默认配置 -recycle: - #过期时间设置 - timeout: - #用户上传文件至临时路径下后文件最大有效时长，以小时为单位 - file-valid: 24 - #用户删除数据后，默认其文件最大有效时长，以天为单位 - date: 7 - #用户删除某一算法后，其算法文件最大有效时长，以天为单位 - algorithm-valid: 3 - #用户删除某一模型后，其模型文件最大有效时长，以天为单位 - model-valid: 3 - #用户删除训练任务后，其训练管理文件最大有效时长，以天为单位 - train-valid: 3 diff --git a/dubhe-server/dubhe-admin/src/main/resources/generator.properties b/dubhe-server/dubhe-admin/src/main/resources/generator.properties deleted file mode 100644 index 78e13bf..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/generator.properties +++ /dev/null @@ -1,20 +0,0 @@ -#数据库类型转Java类型 -tinyint=Integer -smallint=Integer -mediumint=Integer -int=Integer -integer=Integer -bigint=Long -float=Float -double=Double -decimal=BigDecimal -bit=Boolean -char=String -varchar=String -tinytext=String -text=String -mediumtext=String -longtext=String -date=Timestamp -datetime=Timestamp -timestamp=Timestamp \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/kubeconfig b/dubhe-server/dubhe-admin/src/main/resources/kubeconfig deleted file mode 100644 index 76cf823..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/kubeconfig +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: - server: - name: kubernetes -contexts: -- context: - cluster: kubernetes - user: kubernetes-admin - name: kubernetes-admin@kubernetes -current-context: kubernetes-admin@kubernetes -kind: Config -preferences: {} -users: -- name: kubernetes-admin - user: - client-certificate-data: - client-key-data: \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/logback-spring-dev.xml b/dubhe-server/dubhe-admin/src/main/resources/logback-spring-dev.xml deleted file mode 100644 index d873018..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/logback-spring-dev.xml +++ /dev/null @@ -1,263 +0,0 @@ - - - dubhe - - - - - - - - - ${log.pattern} - ${log.charset} - - - INFO - INFO - ACCEPT - DENY - - - - - - logs/${log.path}/info/dubhe-info.log - - logs/${log.path}/info/dubhe-${app.active}-info-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - INFO - INFO,K8S_CALLBACK - ACCEPT - DENY - - - - - - logs/${log.path}/debug/dubhe-debug.log - - logs/${log.path}/debug/dubhe-${app.active}-debug-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - DEBUG - DEBUG - ACCEPT - DENY - - - - - - logs/${log.path}/error/dubhe-error.log - - logs/${log.path}/error/dubhe-${app.active}-error-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - ERROR - ERROR - ACCEPT - DENY - - - - - - logs/${log.path}/warn/dubhe-warn.log - - logs/${log.path}/warn/dubhe-${app.active}-warn-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - - WARN - WARN - ACCEPT - DENY - - - - - - logs/${log.path}/trace/dubhe-trace.log - - logs/${log.path}/trace/dubhe-${app.active}-trace-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - TRACE - TRACE - ACCEPT - DENY - - - - - - - logs/${log.path}/info/dubhe-schedule.log - - logs/${log.path}/info/dubhe-${app.active}-schedule-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - INFO - SCHEDULE - ACCEPT - DENY - - - - - - logs/${log.path}/info/dubhe-request.log - - logs/${log.path}/info/dubhe-${app.active}-request-%d{yyyy-MM-dd}.%i.log - - - 50MB - 7 - 250MB - - - %m%n - ${log.charset} - - - true - - INFO - - GLOBAL_REQUEST - ACCEPT - DENY - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml b/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml deleted file mode 100644 index 8569f4d..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/mapper/ModelOptBuiltInMapper.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - diff --git a/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml b/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml deleted file mode 100644 index cdd4e00..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/mapper/PtTrainJobMapper.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/dubhe-server/dubhe-admin/src/main/resources/server.crt b/dubhe-server/dubhe-admin/src/main/resources/server.crt deleted file mode 100644 index d36060c..0000000 --- a/dubhe-server/dubhe-admin/src/main/resources/server.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDkTCCAnmgAwIBAgIJALnQkyWsbGjVMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg -Q29tcGFueSBMdGQxGzAZBgNVBAMMEiouc2VydmluZy5kdWJoZS5haTAeFw0yMDEw -MTYwMzAwMzVaFw0zMDEwMTQwMzAwMzVaMF8xCzAJBgNVBAYTAlhYMRUwEwYDVQQH -DAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxGzAZ -BgNVBAMMEiouc2VydmluZy5kdWJoZS5haTCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAOjZBySFMRPMjAgdDffVUrU0ZeZNlpZo97l4Malv5iyjbx5ULNm7 -WZT8PXbBupUw32Y0JA8Sea276Zzs6eluMmx98v7R7b5h0zHIO3j/ObRiH6wOwUsN -Xb5B6vTjcrX02rXC+jBO4wh2URJRafAtnPK7f0QsS0a+n6g3LpG1bT1MUIEr74iB -AuwrmVw2BHGy3qudUC9X4FXJoRcPhEFIoAM2A6Fj20R8euYIXZYEgG73TzxxhEPy -TY5kRqVIHhLSd16RUkrdVuY7TQR2ZjT4ZS1ZQkNgYekSF8VgB2AzeymdyPks3OBR -xfpyMcvsX+tVCYnbJ47rDWkk12mrIYFI0TsCAwEAAaNQME4wHQYDVR0OBBYEFDMT -UzFb/OMzwkAwfGVieAo6hapXMB8GA1UdIwQYMBaAFDMTUzFb/OMzwkAwfGVieAo6 -hapXMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAm8L2WkH3m5PkG2 -nrPQj1UPWfsGbpuUVnRO4qYr+IlUXK683a0+vl++iLkqlIL2MgkbDhFBwAUFVI3P -Bw7jtq9/kKWYb6ItQXWC7LmLYk0onEQACbQ41E6DbzJD0225L4BQFDiF+8OZUB3v -NtCrbE1u+5D2+X4ig0SW/FZhoPW87g28Mw+4Feyx29EOwzUaGz9ZObd+Vnw8ff5B -rUJFBSSXV0A7Iyd/QZt/GAQrcCwi8VDIbFXHm/ekyZcHaISwMQadNMMTKDCH+BNH -Wcr5py6d/NPYZJa1GqI1EUIMqpea8oxl9CahwbpIgO76AVXEditIGqRct/dIeq8M -Hv3qcSs= ------END CERTIFICATE----- diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java deleted file mode 100755 index 6f756eb..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/BaseTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.apache.shiro.SecurityUtils; -import org.apache.shiro.subject.Subject; -import org.apache.shiro.util.ThreadContext; -import org.apache.shiro.web.mgt.DefaultWebSecurityManager; -import org.dubhe.support.login.UsernamePasswordCaptchaToken; -import org.junit.Assert; -import org.junit.Before; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.MediaType; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.context.web.WebAppConfiguration; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.ResultMatcher; -import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.springframework.web.context.WebApplicationContext; - -import java.io.UnsupportedEncodingException; - -/** - * @description 基础测试类 - * @date 2020-04-20 - */ -@ActiveProfiles(value = "dev") -@RunWith(SpringRunner.class) -@SpringBootTest -@EnableTransactionManagement -@WebAppConfiguration -public class BaseTest { - - @Autowired - private WebApplicationContext wac; - protected MockMvc mockMvc; - - @Autowired - private DefaultWebSecurityManager defaultWebSecurityManager; - - public BaseTest() { - } - - @Before - public void setup() { - this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); - SecurityUtils.setSecurityManager(defaultWebSecurityManager); - ThreadContext.bind(defaultWebSecurityManager); - Subject subject = SecurityUtils.getSubject(); - UsernamePasswordCaptchaToken token = new UsernamePasswordCaptchaToken("admin", "123456"); - token.setRememberMe(true); - subject.login(token); - } - - /** - * mockMvcTest - * content 返回结果 - * status 返回状态码 - * - * @param mockHttpServletRequestBuilder 模拟HTTP请求 - * @param s 传入参数 - * @param ok 预期结果 - * @param i 预期状态码 - * @throws throws Exception - */ - public void mockMvcTest(MockHttpServletRequestBuilder mockHttpServletRequestBuilder, String s, ResultMatcher ok, int i) throws Exception { - MockHttpServletResponse response = this.mockMvc.perform( - mockHttpServletRequestBuilder - .contentType(MediaType.APPLICATION_JSON) - .content(s) - ).andExpect(ok) - .andReturn() - .getResponse(); - response.setCharacterEncoding("UTF-8"); - //得到返回状态码 - int status = response.getStatus(); - //得到返回结果 - String content = response.getContentAsString(); - //断言，判断返回代码是否正确 - Assert.assertEquals(i, status); - System.out.println(content); - } - - /** - * @param @param response - * @param @param i - * @param @throws UnsupportedEncodingException 入参 - * @return void 返回类型 - * @throws @Title: mockMvcWithNoRequestBody - */ - public void mockMvcWithNoRequestBody(MockHttpServletResponse response, int i) throws UnsupportedEncodingException { - response.setCharacterEncoding("UTF-8"); - // 得到返回代码 - int status = response.getStatus(); - // 得到返回结果 - String content = response.getContentAsString(); - // 断言，判断返回代码是否正确 - Assert.assertEquals(i, status); - System.out.println("返回的参数" + content); - } - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java deleted file mode 100644 index db56668..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/DictControllerTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.MediaType; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.web.context.WebApplicationContext; -/** - * @description 实体类 - * @date 2020-03-25 - */ -@RunWith(SpringRunner.class) -@SpringBootTest -public class DictControllerTest { - @Autowired - private WebApplicationContext wac; - private MockMvc mockMvc; - - public DictControllerTest() { - } - - @Before - public void setup() { - this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).build(); - } - - @Test - public void whenQueryUserAll() throws Exception { - this.mockMvc.perform(MockMvcRequestBuilders.get("/api/dict/all", new Object[0]).contentType(MediaType.APPLICATION_JSON_UTF8)).andExpect(MockMvcResultMatchers.status().isOk()); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java deleted file mode 100644 index e95e64c..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/ForwardTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/** Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Test; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; - -/** - * @description 代理转发单元测试 - * @date 2020-06-23 - */ - -public class ForwardTest extends BaseTest { - @Test - public void ForwardControllerTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm").param("algorithmSource", "2")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java deleted file mode 100644 index b4051af..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sCallBackToolTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.dubhe.enums.BizEnum; -import org.dubhe.utils.K8sCallBackTool; -import org.dubhe.utils.K8sNameTool; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; - -/** - * @description K8sCallBackToolTest测试类 - * @date 2020-05-28 - */ -@ActiveProfiles(value = "dev") -@RunWith(SpringRunner.class) -@SpringBootTest(classes= AppRun.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) -public class K8sCallBackToolTest{ - - @Autowired - private K8sCallBackTool k8sCallBackTool; - - @Autowired - private K8sNameTool k8sNameTool; - - @Test - public void token(){ - System.out.println(k8sCallBackTool.generateToken()); - Assert.assertTrue(k8sCallBackTool.validateToken(k8sCallBackTool.generateToken())); - } - - @Test - public void getPodCallbackUrl(){ - Assert.assertEquals("localhost:8000/api/k8s/callback/pod/notebook",k8sCallBackTool.getPodCallbackUrl(k8sNameTool.getPodLabel(BizEnum.NOTEBOOK))); - } - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java deleted file mode 100644 index 5c181f7..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/K8sNameToolTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - - -import org.dubhe.enums.BizEnum; -import org.dubhe.enums.BizNfsEnum; -import org.dubhe.utils.K8sNameTool; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; - - -/** - * @description K8sNameToolTest测试类 - * @date 2020-05-13 - */ -@ActiveProfiles(value = "dev") -@RunWith(SpringRunner.class) -@SpringBootTest -public class K8sNameToolTest { - - @Autowired - private K8sNameTool k8sNameTool; - - @Test - public void generateResourceName(){ - Assert.assertEquals("notebook-rn-info", k8sNameTool.generateResourceName(BizEnum.NOTEBOOK,"info")); - } - - @Test - public void generateNameSpace(){ - Assert.assertEquals("namespace-0", k8sNameTool.generateNamespace(0L)); - } - - @Test - public void getUserIdFromNameSpace(){ - Assert.assertSame(0L, k8sNameTool.getUserIdFromNamespace("namespace-0")); - } - - - @Test - public void getNfsPath(){ - Assert.assertTrue(k8sNameTool.getNfsPath(BizNfsEnum.NOTEBOOK,0L).startsWith("/notebook/0/")); - } - - @Test - public void getAbsoluteNfsPath(){ - Assert.assertEquals("/nfs/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.getAbsoluteNfsPath("/notebook/0/20200513162830yody/")); - } - - @Test - public void appendBucket(){ - Assert.assertEquals("/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.appendBucket("/notebook/0/20200513162830yody/")); - Assert.assertEquals("/dubhe-dev/notebook/0/20200513162830yody/",k8sNameTool.appendBucket("notebook/0/20200513162830yody/")); - } - - @Test - public void convertNfsPath(){ - Assert.assertEquals("/algorithm-manage/0/20200513162830yody/",k8sNameTool.convertNfsPath( - "/notebook/0/20200513162830yody/" - ,BizNfsEnum.NOTEBOOK - ,BizNfsEnum.ALGORITHM)); - } - - @Test - public void validateBizNfsPath(){ - Assert.assertTrue(k8sNameTool.validateBizNfsPath( - "/notebook/0/20200513162830yody/" - ,BizNfsEnum.NOTEBOOK)); - Assert.assertFalse(k8sNameTool.validateBizNfsPath( - "/notebook/0/20200513162830yody/" - ,BizNfsEnum.ALGORITHM)); - Assert.assertFalse(k8sNameTool.validateBizNfsPath( - "/dubhe-dev/notebook/0/20200513162830yody/" - ,BizNfsEnum.ALGORITHM)); - Assert.assertFalse(k8sNameTool.validateBizNfsPath( - "/nfs/dubhe-dev/notebook/0/20200513162830yody/" - ,BizNfsEnum.ALGORITHM)); - } - - @Test - public void getByCreateResource(){ - Assert.assertSame(BizNfsEnum.ALGORITHM , BizNfsEnum.getByCreateResource(1)); - Assert.assertNull( BizNfsEnum.getByCreateResource(10)); - } - - @Test - public void removeNfsRootPath(){ - Assert.assertEquals( "/dubhe-dev/algorithm-manage/0/20200519111111test", k8sNameTool.removeNfsRootPath("/nfs/dubhe-dev/algorithm-manage/0/20200519111111test")); - Assert.assertEquals( "/nfs2/dubhe-dev/algorithm-manage/0/20200519111111test", k8sNameTool.removeNfsRootPath("/nfs2/dubhe-dev/algorithm-manage/0/20200519111111test")); - } - - @Test - public void getPodLabel(){ - Assert.assertEquals( "notebook", k8sNameTool.getPodLabel(BizEnum.NOTEBOOK)); - } - - @Test - public void getDatasetPath(){ - Assert.assertEquals( "/dataset", k8sNameTool.getDatasetPath()); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java deleted file mode 100644 index 98ae925..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtImageTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import org.dubhe.domain.dto.PtImageDeleteDTO; -import org.dubhe.domain.dto.PtImageUpdateDTO; -import org.dubhe.domain.dto.PtImageUploadDTO; -import org.junit.Test; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import java.util.Arrays; - -/** - * @description 镜像接口单元测试 - * @date 2020-06-28 - */ -public class PtImageTest extends BaseTest { - - /** - * 查询镜像 - */ - @Test - public void getImageTest() throws Exception { - MultiValueMap params = new LinkedMultiValueMap<>(); - params.add("current", "1"); - params.add("size", "10"); - params.add("sort", "id"); - params.add("order", "desc"); - params.add("imageResource", "0"); - params.add("imageNameOrId", "oneflow"); - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptImage/info").params(params)) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - /** - * 通过projectName查询镜像 - */ - @Test - public void getTagsByImageNameTest() throws Exception { - String imageName = "redis"; - mockMvc.perform( - MockMvcRequestBuilders.get("/api/v1/ptImage") - .param("imageName", imageName)) - .andExpect(MockMvcResultMatchers.status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - } - - /** - * 上传镜像包到harbor - */ - @Test - public void uploadImageTest() throws Exception { - PtImageUploadDTO ptImageUploadDTO = new PtImageUploadDTO(); - ptImageUploadDTO.setImageName("mysql"); - ptImageUploadDTO.setImagePath("F:/mysql.tar"); - ptImageUploadDTO.setImageTag("5.7"); - ptImageUploadDTO.setRemark("测试上传镜像"); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/ptImage/uploadImage"), JSON.toJSONString(ptImageUploadDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 修改镜像信息 - */ - @Test - public void updateImageTest() throws Exception { - PtImageUpdateDTO imageUpdateDTO = new PtImageUpdateDTO(); - imageUpdateDTO.setIds(Arrays.asList()); - imageUpdateDTO.setRemark(""); - - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/ptImage"), JSON.toJSONString(imageUpdateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 删除镜像 - */ - @Test - public void deleteImageTest() throws Exception { - PtImageDeleteDTO imageDeleteDTO = new PtImageDeleteDTO(); - imageDeleteDTO.setIds(Arrays.asList()); - - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/ptImage"), JSON.toJSONString(imageDeleteDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - - /** - * - * 获取镜像名称列表 - */ - @Test - public void getImageNameListTest() throws Exception { - mockMvcTest(MockMvcRequestBuilders.get("/api/v1/ptImage/imageNameList"), "", - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } -} - diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java deleted file mode 100644 index 001fcb8..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtMeasureTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import org.dubhe.domain.dto.PtMeasureDTO; -import org.dubhe.domain.dto.PtMeasureDeleteDTO; -import org.dubhe.domain.dto.PtMeasureUpdateDTO; -import org.junit.Test; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * @description 度量管理接口单元测试 - * @date 2020-11-16 - */ -public class PtMeasureTest extends BaseTest { - - /** - * 查询度量 - */ - @Test - public void getMeasureTest() throws Exception { - MultiValueMap params = new LinkedMultiValueMap<>(); - params.add("current", "1"); - params.add("size", "10"); - params.add("sort", "id"); - params.add("order", "desc"); - params.add("name", "COCO1"); - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptMeasure/info").params(params)) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - - /** - * 新建度量 - */ - @Test - public void addMeasureTest() throws Exception { - PtMeasureDTO ptMeasureCreateDTO = PtMeasureDTO.builder() - .name("测试111701") - .description("测试新建度量") - .url("/nfs/dubhe-dev/upload-temp/1/5b07d66ad27a11eab7cffa3a5ae24f00.json") - .build(); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureCreateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 修改度量信息 - */ - @Test - public void updateMeasureTest() throws Exception { - PtMeasureUpdateDTO ptMeasureUpdateDTO = PtMeasureUpdateDTO.builder() - .id(1L) - .name("COCO1") - .description("测试修改度量") - .url("E:/chromeDownload/5b07d66ad27a11eab7cffa3a5ae24f00.json") - .build(); - - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureUpdateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 删除度量 - */ - @Test - public void deleteMeasureTest() throws Exception { - Set ids = Stream.of(1L, 2L).collect(Collectors.toSet()); - PtMeasureDeleteDTO ptMeasureDeleteDTO = PtMeasureDeleteDTO.builder().ids(ids).build(); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/ptMeasure"), JSON.toJSONString(ptMeasureDeleteDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 根据度量名称返回度量文件信息 - */ - @Test - public void getMeasureByNameTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/ptMeasure/byName").param("name", "COCO")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } -} - diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java deleted file mode 100644 index bb550a4..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainAlgorithmUsageTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUsageUpdateDTO; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; - -/** - * @description 算法用途单元测试 - * @date 2020-06-23 - */ - -@RunWith(SpringRunner.class) -@SpringBootTest -public class PtTrainAlgorithmUsageTest extends BaseTest { - - /** - * 修改任务参数 算法id=2在算法表中runcommand为空 - */ - @Test - public void queryAllTest() throws Exception { - - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithmUsage").param("isContainDefault", "1")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - @Test - public void queryAllTest2() throws Exception { - - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithmUsage").param("isContainDefault", "0")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - @Test - public void createTest() throws Exception { - - PtTrainAlgorithmUsageCreateDTO ptTrainAlgorithmUsageCreateDTO = new PtTrainAlgorithmUsageCreateDTO(); - ptTrainAlgorithmUsageCreateDTO.setAuxInfo("untilTesting"); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithmUsage"), - JSON.toJSONString(ptTrainAlgorithmUsageCreateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), - 200); - - } - - - @Test - public void deleteTest() throws Exception { - Long[] longs = new Long[1]; - longs[0] = 38L; - PtTrainAlgorithmUsageDeleteDTO ptTrainAlgorithmUsageDeleteDTO = new PtTrainAlgorithmUsageDeleteDTO(); - ptTrainAlgorithmUsageDeleteDTO.setIds(longs); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithmUsage"), JSON.toJSONString(ptTrainAlgorithmUsageDeleteDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - - } - - @Test - public void updateTest() throws Exception { - PtTrainAlgorithmUsageUpdateDTO ptTrainAlgorithmUsageUpdateDTO = new PtTrainAlgorithmUsageUpdateDTO(); - - ptTrainAlgorithmUsageUpdateDTO.setId(38L); - ptTrainAlgorithmUsageUpdateDTO.setAuxInfo("更新测试"); - - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithmUsage"), - JSON.toJSONString(ptTrainAlgorithmUsageUpdateDTO), MockMvcResultMatchers.status().is2xxSuccessful(), - 200); - } - - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java deleted file mode 100644 index 568ddde..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainLogServiceTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -/** - * @description 用户查询训练日志 - * @date 2020-6-20 - **/ -@SpringBootTest -@RunWith(SpringRunner.class) -public class PtTrainLogServiceTest extends BaseTest { - - @Test - public void searchTrainLogInfo() throws Exception { - MultiValueMap params = new LinkedMultiValueMap<>(); - params.add("jobId", "68"); - params.add("startLine", "1"); - params.add("lines", "50"); - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainLog").params(params)) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java deleted file mode 100644 index 0d42599..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/PtTrainModelOptJobApiTest.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import org.dubhe.domain.dto.PtTrainJobCreateDTO; -import org.dubhe.domain.dto.PtTrainJobDeleteDTO; -import org.dubhe.domain.dto.PtTrainJobStopDTO; -import org.dubhe.domain.dto.PtTrainJobUpdateDTO; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.transaction.annotation.Transactional; - -/** - * @description 训练任务管理模块任务参数管理单元测试 - * @date 2020-5-11 - */ - -@RunWith(SpringRunner.class) -@SpringBootTest -public class PtTrainModelOptJobApiTest extends BaseTest { - - /** - * 作业列表展示 - * - * @throws Exception - */ - @Test - public void getTrainJobTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - /** - * 规格展示 - * - * @throws Exception - */ - @Test - public void getTrainJobSpecsTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/trainJobSpecs").param("resourcesPoolType", "0")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - /** - * 根据jobId查询训练任务详情 - * - * @throws Exception - */ - @Test - public void getTrainJobDetailTest() throws Exception { - mockMvcWithNoRequestBody( - mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/jobDetail").param("id", "538")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), - 200); - } - - /** - * 我的训练任务统计 - * - * @throws Exception - */ - @Test - public void statisticsMineTest() throws Exception { - mockMvcWithNoRequestBody(this.mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/mine")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - /** - * 作业不同版本任务列表展示 - * - * @param @throws Exception 入参 - * @return void 返回类型 - * @throws @date 2020年6月16日 上午10:19:12 - * @Title: getTrainJobVersionTest - * @version V1.0 - */ - @Test - public void getTrainJobVersionTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/trainJobVersionDetail").param("trainId", String.valueOf(371L))) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - - } - - /** - * 数据集状态展示 - * - * @param @throws Exception 入参 - * @return void 返回类型 - * @throws @date 2020年6月16日 上午10:31:25 - * @Title: getTrainDataSourceStatusTest - * @version V1.0 - */ - @Test - public void getTrainDataSourceStatusTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainJob/dataSourceStatus").param("dataSourcePath", - "dataset/68,dataset/20741/versionFile/V0003")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - - } - - /** - * 创建训练任务 - * - */ - @Test - @Transactional(rollbackFor = Exception.class) - public void createTrainJobTest() throws Exception { - PtTrainJobCreateDTO ptTrainJobCreateDTO = new PtTrainJobCreateDTO(); - ptTrainJobCreateDTO.setTrainName("创建训练任务单元测试"); - ptTrainJobCreateDTO.setAlgorithmId(134L); - ptTrainJobCreateDTO.setDataSourceName("CUB-200-2011:V0002"); - ptTrainJobCreateDTO.setDataSourcePath("dataset/242/versionFile/V0002/ofrecord/train"); - ptTrainJobCreateDTO.setDescription("创建训练任务单元测试"); - ptTrainJobCreateDTO.setResourcesPoolType(1).setResourcesPoolNode(1).setTrainType(0); - ptTrainJobCreateDTO.setTrainJobSpecsName("1Core4GB 1TITAN V").setTrainJobSpecsInfo("1Core4GB 1TITAN V").setRunCommand("python atlas_knowledge_distillation.py").setImageName("atlas").setImageTag("2.1"); - JSONObject runParams = new JSONObject(); - runParams.put("epochs", 100); - runParams.put("batch_size", 16); - runParams.put("weight_decay", 1e-4); - runParams.put("learning_rate", 1e-4); - ptTrainJobCreateDTO.setRunParams(runParams); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobCreateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - /** - * 更新训练任务测试 - * - * @param @throws Exception 入参 - * @return void 返回类型 - * @throws @date 2020年6月16日 上午10:46:55 - * @Title: updateTrainJobTest - * @version V1.0 - */ - @Test - public void updateTrainJobTest() throws Exception { - // TODO Auto-generated method stub - PtTrainJobUpdateDTO ptTrainJobUpdateDTO = new PtTrainJobUpdateDTO(); - ptTrainJobUpdateDTO.setId(537L); - ptTrainJobUpdateDTO.setAlgorithmId(134L); - ptTrainJobUpdateDTO.setDataSourceName("CUB-200-2011:V0002"); - ptTrainJobUpdateDTO.setDataSourcePath("dataset/242/versionFile/V0002/ofrecord/train"); - ptTrainJobUpdateDTO.setDescription("修改训练任务单元测试"); - ptTrainJobUpdateDTO.setResourcesPoolType(1).setResourcesPoolNode(1).setTrainType(0); - ptTrainJobUpdateDTO.setTrainJobSpecsName("1Core4GB 1TITAN V").setTrainJobSpecsInfo("1Core4GB 1TITAN V").setRunCommand("python atlas_knowledge_distillation.py").setImageName("atlas").setImageTag("2.1"); - JSONObject runParams = new JSONObject(); - runParams.put("epochs", 100); - runParams.put("batch_size", 16); - runParams.put("weight_decay", 1e-4); - runParams.put("learning_rate", 1e-4); - ptTrainJobUpdateDTO.setRunParams(runParams); - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobUpdateDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - - } - - /** - * 删除训练任务 - * - * @param - * @return void 返回类型 - * @throws @date 2020年6月16日 上午10:50:31 - * @throws Exception - * @Title: deleteTrainJobTest - * @version V1.0 - */ - @Test - public void deleteTrainJobTest() throws Exception { - // TODO Auto-generated method stub - PtTrainJobDeleteDTO ptTrainJobDeleteDTO = new PtTrainJobDeleteDTO(); - ptTrainJobDeleteDTO.setId(118L); - ptTrainJobDeleteDTO.setTrainId(87L); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobDeleteDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - - } - - /** - * - * - * @throws Exception - */ - @Test - public void deleteTrainJobWithNoIdTest() throws Exception { - // TODO Auto-generated method stub - PtTrainJobDeleteDTO ptTrainJobDeleteDTO = new PtTrainJobDeleteDTO(); - ptTrainJobDeleteDTO.setTrainId(36L); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainJob"), JSON.toJSONString(ptTrainJobDeleteDTO), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - - } - - /** - * 停止训练任务测试 - * - * @param @throws Exception 入参 - * @return void 返回类型 - * @throws @date 2020年6月16日 上午10:55:22 - * @Title: testStopTrainJob - * @version V1.0 - */ - @Test - public void testStopTrainJob() throws Exception { - - PtTrainJobStopDTO dto = new PtTrainJobStopDTO(); - dto.setId(46L); - dto.setTrainId(37L); - - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainJob/stop"), JSON.toJSONString(dto), - MockMvcResultMatchers.status().is2xxSuccessful(), 200); - } - - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java deleted file mode 100644 index 560ef35..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainAlgorithmTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import org.dubhe.domain.dto.PtTrainAlgorithmCreateDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmDeleteDTO; -import org.dubhe.domain.dto.PtTrainAlgorithmUpdateDTO; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; - -import java.util.HashSet; -import java.util.Set; - -/** - * @description 算法管理模块算法管理单元测试 - * @date 2020-06-18 - */ -public class TrainAlgorithmTest extends BaseTest { - - /** - * 查询算法列表 - */ - @Test - public void ptTrainAlgorithmQueryTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/algorithm").param("algorithmSource", "2")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - /** - * 查询当前用户的算法个数 - */ - @Test - public void getAlgorithmCountTest() throws Exception { - MockHttpServletResponse response = this.mockMvc.perform( - MockMvcRequestBuilders.get("/api/v1/algorithm/myAlgorithmCount") - ).andExpect(MockMvcResultMatchers.status().isOk()) - .andReturn() - .getResponse(); - response.setCharacterEncoding("UTF-8"); - //得到返回状态码 - int status = response.getStatus(); - //得到返回结果 - String content = response.getContentAsString(); - //断言，判断返回代码是否正确 - Assert.assertEquals(200, status); - System.out.println(content); - - } - - - /** - * 新增算法 - */ - @Test - public void ptTrainAlgorithmCreateTest() throws Exception { - PtTrainAlgorithmCreateDTO ptTrainAlgorithmCreateDTO = new PtTrainAlgorithmCreateDTO(); - JSONObject jsonObject = new JSONObject(); - ptTrainAlgorithmCreateDTO.setAlgorithmName("untilTesting") - .setDescription("untilTesting") - .setCodeDir("upload-temp/1/20201202135732212Bp8F/OneFlow算法.zip"); - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmCreateDTO), MockMvcResultMatchers.status().isOk(), 200); - } - - /** - * 修改算法 - */ - @Test - public void ptTrainAlgorithmUpdateTest() throws Exception { - PtTrainAlgorithmUpdateDTO ptTrainAlgorithmUpdateDTO = new PtTrainAlgorithmUpdateDTO(); - ptTrainAlgorithmUpdateDTO.setId(138L) - .setAlgorithmName("untilTesting"+ System.currentTimeMillis()) - .setDescription("untilTesting"); - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); - } - - - /** - * 删除算法 - */ - @Test - public void ptTrainAlgorithmDeleteTest() throws Exception { - Set ids = new HashSet<>(); - ids.add(138L); - PtTrainAlgorithmDeleteDTO ptTrainAlgorithmDeleteDTO = new PtTrainAlgorithmDeleteDTO(); - ptTrainAlgorithmDeleteDTO.setIds(ids); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/algorithm"), JSON.toJSONString(ptTrainAlgorithmDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); - } - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java deleted file mode 100644 index 3c368dd..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/TrainParamApiTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; -import org.dubhe.domain.dto.PtTrainParamCreateDTO; -import org.dubhe.domain.dto.PtTrainParamDeleteDTO; -import org.dubhe.domain.dto.PtTrainParamUpdateDTO; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.annotation.Rollback; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import org.springframework.transaction.annotation.Transactional; - -import java.util.HashSet; -import java.util.Set; - -/** - * @description 训练任务管理模块任务参数管理单元测试 - * @date 2020-5-11 - */ -@RunWith(SpringRunner.class) -@SpringBootTest -public class TrainParamApiTest extends BaseTest { - - /** - * 查询任务参数列表 - */ - @Test - public void ptTrainParamQueryTest() throws Exception { - mockMvcWithNoRequestBody(mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/trainParams")) - .andExpect(MockMvcResultMatchers.status().isOk()).andReturn().getResponse(), 200); - } - - - /** - * 新增任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - @Rollback(false) - public void ptTrainParamCreateTest() throws Exception { - PtTrainParamCreateDTO ptTrainParamCreateDTO = new PtTrainParamCreateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("data_url", "/nfs/testuser1/mnist/MNIST_data"); - ptTrainParamCreateDTO.setParamName("新增任务参数名称单元测试").setAlgorithmId(133L).setRunParams(runParams) - .setDescription("新增任务参数名称").setDataSourceName("T1:V0004").setDataSourcePath("dataset/159/versionFile/V0004/ofrecord/train") - .setResourcesPoolType(1).setTrainType(0).setResourcesPoolNode(1).setRunCommand("python p.py").setTrainJobSpecsName("1Core4GB 1TITAN V").setImageName("oneflow").setImageTag("cu102-py37-dist"); - mockMvcTest(MockMvcRequestBuilders.post("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamCreateDTO), MockMvcResultMatchers.status().isOk(), 200); - - } - - - /** - * 修改任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - @Rollback(false) - public void ptTrainParamUpdateTest() throws Exception { - PtTrainParamUpdateDTO ptTrainParamUpdateDTO = new PtTrainParamUpdateDTO(); - JSONObject runParams = new JSONObject(); - runParams.put("key1", 11); - runParams.put("key2", 11); - runParams.put("key3", 11); - runParams.put("key4", 11); - ptTrainParamUpdateDTO.setId(125L).setParamName("修改任务参数名称单元测试" + System.currentTimeMillis()).setAlgorithmId(133L).setRunParams(runParams) - .setDescription("修改任务参数名称单元测试").setDataSourceName("T1:V0004").setDataSourcePath("dataset/159/versionFile/V0004/ofrecord/train") - .setResourcesPoolType(1).setTrainType(0).setResourcesPoolNode(1).setRunCommand("python p.py").setTrainJobSpecsName("1Core4GB 1TITAN V").setImageName("oneflow").setImageTag("cu102-py37-dist"); - mockMvcTest(MockMvcRequestBuilders.put("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamUpdateDTO), MockMvcResultMatchers.status().isOk(), 200); - } - - - /** - * 删除任务参数 - */ - @Test - @Transactional(rollbackFor = Exception.class) - @Rollback(false) - public void ptTrainParamDeleteTest() throws Exception { - Set ids = new HashSet<>(); - ids.add(125L); - PtTrainParamDeleteDTO ptTrainParamDeleteDTO=new PtTrainParamDeleteDTO(); - ptTrainParamDeleteDTO.setIds(ids); - mockMvcTest(MockMvcRequestBuilders.delete("/api/v1/trainParams"), JSON.toJSONString(ptTrainParamDeleteDTO), MockMvcResultMatchers.status().isOk(), 200); - } -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java deleted file mode 100644 index bf196ce..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/FileServiceImplTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.data.service; - -import org.dubhe.BaseTest; -import org.dubhe.data.service.impl.FileServiceImpl; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * @description 文件业务层测试类 - * @date 2020-05-12 - */ -public class FileServiceImplTest extends BaseTest { - - @Autowired - private FileServiceImpl fileService; - - -} diff --git a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java b/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java deleted file mode 100644 index cd12405..0000000 --- a/dubhe-server/dubhe-admin/src/test/java/org/dubhe/data/service/LabelBOServiceImplTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2020 Tianshu AI Platform. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================= - */ - -package org.dubhe.data.service; - -import org.dubhe.BaseTest; -import org.dubhe.data.domain.entity.Label; -import org.dubhe.data.service.impl.LabelServiceImpl; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import java.util.Arrays; -import java.util.List; - -/** - * @description 标签业务层测试类 - * @date 2020-05-12 - */ -public class LabelBOServiceImplTest extends BaseTest { - - @Autowired - private LabelServiceImpl labelService; - - @Test - public void exist() { - System.out.println(labelService.exist(8L)); - } - - @Test - public void save() { - List