Merge branch 'V20211018' into zouap

README.md

@@ -2,7 +2,7 @@
 
 <h1><img src="public/img/favicon.png" alt="logo" width="30" height="30">AiForge - 启智AI开发协作平台</h1>
 
-[![release](https://img.shields.io/badge/release-V1.21.8-blue)](https://git.openi.org.cn/OpenI/aiforge/releases/latest)
+[![release](https://img.shields.io/badge/release-1.21.10.1-blue)](https://git.openi.org.cn/OpenI/aiforge/releases/latest)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 
 

models/models.go

@@ -59,6 +59,9 @@ var (
 	x      *xorm.Engine
 	tables []interface{}
 
+	xStatistic      *xorm.Engine
+	tablesStatistic []interface{}
+
 	// HasEngine specifies if we have a xorm.Engine
 	HasEngine bool
 )
@@ -133,14 +136,17 @@ func init() {
 		new(UserBusinessAnalysis),
 	)
 
+	tablesStatistic = append(tablesStatistic,
+		new(FileChunk))
+
 	gonicNames := []string{"SSL", "UID"}
 	for _, name := range gonicNames {
 		names.LintGonicMapper[name] = true
 	}
 }
 
-func getEngine() (*xorm.Engine, error) {
-	connStr, err := setting.DBConnStr()
+func getEngine(database *setting.DBInfo) (*xorm.Engine, error) {
+	connStr, err := setting.DBConnStr(database)
 	if err != nil {
 		return nil, err
 	}
@@ -154,14 +160,12 @@ func getEngine() (*xorm.Engine, error) {
 	}
 	engine.SetSchema(setting.Database.Schema)
 
-	HasEngine = true
-
 	return engine, nil
 }
 
 // NewTestEngine sets a new test xorm.Engine
 func NewTestEngine(x *xorm.Engine) (err error) {
-	x, err = getEngine()
+	x, err = getEngine(setting.Database)
 	if err != nil {
 		return fmt.Errorf("Connect to database: %v", err)
 	}
@@ -172,43 +176,80 @@ func NewTestEngine(x *xorm.Engine) (err error) {
 	return x.StoreEngine("InnoDB").Sync2(tables...)
 }
 
-// SetEngine sets the xorm.Engine
+// setEngine sets the xorm.Engine
+func setEngine(engine *xorm.Engine, table []interface{}, database *setting.DBInfo) (err error) {
+	engine.SetMapper(names.GonicMapper{})
+	// WARNING: for serv command, MUST remove the output to os.stdout,
+	// so use log file to instead print to stdout.
+	engine.SetLogger(NewXORMLogger(setting.Database.LogSQL))
+	engine.ShowSQL(setting.Database.LogSQL)
+	engine.SetMaxOpenConns(setting.Database.MaxOpenConns)
+	engine.SetMaxIdleConns(setting.Database.MaxIdleConns)
+	engine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
+	engine.Sync2(table...)
+	MigrateCustom(engine)
+	return nil
+}
+
 func SetEngine() (err error) {
-	x, err = getEngine()
+	x, err = getEngine(setting.Database)
 	if err != nil {
 		return fmt.Errorf("Failed to connect to database: %v", err)
 	}
+	if err = setEngine(x, tables, setting.Database); err != nil {
+		return err
+	}
+
+	xStatistic, err = getEngine(setting.DatabaseStatistic)
+	if err != nil {
+		return fmt.Errorf("Failed to connect to database: %v", err)
+	}
+	if err = setEngine(xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil {
+		return err
+	}
 
-	x.SetMapper(names.GonicMapper{})
-	// WARNING: for serv command, MUST remove the output to os.stdout,
-	// so use log file to instead print to stdout.
-	x.SetLogger(NewXORMLogger(setting.Database.LogSQL))
-	x.ShowSQL(setting.Database.LogSQL)
-	x.SetMaxOpenConns(setting.Database.MaxOpenConns)
-	x.SetMaxIdleConns(setting.Database.MaxIdleConns)
-	x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
-	x.Sync2(tables...)
-	MigrateCustom(x)
 	return nil
 }
 
-// NewEngine initializes a new xorm.Engine
 func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
-	if err = SetEngine(); err != nil {
+	x, err = getEngine(setting.Database)
+	if err != nil {
+		return fmt.Errorf("Failed to connect to database: %v", err)
+	}
+	if err = newEngine(ctx, migrateFunc, x, tables, setting.Database); err != nil {
+		return fmt.Errorf("newEngine failed: %v", err)
+	}
+
+	xStatistic, err = getEngine(setting.DatabaseStatistic)
+	if err != nil {
+		return fmt.Errorf("Failed to connect to database: %v", err)
+	}
+	if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil {
+		return fmt.Errorf("newEngine statistic failed: %v", err)
+	}
+
+	HasEngine = true
+
+	return nil
+}
+
+// newEngine initializes a new xorm.Engine
+func newEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error, engine *xorm.Engine, table []interface{}, database *setting.DBInfo) (err error) {
+	if err = setEngine(engine, table, database); err != nil {
 		return err
 	}
 
-	x.SetDefaultContext(ctx)
+	engine.SetDefaultContext(ctx)
 
-	if err = x.Ping(); err != nil {
+	if err = engine.Ping(); err != nil {
 		return err
 	}
 
-	if err = migrateFunc(x); err != nil {
+	if err = migrateFunc(engine); err != nil {
 		return fmt.Errorf("migrate: %v", err)
 	}
 
-	if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil {
+	if err = engine.StoreEngine("InnoDB").Sync2(table...); err != nil {
 		return fmt.Errorf("sync database struct error: %v", err)
 	}
 
@@ -258,6 +299,11 @@ func Ping() error {
 	if x != nil {
 		return x.Ping()
 	}
+
+	if xStatistic != nil {
+		return xStatistic.Ping()
+	}
+
 	return errors.New("database not configured")
 }
 

modules/setting/database.go

@@ -24,111 +24,119 @@ var (
 	EnableSQLite3 bool
 
 	// Database holds the database settings
-	Database = struct {
-		Type              string
-		Host              string
-		Name              string
-		User              string
-		Passwd            string
-		Schema            string
-		SSLMode           string
-		Path              string
-		LogSQL            bool
-		Charset           string
-		Timeout           int // seconds
-		UseSQLite3        bool
-		UseMySQL          bool
-		UseMSSQL          bool
-		UsePostgreSQL     bool
-		DBConnectRetries  int
-		DBConnectBackoff  time.Duration
-		MaxIdleConns      int
-		MaxOpenConns      int
-		ConnMaxLifetime   time.Duration
-		IterateBufferSize int
-	}{
-		Timeout: 500,
-	}
+	Database *DBInfo
+	DatabaseStatistic *DBInfo
 )
 
+type DBInfo struct {
+	Type              string
+	Host              string
+	Name              string
+	User              string
+	Passwd            string
+	Schema            string
+	SSLMode           string
+	Path              string
+	LogSQL            bool
+	Charset           string
+	Timeout           int // seconds
+	UseSQLite3        bool
+	UseMySQL          bool
+	UseMSSQL          bool
+	UsePostgreSQL     bool
+	DBConnectRetries  int
+	DBConnectBackoff  time.Duration
+	MaxIdleConns      int
+	MaxOpenConns      int
+	ConnMaxLifetime   time.Duration
+	IterateBufferSize int
+}
+
 // GetDBTypeByName returns the dataase type as it defined on XORM according the given name
 func GetDBTypeByName(name string) string {
 	return dbTypes[name]
 }
 
-// InitDBConfig loads the database settings
-func InitDBConfig() {
-	sec := Cfg.Section("database")
-	Database.Type = sec.Key("DB_TYPE").String()
-	switch Database.Type {
+// initDBConfig loads the database settings
+func initDBConfig(section string, database *DBInfo) {
+	sec := Cfg.Section(section)
+	database.Type = sec.Key("DB_TYPE").String()
+	switch database.Type {
 	case "sqlite3":
-		Database.UseSQLite3 = true
+		database.UseSQLite3 = true
 	case "mysql":
-		Database.UseMySQL = true
+		database.UseMySQL = true
 	case "postgres":
-		Database.UsePostgreSQL = true
+		database.UsePostgreSQL = true
 	case "mssql":
-		Database.UseMSSQL = true
+		database.UseMSSQL = true
 	}
-	Database.Host = sec.Key("HOST").String()
-	Database.Name = sec.Key("NAME").String()
-	Database.User = sec.Key("USER").String()
-	if len(Database.Passwd) == 0 {
-		Database.Passwd = sec.Key("PASSWD").String()
+	database.Host = sec.Key("HOST").String()
+	database.Name = sec.Key("NAME").String()
+	database.User = sec.Key("USER").String()
+	if len(database.Passwd) == 0 {
+		database.Passwd = sec.Key("PASSWD").String()
 	}
-	Database.Schema = sec.Key("SCHEMA").String()
-	Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
-	Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
-	Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
-	Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
-	Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
-	if Database.UseMySQL {
-		Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second)
+	database.Schema = sec.Key("SCHEMA").String()
+	database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
+	database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
+	database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
+	database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
+	database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
+	if database.UseMySQL {
+		database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second)
 	} else {
-		Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(0)
+		database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(0)
 	}
-	Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
+	database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
 
-	Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
-	Database.LogSQL = sec.Key("LOG_SQL").MustBool(true)
-	Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
-	Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
+	database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
+	database.LogSQL = sec.Key("LOG_SQL").MustBool(true)
+	database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
+	database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
+}
+
+func InitDBConfig() {
+	Database = new(DBInfo)
+	DatabaseStatistic = new(DBInfo)
+	initDBConfig("database", Database)
+	initDBConfig("database_statistic", DatabaseStatistic)
 }
 
 // DBConnStr returns database connection string
-func DBConnStr() (string, error) {
+func DBConnStr(database *DBInfo) (string, error) {
 	connStr := ""
 	var Param = "?"
-	if strings.Contains(Database.Name, Param) {
+	if strings.Contains(database.Name, Param) {
 		Param = "&"
 	}
-	switch Database.Type {
+	switch database.Type {
 	case "mysql":
 		connType := "tcp"
-		if Database.Host[0] == '/' { // looks like a unix socket
+		if database.Host[0] == '/' { // looks like a unix socket
 			connType = "unix"
 		}
-		tls := Database.SSLMode
+		tls := database.SSLMode
 		if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
 			tls = "false"
 		}
 		connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
-			Database.User, Database.Passwd, connType, Database.Host, Database.Name, Param, Database.Charset, tls)
+			database.User, database.Passwd, connType, database.Host, database.Name, Param, database.Charset, tls)
 	case "postgres":
-		connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Param, Database.SSLMode)
+		connStr = getPostgreSQLConnectionString(database.Host, database.User, database.Passwd, database.Name, Param, database.SSLMode)
 	case "mssql":
-		host, port := ParseMSSQLHostPort(Database.Host)
-		connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd)
+		host, port := ParseMSSQLHostPort(database.Host)
+		connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, database.Name, database.User, database.Passwd)
 	case "sqlite3":
 		if !EnableSQLite3 {
 			return "", errors.New("this binary version does not build support for SQLite3")
 		}
-		if err := os.MkdirAll(path.Dir(Database.Path), os.ModePerm); err != nil {
+		if err := os.MkdirAll(path.Dir(database.Path), os.ModePerm); err != nil {
 			return "", fmt.Errorf("Failed to create directories: %v", err)
 		}
-		connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", Database.Path, Database.Timeout)
+		connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", database.Path, database.Timeout)
 	default:
-		return "", fmt.Errorf("Unknown database type: %s", Database.Type)
+		return "", fmt.Errorf("Unknown database type: %s", database.Type)
 	}
 
 	return connStr, nil