diff --git a/README.md b/README.md index 8a270f4fa..afb2da5d9 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

logoAiForge - 启智AI开发协作平台

-[![release](https://img.shields.io/badge/release-V1.21.8-blue)](https://git.openi.org.cn/OpenI/aiforge/releases/latest) +[![release](https://img.shields.io/badge/release-1.21.10.1-blue)](https://git.openi.org.cn/OpenI/aiforge/releases/latest) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) diff --git a/models/models.go b/models/models.go index c164cd911..32fed9b80 100755 --- a/models/models.go +++ b/models/models.go @@ -59,6 +59,9 @@ var ( x *xorm.Engine tables []interface{} + xStatistic *xorm.Engine + tablesStatistic []interface{} + // HasEngine specifies if we have a xorm.Engine HasEngine bool ) @@ -133,14 +136,17 @@ func init() { new(UserBusinessAnalysis), ) + tablesStatistic = append(tablesStatistic, + new(FileChunk)) + gonicNames := []string{"SSL", "UID"} for _, name := range gonicNames { names.LintGonicMapper[name] = true } } -func getEngine() (*xorm.Engine, error) { - connStr, err := setting.DBConnStr() +func getEngine(database *setting.DBInfo) (*xorm.Engine, error) { + connStr, err := setting.DBConnStr(database) if err != nil { return nil, err } @@ -154,14 +160,12 @@ func getEngine() (*xorm.Engine, error) { } engine.SetSchema(setting.Database.Schema) - HasEngine = true - return engine, nil } // NewTestEngine sets a new test xorm.Engine func NewTestEngine(x *xorm.Engine) (err error) { - x, err = getEngine() + x, err = getEngine(setting.Database) if err != nil { return fmt.Errorf("Connect to database: %v", err) } @@ -172,43 +176,80 @@ func NewTestEngine(x *xorm.Engine) (err error) { return x.StoreEngine("InnoDB").Sync2(tables...) } -// SetEngine sets the xorm.Engine +// setEngine sets the xorm.Engine +func setEngine(engine *xorm.Engine, table []interface{}, database *setting.DBInfo) (err error) { + engine.SetMapper(names.GonicMapper{}) + // WARNING: for serv command, MUST remove the output to os.stdout, + // so use log file to instead print to stdout. + engine.SetLogger(NewXORMLogger(setting.Database.LogSQL)) + engine.ShowSQL(setting.Database.LogSQL) + engine.SetMaxOpenConns(setting.Database.MaxOpenConns) + engine.SetMaxIdleConns(setting.Database.MaxIdleConns) + engine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) + engine.Sync2(table...) + MigrateCustom(engine) + return nil +} + func SetEngine() (err error) { - x, err = getEngine() + x, err = getEngine(setting.Database) if err != nil { return fmt.Errorf("Failed to connect to database: %v", err) } + if err = setEngine(x, tables, setting.Database); err != nil { + return err + } + + xStatistic, err = getEngine(setting.DatabaseStatistic) + if err != nil { + return fmt.Errorf("Failed to connect to database: %v", err) + } + if err = setEngine(xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { + return err + } - x.SetMapper(names.GonicMapper{}) - // WARNING: for serv command, MUST remove the output to os.stdout, - // so use log file to instead print to stdout. - x.SetLogger(NewXORMLogger(setting.Database.LogSQL)) - x.ShowSQL(setting.Database.LogSQL) - x.SetMaxOpenConns(setting.Database.MaxOpenConns) - x.SetMaxIdleConns(setting.Database.MaxIdleConns) - x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) - x.Sync2(tables...) - MigrateCustom(x) return nil } -// NewEngine initializes a new xorm.Engine func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { - if err = SetEngine(); err != nil { + x, err = getEngine(setting.Database) + if err != nil { + return fmt.Errorf("Failed to connect to database: %v", err) + } + if err = newEngine(ctx, migrateFunc, x, tables, setting.Database); err != nil { + return fmt.Errorf("newEngine failed: %v", err) + } + + xStatistic, err = getEngine(setting.DatabaseStatistic) + if err != nil { + return fmt.Errorf("Failed to connect to database: %v", err) + } + if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil { + return fmt.Errorf("newEngine statistic failed: %v", err) + } + + HasEngine = true + + return nil +} + +// newEngine initializes a new xorm.Engine +func newEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error, engine *xorm.Engine, table []interface{}, database *setting.DBInfo) (err error) { + if err = setEngine(engine, table, database); err != nil { return err } - x.SetDefaultContext(ctx) + engine.SetDefaultContext(ctx) - if err = x.Ping(); err != nil { + if err = engine.Ping(); err != nil { return err } - if err = migrateFunc(x); err != nil { + if err = migrateFunc(engine); err != nil { return fmt.Errorf("migrate: %v", err) } - if err = x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + if err = engine.StoreEngine("InnoDB").Sync2(table...); err != nil { return fmt.Errorf("sync database struct error: %v", err) } @@ -258,6 +299,11 @@ func Ping() error { if x != nil { return x.Ping() } + + if xStatistic != nil { + return xStatistic.Ping() + } + return errors.New("database not configured") } diff --git a/modules/setting/database.go b/modules/setting/database.go old mode 100644 new mode 100755 index 85043e8c1..e4c3a4149 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -24,111 +24,119 @@ var ( EnableSQLite3 bool // Database holds the database settings - Database = struct { - Type string - Host string - Name string - User string - Passwd string - Schema string - SSLMode string - Path string - LogSQL bool - Charset string - Timeout int // seconds - UseSQLite3 bool - UseMySQL bool - UseMSSQL bool - UsePostgreSQL bool - DBConnectRetries int - DBConnectBackoff time.Duration - MaxIdleConns int - MaxOpenConns int - ConnMaxLifetime time.Duration - IterateBufferSize int - }{ - Timeout: 500, - } + Database *DBInfo + DatabaseStatistic *DBInfo ) +type DBInfo struct { + Type string + Host string + Name string + User string + Passwd string + Schema string + SSLMode string + Path string + LogSQL bool + Charset string + Timeout int // seconds + UseSQLite3 bool + UseMySQL bool + UseMSSQL bool + UsePostgreSQL bool + DBConnectRetries int + DBConnectBackoff time.Duration + MaxIdleConns int + MaxOpenConns int + ConnMaxLifetime time.Duration + IterateBufferSize int +} + // GetDBTypeByName returns the dataase type as it defined on XORM according the given name func GetDBTypeByName(name string) string { return dbTypes[name] } -// InitDBConfig loads the database settings -func InitDBConfig() { - sec := Cfg.Section("database") - Database.Type = sec.Key("DB_TYPE").String() - switch Database.Type { +// initDBConfig loads the database settings +func initDBConfig(section string, database *DBInfo) { + sec := Cfg.Section(section) + database.Type = sec.Key("DB_TYPE").String() + switch database.Type { case "sqlite3": - Database.UseSQLite3 = true + database.UseSQLite3 = true case "mysql": - Database.UseMySQL = true + database.UseMySQL = true case "postgres": - Database.UsePostgreSQL = true + database.UsePostgreSQL = true case "mssql": - Database.UseMSSQL = true + database.UseMSSQL = true } - Database.Host = sec.Key("HOST").String() - Database.Name = sec.Key("NAME").String() - Database.User = sec.Key("USER").String() - if len(Database.Passwd) == 0 { - Database.Passwd = sec.Key("PASSWD").String() + database.Host = sec.Key("HOST").String() + database.Name = sec.Key("NAME").String() + database.User = sec.Key("USER").String() + if len(database.Passwd) == 0 { + database.Passwd = sec.Key("PASSWD").String() } - Database.Schema = sec.Key("SCHEMA").String() - Database.SSLMode = sec.Key("SSL_MODE").MustString("disable") - Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"}) - Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) - Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) - Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) - if Database.UseMySQL { - Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second) + database.Schema = sec.Key("SCHEMA").String() + database.SSLMode = sec.Key("SSL_MODE").MustString("disable") + database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"}) + database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db")) + database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500) + database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) + if database.UseMySQL { + database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second) } else { - Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(0) + database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(0) } - Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0) + database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0) - Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50) - Database.LogSQL = sec.Key("LOG_SQL").MustBool(true) - Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10) - Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second) + database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50) + database.LogSQL = sec.Key("LOG_SQL").MustBool(true) + database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10) + database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second) +} + +func InitDBConfig() { + Database = new(DBInfo) + DatabaseStatistic = new(DBInfo) + initDBConfig("database", Database) + initDBConfig("database_statistic", DatabaseStatistic) } // DBConnStr returns database connection string -func DBConnStr() (string, error) { +func DBConnStr(database *DBInfo) (string, error) { connStr := "" var Param = "?" - if strings.Contains(Database.Name, Param) { + if strings.Contains(database.Name, Param) { Param = "&" } - switch Database.Type { + switch database.Type { case "mysql": connType := "tcp" - if Database.Host[0] == '/' { // looks like a unix socket + if database.Host[0] == '/' { // looks like a unix socket connType = "unix" } - tls := Database.SSLMode + tls := database.SSLMode if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL tls = "false" } connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s", - Database.User, Database.Passwd, connType, Database.Host, Database.Name, Param, Database.Charset, tls) + database.User, database.Passwd, connType, database.Host, database.Name, Param, database.Charset, tls) case "postgres": - connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Param, Database.SSLMode) + connStr = getPostgreSQLConnectionString(database.Host, database.User, database.Passwd, database.Name, Param, database.SSLMode) case "mssql": - host, port := ParseMSSQLHostPort(Database.Host) - connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd) + host, port := ParseMSSQLHostPort(database.Host) + connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, database.Name, database.User, database.Passwd) case "sqlite3": if !EnableSQLite3 { return "", errors.New("this binary version does not build support for SQLite3") } - if err := os.MkdirAll(path.Dir(Database.Path), os.ModePerm); err != nil { + if err := os.MkdirAll(path.Dir(database.Path), os.ModePerm); err != nil { return "", fmt.Errorf("Failed to create directories: %v", err) } - connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", Database.Path, Database.Timeout) + connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", database.Path, database.Timeout) default: - return "", fmt.Errorf("Unknown database type: %s", Database.Type) + return "", fmt.Errorf("Unknown database type: %s", database.Type) } return connStr, nil