From d2eed594ea0afba5d140ca296bac33634f1fc65a Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sat, 15 Feb 2014 19:29:53 +0700 Subject: [PATCH 1/6] Migration Driver for SQLite Need more tests. --- pkg/database/migrate/sqlite.go | 135 +++++++++++++++++++++++ pkg/database/migrate/sqlite_test.go | 164 ++++++++++++++++++++++++++++ pkg/database/migrate/util.go | 32 ++++++ 3 files changed, 331 insertions(+) create mode 100644 pkg/database/migrate/sqlite.go create mode 100644 pkg/database/migrate/sqlite_test.go create mode 100644 pkg/database/migrate/util.go diff --git a/pkg/database/migrate/sqlite.go b/pkg/database/migrate/sqlite.go new file mode 100644 index 000000000..a2074b08e --- /dev/null +++ b/pkg/database/migrate/sqlite.go @@ -0,0 +1,135 @@ +package migrate + +import ( + "database/sql" + "fmt" + "strings" + + _ "github.com/mattn/go-sqlite3" + "github.com/dchest/uniuri" +) + +type SQLiteDriver MigrationDriver + +func SQLite(tx *sql.Tx) Operation { + return &SQLiteDriver{Tx: tx} +} + +func (s *SQLiteDriver) CreateTable(tableName string, args []string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("CREATE TABLE %s (%s);", tableName, strings.Join(args, ", "))) +} + +func (s *SQLiteDriver) RenameTable(tableName, newName string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s;", tableName, newName)) +} + +func (s *SQLiteDriver) DropTable(tableName string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s;", tableName)) +} + +func (s *SQLiteDriver) AddColumn(tableName, columnSpec string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s;", tableName, columnSpec)) +} + +func (s *SQLiteDriver) DropColumns(tableName string, columnsToDrop []string) (sql.Result, error) { + + if len(columnsToDrop) == 0 { + return nil, fmt.Errorf("No columns to drop.") + } + + sql, err := s.getDDLFromTable(tableName) + if err != nil { + return nil, err + } + + columns, err := fetchColumns(sql) + if err != nil { + return nil, err + } + + columnNames := selectName(columns) + preparedColumns := make([]string, len(columnNames)-len(columnsToDrop)) + for k, column := range columnNames { + listed := false + for _, dropped := range columnsToDrop { + if column == dropped { + listed = true + break + } + } + if !listed { + preparedColumns = append(preparedColumns, columns[k]) + } + } + + if len(preparedColumns) == 0 { + return nil, fmt.Errorf("No columns match, drops nothing.") + } + + // Rename old table, here's our proxy + proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16)) + if result, err := s.RenameTable(tableName, proxyName); err != nil { + return result, err + } + + // Recreate table with dropped columns omitted + if result, err := s.CreateTable(tableName, preparedColumns); err != nil { + return result, err + } + + // Move data from old table + if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s;", tableName, + strings.Join(selectName(preparedColumns), ", "), proxyName)); err != nil { + return result, err + } + + // Clean up proxy table + return s.Tx.Exec(fmt.Sprintf("DROP TABLE %s;", proxyName)) +} + +func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) { + sql, err := s.getDDLFromTable(tableName) + if err != nil { + return nil, err + } + + columns, err := fetchColumns(sql) + if err != nil { + return nil, err + } + + oldColumns := make([]string, len(columnChanges)) + newColumns := make([]string, len(columnChanges)) + for k, column := range selectName(columns) { + for Old, New := range columnChanges { + if column == Old { + columnToAdd := strings.Replace(columns[k], Old, New, 1) + + if results, err := s.AddColumn(tableName, columnToAdd); err != nil { + return results, err + } + + oldColumns = append(oldColumns, Old) + newColumns = append(newColumns, New) + break + } + } + } + + statement := fmt.Sprintf("UPDATE %s SET %s;", tableName, setForUpdate(oldColumns, newColumns)) + if results, err := s.Tx.Exec(statement); err != nil { + return results, err + } + + return s.DropColumns(tableName, oldColumns) +} + +func (s *SQLiteDriver) getDDLFromTable(tableName string) (string, error) { + var sql string + query := `SELECT sql FROM sqlite_master WHERE type='table' and name='?';` + err := s.Tx.QueryRow(query, tableName).Scan(&sql) + if err != nil { + return "", err + } + return sql, nil +} diff --git a/pkg/database/migrate/sqlite_test.go b/pkg/database/migrate/sqlite_test.go new file mode 100644 index 000000000..ab7d8492b --- /dev/null +++ b/pkg/database/migrate/sqlite_test.go @@ -0,0 +1,164 @@ +package migrate + +import ( + "database/sql" + "os" + "testing" + + "github.com/russross/meddler" +) + +type Sample struct { + ID int64 `meddler:"id,pk"` + Imel string `meddler:"imel"` + Name string `meddler:"name"` +} + +type RenameSample struct { + ID int64 `meddler:"id,pk"` + Email string `meddler:"email"` + Name string `meddler:"name"` +} + +type AddColumnSample struct { + ID int64 `meddler:"id,pk"` + Imel string `meddler:"imel"` + Name string `meddler:"name"` + Num int64 `meddler:"num"` +} + +type RemoveColumnSample struct { + ID int64 `meddler:"id,pk"` + Name string `meddler:"name"` +} + +// ---------- revision 1 + +type revision1 struct{} + +func (r *revision1) Up(op Operation) error { + _, err := op.CreateTable("samples", []string{ + "id INTEGER PRIMARY KEY AUTOINCREMENT", + "imel VARCHAR(255) UNIQUE", + "name VARCHAR(255)", + }) + return err +} + +func (r *revision1) Down(op Operation) error { + _, err := op.DropTable("samples") + return err +} + +func (r *revision1) Revision() int64 { + return 1 +} + +// ---------- end of revision 1 + +// ---------- revision 2 + +type revision2 struct{} + +func (r *revision2) Up(op Operation) error { + _, err := op.RenameTable("samples", "examples") + return err +} + +func (r *revision2) Down(op Operation) error { + _, err := op.RenameTable("examples", "samples") + return err +} + +func (r *revision2) Revision() int64 { + return 2 +} + +// ---------- end of revision 2 + +var db *sql.DB + +var testSchema = ` +CREATE TABLE samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + imel VARCHAR(255) UNIQUE, + name VARCHAR(255), +); +` + +var dataDump = []string{ + `INSERT INTO samples (imel, name) VALUES ('test@example.com', 'Test Tester');`, + `INSERT INTO samples (imel, name) VALUES ('foo@bar.com', 'Foo Bar');`, + `INSERT INTO samples (imel, name) VALUES ('crash@bandicoot.io', 'Crash Bandicoot');`, +} + +func TestMigrateCreateTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + sample := Sample{ + ID: 1, + Imel: "test@example.com", + Name: "Test Tester", + } + if err := meddler.Save(db, "samples", &sample); err != nil { + t.Errorf("Can not save data: %q", err) + } +} + +func TestMigrateRenameTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + loadFixture(t) + + if err := mgr.Add(&revision2{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + sample := Sample{} + if err := meddler.QueryRow(db, &sample, `SELECT * FROM examples WHERE id = ?`, 2); err != nil { + t.Errorf("Can not fetch data: %q", err) + } + + if sample.Imel != "foo@bar.com" { + t.Errorf("Column doesn't match\n\texpect:\t%s\n\tget:\t%s", "foo@bar.com", sample.Imel) + } +} + +func setUp() error { + var err error + db, err = sql.Open("sqlite3", "migration_tests.sqlite") + return err +} + +func tearDown() { + db.Close() + os.Remove("migration_tests.sqlite") +} + +func loadFixture(t *testing.T) { + for _, sql := range dataDump { + if _, err := db.Exec(sql); err != nil { + t.Errorf("Can not insert into database: %q", err) + } + } +} diff --git a/pkg/database/migrate/util.go b/pkg/database/migrate/util.go new file mode 100644 index 000000000..1dfec95d7 --- /dev/null +++ b/pkg/database/migrate/util.go @@ -0,0 +1,32 @@ +package migrate + +import ( + "fmt" + "strings" +) + +func fetchColumns(sql string) ([]string, error) { + if !strings.HasPrefix(sql, "CREATE TABLE ") { + return []string{}, fmt.Errorf("Sql input is not a DDL statement.") + } + + parenIdx := strings.Index(sql, "(") + return strings.Split(sql[parenIdx+1:len(sql)-1], ","), nil +} + +func selectName(columns []string) []string { + results := make([]string, len(columns)) + for _, column := range columns { + col := strings.SplitN(strings.Trim(column, " \n\t"), " ", 2) + results = append(results, col[0]) + } + return results +} + +func setForUpdate(left []string, right []string) string { + results := make([]string, len(left)) + for k, str := range left { + results = append(results, fmt.Sprintf("%s = %s", str, right[k])) + } + return strings.Join(results, ", ") +} From 54a9544044a0a7c6cbcff559dc662eb271bfcdcc Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sat, 15 Feb 2014 20:16:54 +0700 Subject: [PATCH 2/6] Integrate MigrationDriver to migrate.go --- pkg/database/migrate/migrate.go | 36 +++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/pkg/database/migrate/migrate.go b/pkg/database/migrate/migrate.go index 715596ad2..66bc96095 100644 --- a/pkg/database/migrate/migrate.go +++ b/pkg/database/migrate/migrate.go @@ -49,17 +49,41 @@ const deleteRevisionStmt = ` DELETE FROM migration where revision = ? ` +// Operation interface covers basic migration operations. +// Implementation details is specific for each database, +// see migrate/sqlite.go for implementation reference. +type Operation interface { + + CreateTable(tableName string, args []string) (sql.Result, error) + + RenameTable(tableName, newName string) (sql.Result, error) + + DropTable(tableName string) (sql.Result, error) + + AddColumn(tableName, columnSpec string) (sql.Result, error) + + DropColumns(tableName string, columnsToDrop []string) (sql.Result, error) + + RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) +} + type Revision interface { - Up(tx *sql.Tx) error - Down(tx *sql.Tx) error + Up(op Operation) error + Down(op Operation) error Revision() int64 } +type MigrationDriver struct { + Tx *sql.Tx +} + type Migration struct { db *sql.DB revs []Revision } +var Driver func(tx *sql.Tx) Operation + func New(db *sql.DB) *Migration { return &Migration{db: db} } @@ -119,12 +143,14 @@ func (m *Migration) up(target, current int64) error { return err } + op := Driver(tx) + // loop through and execute revisions for _, rev := range m.revs { if rev.Revision() >= target { current = rev.Revision() // execute the revision Upgrade. - if err := rev.Up(tx); err != nil { + if err := rev.Up(op); err != nil { log.Printf("Failed to upgrade to Revision Number %v\n", current) log.Println(err) return tx.Rollback() @@ -150,6 +176,8 @@ func (m *Migration) down(target, current int64) error { return err } + op := Driver(tx) + // reverse the list of revisions revs := []Revision{} for _, rev := range m.revs { @@ -162,7 +190,7 @@ func (m *Migration) down(target, current int64) error { if rev.Revision() > target { current = rev.Revision() // execute the revision Upgrade. - if err := rev.Down(tx); err != nil { + if err := rev.Down(op); err != nil { log.Printf("Failed to downgrade to Revision Number %v\n", current) log.Println(err) return tx.Rollback() From 4465b2654dd2c0ca439c9e4b7e7aff6a61b36c25 Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sat, 15 Feb 2014 22:17:22 +0700 Subject: [PATCH 3/6] Fix migration step not checked against current version. Add tests for DropColumns. --- pkg/database/migrate/migrate.go | 7 ++- pkg/database/migrate/sqlite.go | 11 ++--- pkg/database/migrate/sqlite_test.go | 68 +++++++++++++++++++++++++++++ pkg/database/migrate/util.go | 4 +- 4 files changed, 79 insertions(+), 11 deletions(-) diff --git a/pkg/database/migrate/migrate.go b/pkg/database/migrate/migrate.go index 66bc96095..cc96f7ed7 100644 --- a/pkg/database/migrate/migrate.go +++ b/pkg/database/migrate/migrate.go @@ -53,7 +53,6 @@ DELETE FROM migration where revision = ? // Implementation details is specific for each database, // see migrate/sqlite.go for implementation reference. type Operation interface { - CreateTable(tableName string, args []string) (sql.Result, error) RenameTable(tableName, newName string) (sql.Result, error) @@ -147,7 +146,7 @@ func (m *Migration) up(target, current int64) error { // loop through and execute revisions for _, rev := range m.revs { - if rev.Revision() >= target { + if rev.Revision() > current { current = rev.Revision() // execute the revision Upgrade. if err := rev.Up(op); err != nil { @@ -191,7 +190,7 @@ func (m *Migration) down(target, current int64) error { current = rev.Revision() // execute the revision Upgrade. if err := rev.Down(op); err != nil { - log.Printf("Failed to downgrade to Revision Number %v\n", current) + log.Printf("Failed to downgrade from Revision Number %v\n", current) log.Println(err) return tx.Rollback() } @@ -202,7 +201,7 @@ func (m *Migration) down(target, current int64) error { return tx.Rollback() } - log.Printf("Successfully downgraded to Revision %v\n", current) + log.Printf("Successfully downgraded from Revision %v\n", current) } } diff --git a/pkg/database/migrate/sqlite.go b/pkg/database/migrate/sqlite.go index a2074b08e..d79e76deb 100644 --- a/pkg/database/migrate/sqlite.go +++ b/pkg/database/migrate/sqlite.go @@ -5,8 +5,8 @@ import ( "fmt" "strings" - _ "github.com/mattn/go-sqlite3" "github.com/dchest/uniuri" + _ "github.com/mattn/go-sqlite3" ) type SQLiteDriver MigrationDriver @@ -48,7 +48,8 @@ func (s *SQLiteDriver) DropColumns(tableName string, columnsToDrop []string) (sq } columnNames := selectName(columns) - preparedColumns := make([]string, len(columnNames)-len(columnsToDrop)) + + var preparedColumns []string for k, column := range columnNames { listed := false for _, dropped := range columnsToDrop { @@ -98,8 +99,8 @@ func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string] return nil, err } - oldColumns := make([]string, len(columnChanges)) - newColumns := make([]string, len(columnChanges)) + var oldColumns []string + var newColumns []string for k, column := range selectName(columns) { for Old, New := range columnChanges { if column == Old { @@ -126,7 +127,7 @@ func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string] func (s *SQLiteDriver) getDDLFromTable(tableName string) (string, error) { var sql string - query := `SELECT sql FROM sqlite_master WHERE type='table' and name='?';` + query := `SELECT sql FROM sqlite_master WHERE type='table' and name=?;` err := s.Tx.QueryRow(query, tableName).Scan(&sql) if err != nil { return "", err diff --git a/pkg/database/migrate/sqlite_test.go b/pkg/database/migrate/sqlite_test.go index ab7d8492b..b9b4d9e11 100644 --- a/pkg/database/migrate/sqlite_test.go +++ b/pkg/database/migrate/sqlite_test.go @@ -76,6 +76,29 @@ func (r *revision2) Revision() int64 { // ---------- end of revision 2 +// ---------- revision 3 + +type revision3 struct{} + +func (r *revision3) Up(op Operation) error { + if _, err := op.AddColumn("samples", "url VARCHAR(255)"); err != nil { + return err + } + _, err := op.AddColumn("samples", "likes INTEGER") + return err +} + +func (r *revision3) Down(op Operation) error { + _, err := op.DropColumns("samples", []string{"likes", "url"}) + return err +} + +func (r *revision3) Revision() int64 { + return 3 +} + +// ---------- end of revision 3 + var db *sql.DB var testSchema = ` @@ -144,6 +167,51 @@ func TestMigrateRenameTable(t *testing.T) { } } +type TableInfo struct { + CID int64 `meddler:"cid,pk"` + Name string `meddler:"name"` + Type string `meddler:"type"` + Notnull bool `meddler:"notnull"` + DfltValue interface{} `meddler:"dflt_value"` + PK bool `meddler:"pk"` +} + +func TestMigrateAddRemoveColumns(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Add(&revision3{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var columns []*TableInfo + if err := meddler.QueryAll(db, &columns, `PRAGMA table_info(samples);`); err != nil { + t.Errorf("Can not access table info: %q", err) + } + + if len(columns) < 5 { + t.Errorf("Expect length columns: %d\nGot: %d", 5, len(columns)) + } + + if err := mgr.MigrateTo(1); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var another_columns []*TableInfo + if err := meddler.QueryAll(db, &another_columns, `PRAGMA table_info(samples);`); err != nil { + t.Errorf("Can not access table info: %q", err) + } + + if len(another_columns) != 3 { + t.Errorf("Expect length columns: %d\nGot: %d", 3, len(columns)) + } +} + func setUp() error { var err error db, err = sql.Open("sqlite3", "migration_tests.sqlite") diff --git a/pkg/database/migrate/util.go b/pkg/database/migrate/util.go index 1dfec95d7..a0f6bfb59 100644 --- a/pkg/database/migrate/util.go +++ b/pkg/database/migrate/util.go @@ -15,7 +15,7 @@ func fetchColumns(sql string) ([]string, error) { } func selectName(columns []string) []string { - results := make([]string, len(columns)) + var results []string for _, column := range columns { col := strings.SplitN(strings.Trim(column, " \n\t"), " ", 2) results = append(results, col[0]) @@ -24,7 +24,7 @@ func selectName(columns []string) []string { } func setForUpdate(left []string, right []string) string { - results := make([]string, len(left)) + var results []string for k, str := range left { results = append(results, fmt.Sprintf("%s = %s", str, right[k])) } From 8ce87f0d2c08c73d84575b1cfc181b3111c4241a Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sun, 16 Feb 2014 00:56:03 +0700 Subject: [PATCH 4/6] More tests for alter columns migration Also change the way we handle columns rename migration. SQLite restrict column addition not to have PRIMARY KEY and/or UNIQUE attribute, so we have to change from: add new column -> migrate data from old column to new column -> rename old table -> create new table with old columns removed -> migrate data from old table to the new table -> drop old table to directly: rename old table -> create new table with renamed columns -> migrate data from old table to the new table -> drop old table --- pkg/database/migrate/migrate.go | 2 +- pkg/database/migrate/sqlite.go | 44 ++++++++++------ pkg/database/migrate/sqlite_test.go | 78 +++++++++++++++++++++++++---- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/pkg/database/migrate/migrate.go b/pkg/database/migrate/migrate.go index cc96f7ed7..f4a286d7b 100644 --- a/pkg/database/migrate/migrate.go +++ b/pkg/database/migrate/migrate.go @@ -146,7 +146,7 @@ func (m *Migration) up(target, current int64) error { // loop through and execute revisions for _, rev := range m.revs { - if rev.Revision() > current { + if rev.Revision() > current && rev.Revision() <= target { current = rev.Revision() // execute the revision Upgrade. if err := rev.Up(op); err != nil { diff --git a/pkg/database/migrate/sqlite.go b/pkg/database/migrate/sqlite.go index d79e76deb..512a1e2af 100644 --- a/pkg/database/migrate/sqlite.go +++ b/pkg/database/migrate/sqlite.go @@ -85,7 +85,7 @@ func (s *SQLiteDriver) DropColumns(tableName string, columnsToDrop []string) (sq } // Clean up proxy table - return s.Tx.Exec(fmt.Sprintf("DROP TABLE %s;", proxyName)) + return s.DropTable(proxyName) } func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) { @@ -99,30 +99,46 @@ func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string] return nil, err } - var oldColumns []string + // We need a list of columns name to migrate data to the new table + var oldColumnsName = selectName(columns) + + // newColumns will be used to create the new table var newColumns []string - for k, column := range selectName(columns) { + + for k, column := range oldColumnsName { + added := false for Old, New := range columnChanges { if column == Old { columnToAdd := strings.Replace(columns[k], Old, New, 1) - - if results, err := s.AddColumn(tableName, columnToAdd); err != nil { - return results, err - } - - oldColumns = append(oldColumns, Old) - newColumns = append(newColumns, New) + newColumns = append(newColumns, columnToAdd) + added = true break } } + if !added { + newColumns = append(newColumns, columns[k]) + } } - statement := fmt.Sprintf("UPDATE %s SET %s;", tableName, setForUpdate(oldColumns, newColumns)) - if results, err := s.Tx.Exec(statement); err != nil { - return results, err + // Rename current table + proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16)) + if result, err := s.RenameTable(tableName, proxyName); err != nil { + return result, err } - return s.DropColumns(tableName, oldColumns) + // Create new table with the new columns + if result, err := s.CreateTable(tableName, newColumns); err != nil { + return result, err + } + + // Migrate data + if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s", tableName, + strings.Join(oldColumnsName, ", "), proxyName)); err != nil { + return result, err + } + + // Clean up proxy table + return s.DropTable(proxyName) } func (s *SQLiteDriver) getDDLFromTable(tableName string) (string, error) { diff --git a/pkg/database/migrate/sqlite_test.go b/pkg/database/migrate/sqlite_test.go index b9b4d9e11..17ffd4fe3 100644 --- a/pkg/database/migrate/sqlite_test.go +++ b/pkg/database/migrate/sqlite_test.go @@ -24,14 +24,10 @@ type AddColumnSample struct { ID int64 `meddler:"id,pk"` Imel string `meddler:"imel"` Name string `meddler:"name"` + Url string `meddler:"url"` Num int64 `meddler:"num"` } -type RemoveColumnSample struct { - ID int64 `meddler:"id,pk"` - Name string `meddler:"name"` -} - // ---------- revision 1 type revision1 struct{} @@ -84,12 +80,12 @@ func (r *revision3) Up(op Operation) error { if _, err := op.AddColumn("samples", "url VARCHAR(255)"); err != nil { return err } - _, err := op.AddColumn("samples", "likes INTEGER") + _, err := op.AddColumn("samples", "num INTEGER") return err } func (r *revision3) Down(op Operation) error { - _, err := op.DropColumns("samples", []string{"likes", "url"}) + _, err := op.DropColumns("samples", []string{"num", "url"}) return err } @@ -99,6 +95,30 @@ func (r *revision3) Revision() int64 { // ---------- end of revision 3 +// ---------- revision 4 + +type revision4 struct{} + +func (r *revision4) Up(op Operation) error { + _, err := op.RenameColumns("samples", map[string]string{ + "imel": "email", + }) + return err +} + +func (r *revision4) Down(op Operation) error { + _, err := op.RenameColumns("samples", map[string]string{ + "email": "imel", + }) + return err +} + +func (r *revision4) Revision() int64 { + return 4 +} + +// ---------- + var db *sql.DB var testSchema = ` @@ -163,7 +183,7 @@ func TestMigrateRenameTable(t *testing.T) { } if sample.Imel != "foo@bar.com" { - t.Errorf("Column doesn't match\n\texpect:\t%s\n\tget:\t%s", "foo@bar.com", sample.Imel) + t.Errorf("Column doesn't match. Expect: %s, got: %s", "foo@bar.com", sample.Imel) } } @@ -198,6 +218,17 @@ func TestMigrateAddRemoveColumns(t *testing.T) { t.Errorf("Expect length columns: %d\nGot: %d", 5, len(columns)) } + var row = AddColumnSample{ + ID: 33, + Name: "Foo", + Imel: "foo@bar.com", + Url: "http://example.com", + Num: 42, + } + if err := meddler.Save(db, "samples", &row); err != nil { + t.Errorf("Can not save into database: %q", err) + } + if err := mgr.MigrateTo(1); err != nil { t.Errorf("Can not migrate: %q", err) } @@ -208,7 +239,36 @@ func TestMigrateAddRemoveColumns(t *testing.T) { } if len(another_columns) != 3 { - t.Errorf("Expect length columns: %d\nGot: %d", 3, len(columns)) + t.Errorf("Expect length columns = %d, got: %d", 3, len(columns)) + } +} + +func TestRenameColumn(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Add(&revision4{}).MigrateTo(1); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + loadFixture(t) + + if err := mgr.MigrateTo(4); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + row := RenameSample{} + if err := meddler.QueryRow(db, &row, `SELECT * FROM samples WHERE id = 3;`); err != nil { + t.Errorf("Can not query database: %q", err) + } + + if row.Email != "crash@bandicoot.io" { + t.Errorf("Expect %s, got %s", "crash@bandicoot.io", row.Email) } } From da8d4346eec5b6bec62b993ee83e845a378c909a Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sun, 16 Feb 2014 02:23:26 +0700 Subject: [PATCH 5/6] Add tests to migrate existing database without migration attributes. --- pkg/database/migrate/sqlite_test.go | 41 ++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/pkg/database/migrate/sqlite_test.go b/pkg/database/migrate/sqlite_test.go index 17ffd4fe3..af81588f7 100644 --- a/pkg/database/migrate/sqlite_test.go +++ b/pkg/database/migrate/sqlite_test.go @@ -123,9 +123,9 @@ var db *sql.DB var testSchema = ` CREATE TABLE samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, imel VARCHAR(255) UNIQUE, - name VARCHAR(255), + name VARCHAR(255) ); ` @@ -205,7 +205,7 @@ func TestMigrateAddRemoveColumns(t *testing.T) { Driver = SQLite mgr := New(db) - if err := mgr.Add(&revision1{}).Add(&revision3{}).Migrate(); err != nil { + if err := mgr.Add(&revision1{}, &revision3{}).Migrate(); err != nil { t.Errorf("Can not migrate: %q", err) } @@ -252,7 +252,7 @@ func TestRenameColumn(t *testing.T) { Driver = SQLite mgr := New(db) - if err := mgr.Add(&revision1{}).Add(&revision4{}).MigrateTo(1); err != nil { + if err := mgr.Add(&revision1{}, &revision4{}).MigrateTo(1); err != nil { t.Errorf("Can not migrate: %q", err) } @@ -272,6 +272,39 @@ func TestRenameColumn(t *testing.T) { } } +func TestMigrateExistingTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + if _, err := db.Exec(testSchema); err != nil { + t.Errorf("Can not create database: %q", err) + } + + loadFixture(t) + + mgr := New(db) + if err := mgr.Add(&revision4{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var rows []*RenameSample + if err := meddler.QueryAll(db, &rows, `SELECT * from samples;`); err != nil { + t.Errorf("Can not query database: %q", err) + } + + if len(rows) != 3 { + t.Errorf("Expect rows length = %d, got %d", 3, len(rows)) + } + + if rows[1].Email != "foo@bar.com" { + t.Errorf("Expect email = %s, got %s", "foo@bar.com", rows[1].Email) + } +} + func setUp() error { var err error db, err = sql.Open("sqlite3", "migration_tests.sqlite") From 4ef0f1b437d3ddc09e59c5d5f9c4a4eefdf258cf Mon Sep 17 00:00:00 2001 From: Nurahmadie Date: Sun, 16 Feb 2014 03:47:30 +0700 Subject: [PATCH 6/6] Integrate migrations with drone. Also add migration to tests. --- Makefile | 1 + cmd/droned/drone.go | 7 +++++- ...402200603_rename_privelege_to_privilege.go | 23 +++++++++++++++++++ pkg/database/migrate/all.go | 11 +++++++++ pkg/database/repos.go | 8 +++---- pkg/database/testing/testing.go | 5 ++++ pkg/handler/handler.go | 4 ++-- pkg/model/repo.go | 4 ++-- 8 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 pkg/database/migrate/201402200603_rename_privelege_to_privilege.go create mode 100644 pkg/database/migrate/all.go diff --git a/Makefile b/Makefile index c14a65db9..86b8cf57c 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ test: go test -v github.com/drone/drone/pkg/channel go test -v github.com/drone/drone/pkg/database go test -v github.com/drone/drone/pkg/database/encrypt + go test -v github.com/drone/drone/pkg/database/migrate go test -v github.com/drone/drone/pkg/database/testing go test -v github.com/drone/drone/pkg/mail go test -v github.com/drone/drone/pkg/model diff --git a/cmd/droned/drone.go b/cmd/droned/drone.go index 86b6ba13f..4809903b1 100644 --- a/cmd/droned/drone.go +++ b/cmd/droned/drone.go @@ -15,6 +15,7 @@ import ( "github.com/drone/drone/pkg/channel" "github.com/drone/drone/pkg/database" + "github.com/drone/drone/pkg/database/migrate" "github.com/drone/drone/pkg/handler" ) @@ -55,8 +56,9 @@ func main() { // setup the database connection and register with the // global database package. func setupDatabase() { - // inform meddler we're using sqlite + // inform meddler and migration we're using sqlite meddler.Default = meddler.SQLite + migrate.Driver = migrate.SQLite // connect to the SQLite database db, err := sql.Open(driver, datasource) @@ -65,6 +67,9 @@ func setupDatabase() { } database.Set(db) + + migration := migrate.New(db) + migration.All().Migrate() } // setup routes for static assets. These assets may diff --git a/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go b/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go new file mode 100644 index 000000000..379a6649e --- /dev/null +++ b/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go @@ -0,0 +1,23 @@ +package migrate + +type Rev1 struct{} + +var RenamePrivelegedToPrivileged = &Rev1{} + +func (r *Rev1) Revision() int64 { + return 201402200603 +} + +func (r *Rev1) Up(op Operation) error { + _, err := op.RenameColumns("repos", map[string]string{ + "priveleged": "privileged", + }) + return err +} + +func (r *Rev1) Down(op Operation) error { + _, err := op.RenameColumns("repos", map[string]string{ + "privileged": "priveleged", + }) + return err +} diff --git a/pkg/database/migrate/all.go b/pkg/database/migrate/all.go new file mode 100644 index 000000000..3933a93ca --- /dev/null +++ b/pkg/database/migrate/all.go @@ -0,0 +1,11 @@ +package migrate + +func (m *Migration) All() *Migration { + + // List all migrations here + m.Add(RenamePrivelegedToPrivileged) + + // m.Add(...) + // ... + return m +} diff --git a/pkg/database/repos.go b/pkg/database/repos.go index e4407c028..0d9e2f15e 100644 --- a/pkg/database/repos.go +++ b/pkg/database/repos.go @@ -13,7 +13,7 @@ const repoTable = "repos" // SQL Queries to retrieve a list of all repos belonging to a User. const repoStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE user_id = ? AND team_id = 0 ORDER BY slug ASC @@ -22,7 +22,7 @@ ORDER BY slug ASC // SQL Queries to retrieve a list of all repos belonging to a Team. const repoTeamStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE team_id = ? ORDER BY slug ASC @@ -31,7 +31,7 @@ ORDER BY slug ASC // SQL Queries to retrieve a repo by id. const repoFindStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE id = ? ` @@ -39,7 +39,7 @@ WHERE id = ? // SQL Queries to retrieve a repo by name. const repoFindSlugStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE slug = ? ` diff --git a/pkg/database/testing/testing.go b/pkg/database/testing/testing.go index 11106e2d3..69fde3787 100644 --- a/pkg/database/testing/testing.go +++ b/pkg/database/testing/testing.go @@ -7,6 +7,7 @@ import ( "github.com/drone/drone/pkg/database" "github.com/drone/drone/pkg/database/encrypt" + "github.com/drone/drone/pkg/database/migrate" . "github.com/drone/drone/pkg/model" _ "github.com/mattn/go-sqlite3" @@ -31,6 +32,7 @@ func init() { // notify meddler that we are working with sqlite meddler.Default = meddler.SQLite + migrate.Driver = migrate.SQLite } func Setup() { @@ -40,6 +42,9 @@ func Setup() { // make sure all the tables and indexes are created database.Set(db) + migration := migrate.New(db) + migration.All().Migrate() + // create dummy user data user1 := User{ Password: "$2a$10$b8d63QsTL38vx7lj0HEHfOdbu1PCAg6Gfca74UavkXooIBx9YxopS", diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 2b3d28069..bf9914e4f 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -41,7 +41,7 @@ func (h UserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // AdminHandler wraps the default http.HandlerFunc to include // the currently authenticated User in the method signature, // in addition to handling an error as the return value. It also -// verifies the user has Administrative priveleges. +// verifies the user has Administrative privileges. type AdminHandler func(w http.ResponseWriter, r *http.Request, user *User) error func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -51,7 +51,7 @@ func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // User MUST have administrative priveleges in order + // User MUST have administrative privileges in order // to execute the handler. if user.Admin == false { RenderNotFound(w) diff --git a/pkg/model/repo.go b/pkg/model/repo.go index 5b89ad71c..8d3ab202b 100644 --- a/pkg/model/repo.go +++ b/pkg/model/repo.go @@ -88,9 +88,9 @@ type Repo struct { // before exceeding its timelimit and being killed. Timeout int64 `meddler:"timeout" json:"timeout"` - // Indicates the build should be executed in priveleged + // Indicates the build should be executed in privileged // mode. This could, for example, be used to run Docker in Docker. - Priveleged bool `meddler:"priveleged" json:"priveleged"` + Privileged bool `meddler:"privileged" json:"privileged"` // Foreign keys signify the User that created // the repository and team account linked to