diff --git a/Makefile b/Makefile index d0f03fd8e..9015f82a6 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ test: go test -v github.com/drone/drone/pkg/channel go test -v github.com/drone/drone/pkg/database go test -v github.com/drone/drone/pkg/database/encrypt + go test -v github.com/drone/drone/pkg/database/migrate go test -v github.com/drone/drone/pkg/database/testing go test -v github.com/drone/drone/pkg/mail go test -v github.com/drone/drone/pkg/model diff --git a/cmd/droned/drone.go b/cmd/droned/drone.go index b2574e528..177732221 100644 --- a/cmd/droned/drone.go +++ b/cmd/droned/drone.go @@ -15,6 +15,7 @@ import ( "github.com/drone/drone/pkg/channel" "github.com/drone/drone/pkg/database" + "github.com/drone/drone/pkg/database/migrate" "github.com/drone/drone/pkg/handler" ) @@ -55,8 +56,9 @@ func main() { // setup the database connection and register with the // global database package. func setupDatabase() { - // inform meddler we're using sqlite + // inform meddler and migration we're using sqlite meddler.Default = meddler.SQLite + migrate.Driver = migrate.SQLite // connect to the SQLite database db, err := sql.Open(driver, datasource) @@ -65,6 +67,9 @@ func setupDatabase() { } database.Set(db) + + migration := migrate.New(db) + migration.All().Migrate() } // setup routes for static assets. These assets may diff --git a/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go b/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go new file mode 100644 index 000000000..379a6649e --- /dev/null +++ b/pkg/database/migrate/201402200603_rename_privelege_to_privilege.go @@ -0,0 +1,23 @@ +package migrate + +type Rev1 struct{} + +var RenamePrivelegedToPrivileged = &Rev1{} + +func (r *Rev1) Revision() int64 { + return 201402200603 +} + +func (r *Rev1) Up(op Operation) error { + _, err := op.RenameColumns("repos", map[string]string{ + "priveleged": "privileged", + }) + return err +} + +func (r *Rev1) Down(op Operation) error { + _, err := op.RenameColumns("repos", map[string]string{ + "privileged": "priveleged", + }) + return err +} diff --git a/pkg/database/migrate/all.go b/pkg/database/migrate/all.go new file mode 100644 index 000000000..3933a93ca --- /dev/null +++ b/pkg/database/migrate/all.go @@ -0,0 +1,11 @@ +package migrate + +func (m *Migration) All() *Migration { + + // List all migrations here + m.Add(RenamePrivelegedToPrivileged) + + // m.Add(...) + // ... + return m +} diff --git a/pkg/database/migrate/migrate.go b/pkg/database/migrate/migrate.go index 715596ad2..f4a286d7b 100644 --- a/pkg/database/migrate/migrate.go +++ b/pkg/database/migrate/migrate.go @@ -49,17 +49,40 @@ const deleteRevisionStmt = ` DELETE FROM migration where revision = ? ` +// Operation interface covers basic migration operations. +// Implementation details is specific for each database, +// see migrate/sqlite.go for implementation reference. +type Operation interface { + CreateTable(tableName string, args []string) (sql.Result, error) + + RenameTable(tableName, newName string) (sql.Result, error) + + DropTable(tableName string) (sql.Result, error) + + AddColumn(tableName, columnSpec string) (sql.Result, error) + + DropColumns(tableName string, columnsToDrop []string) (sql.Result, error) + + RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) +} + type Revision interface { - Up(tx *sql.Tx) error - Down(tx *sql.Tx) error + Up(op Operation) error + Down(op Operation) error Revision() int64 } +type MigrationDriver struct { + Tx *sql.Tx +} + type Migration struct { db *sql.DB revs []Revision } +var Driver func(tx *sql.Tx) Operation + func New(db *sql.DB) *Migration { return &Migration{db: db} } @@ -119,12 +142,14 @@ func (m *Migration) up(target, current int64) error { return err } + op := Driver(tx) + // loop through and execute revisions for _, rev := range m.revs { - if rev.Revision() >= target { + if rev.Revision() > current && rev.Revision() <= target { current = rev.Revision() // execute the revision Upgrade. - if err := rev.Up(tx); err != nil { + if err := rev.Up(op); err != nil { log.Printf("Failed to upgrade to Revision Number %v\n", current) log.Println(err) return tx.Rollback() @@ -150,6 +175,8 @@ func (m *Migration) down(target, current int64) error { return err } + op := Driver(tx) + // reverse the list of revisions revs := []Revision{} for _, rev := range m.revs { @@ -162,8 +189,8 @@ func (m *Migration) down(target, current int64) error { if rev.Revision() > target { current = rev.Revision() // execute the revision Upgrade. - if err := rev.Down(tx); err != nil { - log.Printf("Failed to downgrade to Revision Number %v\n", current) + if err := rev.Down(op); err != nil { + log.Printf("Failed to downgrade from Revision Number %v\n", current) log.Println(err) return tx.Rollback() } @@ -174,7 +201,7 @@ func (m *Migration) down(target, current int64) error { return tx.Rollback() } - log.Printf("Successfully downgraded to Revision %v\n", current) + log.Printf("Successfully downgraded from Revision %v\n", current) } } diff --git a/pkg/database/migrate/sqlite.go b/pkg/database/migrate/sqlite.go new file mode 100644 index 000000000..512a1e2af --- /dev/null +++ b/pkg/database/migrate/sqlite.go @@ -0,0 +1,152 @@ +package migrate + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/dchest/uniuri" + _ "github.com/mattn/go-sqlite3" +) + +type SQLiteDriver MigrationDriver + +func SQLite(tx *sql.Tx) Operation { + return &SQLiteDriver{Tx: tx} +} + +func (s *SQLiteDriver) CreateTable(tableName string, args []string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("CREATE TABLE %s (%s);", tableName, strings.Join(args, ", "))) +} + +func (s *SQLiteDriver) RenameTable(tableName, newName string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s;", tableName, newName)) +} + +func (s *SQLiteDriver) DropTable(tableName string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s;", tableName)) +} + +func (s *SQLiteDriver) AddColumn(tableName, columnSpec string) (sql.Result, error) { + return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s;", tableName, columnSpec)) +} + +func (s *SQLiteDriver) DropColumns(tableName string, columnsToDrop []string) (sql.Result, error) { + + if len(columnsToDrop) == 0 { + return nil, fmt.Errorf("No columns to drop.") + } + + sql, err := s.getDDLFromTable(tableName) + if err != nil { + return nil, err + } + + columns, err := fetchColumns(sql) + if err != nil { + return nil, err + } + + columnNames := selectName(columns) + + var preparedColumns []string + for k, column := range columnNames { + listed := false + for _, dropped := range columnsToDrop { + if column == dropped { + listed = true + break + } + } + if !listed { + preparedColumns = append(preparedColumns, columns[k]) + } + } + + if len(preparedColumns) == 0 { + return nil, fmt.Errorf("No columns match, drops nothing.") + } + + // Rename old table, here's our proxy + proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16)) + if result, err := s.RenameTable(tableName, proxyName); err != nil { + return result, err + } + + // Recreate table with dropped columns omitted + if result, err := s.CreateTable(tableName, preparedColumns); err != nil { + return result, err + } + + // Move data from old table + if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s;", tableName, + strings.Join(selectName(preparedColumns), ", "), proxyName)); err != nil { + return result, err + } + + // Clean up proxy table + return s.DropTable(proxyName) +} + +func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) { + sql, err := s.getDDLFromTable(tableName) + if err != nil { + return nil, err + } + + columns, err := fetchColumns(sql) + if err != nil { + return nil, err + } + + // We need a list of columns name to migrate data to the new table + var oldColumnsName = selectName(columns) + + // newColumns will be used to create the new table + var newColumns []string + + for k, column := range oldColumnsName { + added := false + for Old, New := range columnChanges { + if column == Old { + columnToAdd := strings.Replace(columns[k], Old, New, 1) + newColumns = append(newColumns, columnToAdd) + added = true + break + } + } + if !added { + newColumns = append(newColumns, columns[k]) + } + } + + // Rename current table + proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16)) + if result, err := s.RenameTable(tableName, proxyName); err != nil { + return result, err + } + + // Create new table with the new columns + if result, err := s.CreateTable(tableName, newColumns); err != nil { + return result, err + } + + // Migrate data + if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s", tableName, + strings.Join(oldColumnsName, ", "), proxyName)); err != nil { + return result, err + } + + // Clean up proxy table + return s.DropTable(proxyName) +} + +func (s *SQLiteDriver) getDDLFromTable(tableName string) (string, error) { + var sql string + query := `SELECT sql FROM sqlite_master WHERE type='table' and name=?;` + err := s.Tx.QueryRow(query, tableName).Scan(&sql) + if err != nil { + return "", err + } + return sql, nil +} diff --git a/pkg/database/migrate/sqlite_test.go b/pkg/database/migrate/sqlite_test.go new file mode 100644 index 000000000..af81588f7 --- /dev/null +++ b/pkg/database/migrate/sqlite_test.go @@ -0,0 +1,325 @@ +package migrate + +import ( + "database/sql" + "os" + "testing" + + "github.com/russross/meddler" +) + +type Sample struct { + ID int64 `meddler:"id,pk"` + Imel string `meddler:"imel"` + Name string `meddler:"name"` +} + +type RenameSample struct { + ID int64 `meddler:"id,pk"` + Email string `meddler:"email"` + Name string `meddler:"name"` +} + +type AddColumnSample struct { + ID int64 `meddler:"id,pk"` + Imel string `meddler:"imel"` + Name string `meddler:"name"` + Url string `meddler:"url"` + Num int64 `meddler:"num"` +} + +// ---------- revision 1 + +type revision1 struct{} + +func (r *revision1) Up(op Operation) error { + _, err := op.CreateTable("samples", []string{ + "id INTEGER PRIMARY KEY AUTOINCREMENT", + "imel VARCHAR(255) UNIQUE", + "name VARCHAR(255)", + }) + return err +} + +func (r *revision1) Down(op Operation) error { + _, err := op.DropTable("samples") + return err +} + +func (r *revision1) Revision() int64 { + return 1 +} + +// ---------- end of revision 1 + +// ---------- revision 2 + +type revision2 struct{} + +func (r *revision2) Up(op Operation) error { + _, err := op.RenameTable("samples", "examples") + return err +} + +func (r *revision2) Down(op Operation) error { + _, err := op.RenameTable("examples", "samples") + return err +} + +func (r *revision2) Revision() int64 { + return 2 +} + +// ---------- end of revision 2 + +// ---------- revision 3 + +type revision3 struct{} + +func (r *revision3) Up(op Operation) error { + if _, err := op.AddColumn("samples", "url VARCHAR(255)"); err != nil { + return err + } + _, err := op.AddColumn("samples", "num INTEGER") + return err +} + +func (r *revision3) Down(op Operation) error { + _, err := op.DropColumns("samples", []string{"num", "url"}) + return err +} + +func (r *revision3) Revision() int64 { + return 3 +} + +// ---------- end of revision 3 + +// ---------- revision 4 + +type revision4 struct{} + +func (r *revision4) Up(op Operation) error { + _, err := op.RenameColumns("samples", map[string]string{ + "imel": "email", + }) + return err +} + +func (r *revision4) Down(op Operation) error { + _, err := op.RenameColumns("samples", map[string]string{ + "email": "imel", + }) + return err +} + +func (r *revision4) Revision() int64 { + return 4 +} + +// ---------- + +var db *sql.DB + +var testSchema = ` +CREATE TABLE samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + imel VARCHAR(255) UNIQUE, + name VARCHAR(255) +); +` + +var dataDump = []string{ + `INSERT INTO samples (imel, name) VALUES ('test@example.com', 'Test Tester');`, + `INSERT INTO samples (imel, name) VALUES ('foo@bar.com', 'Foo Bar');`, + `INSERT INTO samples (imel, name) VALUES ('crash@bandicoot.io', 'Crash Bandicoot');`, +} + +func TestMigrateCreateTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + sample := Sample{ + ID: 1, + Imel: "test@example.com", + Name: "Test Tester", + } + if err := meddler.Save(db, "samples", &sample); err != nil { + t.Errorf("Can not save data: %q", err) + } +} + +func TestMigrateRenameTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + loadFixture(t) + + if err := mgr.Add(&revision2{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + sample := Sample{} + if err := meddler.QueryRow(db, &sample, `SELECT * FROM examples WHERE id = ?`, 2); err != nil { + t.Errorf("Can not fetch data: %q", err) + } + + if sample.Imel != "foo@bar.com" { + t.Errorf("Column doesn't match. Expect: %s, got: %s", "foo@bar.com", sample.Imel) + } +} + +type TableInfo struct { + CID int64 `meddler:"cid,pk"` + Name string `meddler:"name"` + Type string `meddler:"type"` + Notnull bool `meddler:"notnull"` + DfltValue interface{} `meddler:"dflt_value"` + PK bool `meddler:"pk"` +} + +func TestMigrateAddRemoveColumns(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}, &revision3{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var columns []*TableInfo + if err := meddler.QueryAll(db, &columns, `PRAGMA table_info(samples);`); err != nil { + t.Errorf("Can not access table info: %q", err) + } + + if len(columns) < 5 { + t.Errorf("Expect length columns: %d\nGot: %d", 5, len(columns)) + } + + var row = AddColumnSample{ + ID: 33, + Name: "Foo", + Imel: "foo@bar.com", + Url: "http://example.com", + Num: 42, + } + if err := meddler.Save(db, "samples", &row); err != nil { + t.Errorf("Can not save into database: %q", err) + } + + if err := mgr.MigrateTo(1); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var another_columns []*TableInfo + if err := meddler.QueryAll(db, &another_columns, `PRAGMA table_info(samples);`); err != nil { + t.Errorf("Can not access table info: %q", err) + } + + if len(another_columns) != 3 { + t.Errorf("Expect length columns = %d, got: %d", 3, len(columns)) + } +} + +func TestRenameColumn(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + mgr := New(db) + if err := mgr.Add(&revision1{}, &revision4{}).MigrateTo(1); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + loadFixture(t) + + if err := mgr.MigrateTo(4); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + row := RenameSample{} + if err := meddler.QueryRow(db, &row, `SELECT * FROM samples WHERE id = 3;`); err != nil { + t.Errorf("Can not query database: %q", err) + } + + if row.Email != "crash@bandicoot.io" { + t.Errorf("Expect %s, got %s", "crash@bandicoot.io", row.Email) + } +} + +func TestMigrateExistingTable(t *testing.T) { + defer tearDown() + if err := setUp(); err != nil { + t.Fatalf("Error preparing database: %q", err) + } + + Driver = SQLite + + if _, err := db.Exec(testSchema); err != nil { + t.Errorf("Can not create database: %q", err) + } + + loadFixture(t) + + mgr := New(db) + if err := mgr.Add(&revision4{}).Migrate(); err != nil { + t.Errorf("Can not migrate: %q", err) + } + + var rows []*RenameSample + if err := meddler.QueryAll(db, &rows, `SELECT * from samples;`); err != nil { + t.Errorf("Can not query database: %q", err) + } + + if len(rows) != 3 { + t.Errorf("Expect rows length = %d, got %d", 3, len(rows)) + } + + if rows[1].Email != "foo@bar.com" { + t.Errorf("Expect email = %s, got %s", "foo@bar.com", rows[1].Email) + } +} + +func setUp() error { + var err error + db, err = sql.Open("sqlite3", "migration_tests.sqlite") + return err +} + +func tearDown() { + db.Close() + os.Remove("migration_tests.sqlite") +} + +func loadFixture(t *testing.T) { + for _, sql := range dataDump { + if _, err := db.Exec(sql); err != nil { + t.Errorf("Can not insert into database: %q", err) + } + } +} diff --git a/pkg/database/migrate/util.go b/pkg/database/migrate/util.go new file mode 100644 index 000000000..a0f6bfb59 --- /dev/null +++ b/pkg/database/migrate/util.go @@ -0,0 +1,32 @@ +package migrate + +import ( + "fmt" + "strings" +) + +func fetchColumns(sql string) ([]string, error) { + if !strings.HasPrefix(sql, "CREATE TABLE ") { + return []string{}, fmt.Errorf("Sql input is not a DDL statement.") + } + + parenIdx := strings.Index(sql, "(") + return strings.Split(sql[parenIdx+1:len(sql)-1], ","), nil +} + +func selectName(columns []string) []string { + var results []string + for _, column := range columns { + col := strings.SplitN(strings.Trim(column, " \n\t"), " ", 2) + results = append(results, col[0]) + } + return results +} + +func setForUpdate(left []string, right []string) string { + var results []string + for k, str := range left { + results = append(results, fmt.Sprintf("%s = %s", str, right[k])) + } + return strings.Join(results, ", ") +} diff --git a/pkg/database/repos.go b/pkg/database/repos.go index e4407c028..0d9e2f15e 100644 --- a/pkg/database/repos.go +++ b/pkg/database/repos.go @@ -13,7 +13,7 @@ const repoTable = "repos" // SQL Queries to retrieve a list of all repos belonging to a User. const repoStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE user_id = ? AND team_id = 0 ORDER BY slug ASC @@ -22,7 +22,7 @@ ORDER BY slug ASC // SQL Queries to retrieve a list of all repos belonging to a Team. const repoTeamStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE team_id = ? ORDER BY slug ASC @@ -31,7 +31,7 @@ ORDER BY slug ASC // SQL Queries to retrieve a repo by id. const repoFindStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE id = ? ` @@ -39,7 +39,7 @@ WHERE id = ? // SQL Queries to retrieve a repo by name. const repoFindSlugStmt = ` SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password, -public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id +public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id FROM repos WHERE slug = ? ` diff --git a/pkg/database/testing/testing.go b/pkg/database/testing/testing.go index 11106e2d3..69fde3787 100644 --- a/pkg/database/testing/testing.go +++ b/pkg/database/testing/testing.go @@ -7,6 +7,7 @@ import ( "github.com/drone/drone/pkg/database" "github.com/drone/drone/pkg/database/encrypt" + "github.com/drone/drone/pkg/database/migrate" . "github.com/drone/drone/pkg/model" _ "github.com/mattn/go-sqlite3" @@ -31,6 +32,7 @@ func init() { // notify meddler that we are working with sqlite meddler.Default = meddler.SQLite + migrate.Driver = migrate.SQLite } func Setup() { @@ -40,6 +42,9 @@ func Setup() { // make sure all the tables and indexes are created database.Set(db) + migration := migrate.New(db) + migration.All().Migrate() + // create dummy user data user1 := User{ Password: "$2a$10$b8d63QsTL38vx7lj0HEHfOdbu1PCAg6Gfca74UavkXooIBx9YxopS", diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 2b3d28069..bf9914e4f 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -41,7 +41,7 @@ func (h UserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // AdminHandler wraps the default http.HandlerFunc to include // the currently authenticated User in the method signature, // in addition to handling an error as the return value. It also -// verifies the user has Administrative priveleges. +// verifies the user has Administrative privileges. type AdminHandler func(w http.ResponseWriter, r *http.Request, user *User) error func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -51,7 +51,7 @@ func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // User MUST have administrative priveleges in order + // User MUST have administrative privileges in order // to execute the handler. if user.Admin == false { RenderNotFound(w) diff --git a/pkg/model/repo.go b/pkg/model/repo.go index 5b89ad71c..8d3ab202b 100644 --- a/pkg/model/repo.go +++ b/pkg/model/repo.go @@ -88,9 +88,9 @@ type Repo struct { // before exceeding its timelimit and being killed. Timeout int64 `meddler:"timeout" json:"timeout"` - // Indicates the build should be executed in priveleged + // Indicates the build should be executed in privileged // mode. This could, for example, be used to run Docker in Docker. - Priveleged bool `meddler:"priveleged" json:"priveleged"` + Privileged bool `meddler:"privileged" json:"privileged"` // Foreign keys signify the User that created // the repository and team account linked to