// Package fsmgr manages fs objects package fsmgr import ( "bytes" "compress/zlib" "crypto/sha1" "encoding/hex" "encoding/json" "fmt" "io" "path/filepath" "strings" "syscall" "github.com/haiwen/seafile-server/fileserver/objstore" ) // Seafile is a file object type Seafile struct { data []byte Version int `json:"version"` FileType int `json:"type"` FileID string `json:"file_id"` FileSize uint64 `json:"size"` BlkIDs []string `json:"block_ids"` } // In the JSON encoding generated by C language, there are spaces after the ',' and ':', and the order of the fields is sorted by the key. // So it is not compatible with the json library generated by go. func (file *Seafile) toJSON() ([]byte, error) { var buf bytes.Buffer buf.WriteByte('{') buf.WriteString("\"block_ids\": [") for i, blkID := range file.BlkIDs { data, err := json.Marshal(blkID) if err != nil { return nil, err } buf.Write(data) if i < len(file.BlkIDs)-1 { buf.WriteByte(',') buf.WriteByte(' ') } } buf.WriteByte(']') buf.WriteByte(',') buf.WriteByte(' ') data, err := json.Marshal(file.FileSize) if err != nil { return nil, err } writeField(&buf, "\"size\"", data) buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(SeafMetadataTypeFile) if err != nil { return nil, err } writeField(&buf, "\"type\"", data) buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(file.Version) if err != nil { return nil, err } writeField(&buf, "\"version\"", data) buf.WriteByte('}') return buf.Bytes(), nil } func writeField(buf *bytes.Buffer, key string, value []byte) { buf.WriteString(key) buf.WriteByte(':') buf.WriteByte(' ') buf.Write(value) } // SeafDirent is a dir entry object type SeafDirent struct { Mode uint32 `json:"mode"` ID string `json:"id"` Name string `json:"name"` Mtime int64 `json:"mtime"` Modifier string `json:"modifier"` Size int64 `json:"size"` } func (dent *SeafDirent) toJSON() ([]byte, error) { var buf bytes.Buffer buf.WriteByte('{') data, err := json.Marshal(dent.ID) if err != nil { return nil, err } writeField(&buf, "\"id\"", data) buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(dent.Mode) if err != nil { return nil, err } writeField(&buf, "\"mode\"", data) buf.WriteByte(',') buf.WriteByte(' ') if IsRegular(dent.Mode) { data, err = json.Marshal(dent.Modifier) if err != nil { return nil, err } writeField(&buf, "\"modifier\"", data) buf.WriteByte(',') buf.WriteByte(' ') } data, err = json.Marshal(dent.Mtime) if err != nil { return nil, err } writeField(&buf, "\"mtime\"", data) buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(dent.Name) if err != nil { return nil, err } writeField(&buf, "\"name\"", data) if IsRegular(dent.Mode) { buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(dent.Size) if err != nil { return nil, err } writeField(&buf, "\"size\"", data) } buf.WriteByte('}') return buf.Bytes(), nil } //SeafDir is a dir object type SeafDir struct { data []byte Version int `json:"version"` DirType int `json:"type"` DirID string `json:"dir_id"` Entries []*SeafDirent `json:"dirents"` } func (dir *SeafDir) toJSON() ([]byte, error) { var buf bytes.Buffer buf.WriteByte('{') buf.WriteString("\"dirents\": [") for i, entry := range dir.Entries { data, err := entry.toJSON() if err != nil { return nil, err } buf.Write(data) if i < len(dir.Entries)-1 { buf.WriteByte(',') buf.WriteByte(' ') } } buf.WriteByte(']') buf.WriteByte(',') buf.WriteByte(' ') data, err := json.Marshal(SeafMetadataTypeDir) if err != nil { return nil, err } writeField(&buf, "\"type\"", data) buf.WriteByte(',') buf.WriteByte(' ') data, err = json.Marshal(dir.Version) if err != nil { return nil, err } writeField(&buf, "\"version\"", data) buf.WriteByte('}') return buf.Bytes(), nil } // FileCountInfo contains information of files type FileCountInfo struct { FileCount int64 Size int64 DirCount int64 } // Meta data type of dir or file const ( SeafMetadataTypeInvalid = iota SeafMetadataTypeFile SeafMetadataTypeLink SeafMetadataTypeDir ) var store *objstore.ObjectStore // Empty value of sha1 const ( EmptySha1 = "0000000000000000000000000000000000000000" ) // Init initializes fs manager and creates underlying object store. func Init(seafileConfPath string, seafileDataDir string) { store = objstore.New(seafileConfPath, seafileDataDir, "fs") } // NewDirent initializes a SeafDirent object func NewDirent(id string, name string, mode uint32, mtime int64, modifier string, size int64) *SeafDirent { dent := new(SeafDirent) dent.ID = id if id == "" { dent.ID = EmptySha1 } dent.Name = name dent.Mode = mode dent.Mtime = mtime if IsRegular(mode) { dent.Modifier = modifier dent.Size = size } return dent } // NewSeafdir initializes a SeafDir object func NewSeafdir(version int, entries []*SeafDirent) (*SeafDir, error) { dir := new(SeafDir) dir.Version = version dir.Entries = entries if len(entries) == 0 { dir.DirID = EmptySha1 return dir, nil } jsonstr, err := dir.toJSON() if err != nil { err := fmt.Errorf("failed to convert seafdir to json") return nil, err } dir.data = jsonstr checksum := sha1.Sum(jsonstr) dir.DirID = hex.EncodeToString(checksum[:]) return dir, nil } // NewSeafile initializes a Seafile object func NewSeafile(version int, fileSize int64, blkIDs []string) (*Seafile, error) { seafile := new(Seafile) seafile.Version = version seafile.FileSize = uint64(fileSize) seafile.BlkIDs = blkIDs if len(blkIDs) == 0 { seafile.FileID = EmptySha1 return seafile, nil } jsonstr, err := seafile.toJSON() if err != nil { err := fmt.Errorf("failed to convert seafile to json") return nil, err } seafile.data = jsonstr checkSum := sha1.Sum(jsonstr) seafile.FileID = hex.EncodeToString(checkSum[:]) return seafile, nil } func uncompress(p []byte) ([]byte, error) { b := bytes.NewReader(p) var out bytes.Buffer r, err := zlib.NewReader(b) if err != nil { return nil, err } _, err = io.Copy(&out, r) if err != nil { r.Close() return nil, err } r.Close() return out.Bytes(), nil } func compress(p []byte) ([]byte, error) { var out bytes.Buffer w := zlib.NewWriter(&out) _, err := w.Write(p) if err != nil { w.Close() return nil, err } w.Close() return out.Bytes(), nil } // FromData reads from p and converts JSON-encoded data to Seafile. func (seafile *Seafile) FromData(p []byte) error { b, err := uncompress(p) if err != nil { return err } err = json.Unmarshal(b, seafile) if err != nil { return err } return nil } // ToData converts seafile to JSON-encoded data and writes to w. func (seafile *Seafile) ToData(w io.Writer) error { buf, err := compress(seafile.data) if err != nil { return err } _, err = w.Write(buf) if err != nil { return err } return nil } // ToData converts seafdir to JSON-encoded data and writes to w. func (seafdir *SeafDir) ToData(w io.Writer) error { buf, err := compress(seafdir.data) if err != nil { return err } _, err = w.Write(buf) if err != nil { return err } return nil } // FromData reads from p and converts JSON-encoded data to SeafDir. func (seafdir *SeafDir) FromData(p []byte) error { b, err := uncompress(p) if err != nil { return err } err = json.Unmarshal(b, seafdir) if err != nil { return err } return nil } // ReadRaw reads data in binary format from storage backend. func ReadRaw(repoID string, objID string, w io.Writer) error { err := store.Read(repoID, objID, w) if err != nil { return err } return nil } // WriteRaw writes data in binary format to storage backend. func WriteRaw(repoID string, objID string, r io.Reader) error { err := store.Write(repoID, objID, r, false) if err != nil { return err } return nil } // GetSeafile gets seafile from storage backend. func GetSeafile(repoID string, fileID string) (*Seafile, error) { var buf bytes.Buffer seafile := new(Seafile) if fileID == EmptySha1 { seafile.FileID = EmptySha1 return seafile, nil } err := ReadRaw(repoID, fileID, &buf) if err != nil { errors := fmt.Errorf("failed to read seafile object from storage : %v", err) return nil, errors } err = seafile.FromData(buf.Bytes()) if err != nil { errors := fmt.Errorf("failed to parse seafile object %s/%s : %v", repoID, fileID, err) return nil, errors } if seafile.Version < 1 { errors := fmt.Errorf("seafile object %s/%s version should be > 0", repoID, fileID) return nil, errors } seafile.FileID = fileID return seafile, nil } // SaveSeafile saves seafile to storage backend. func SaveSeafile(repoID string, seafile *Seafile) error { fileID := seafile.FileID if fileID == EmptySha1 { return nil } exist, _ := store.Exists(repoID, fileID) if exist { return nil } seafile.FileType = SeafMetadataTypeFile var buf bytes.Buffer err := seafile.ToData(&buf) if err != nil { errors := fmt.Errorf("failed to convert seafile object %s/%s to json", repoID, fileID) return errors } err = WriteRaw(repoID, fileID, &buf) if err != nil { errors := fmt.Errorf("failed to write seafile object to storage : %v", err) return errors } return nil } // GetSeafdir gets seafdir from storage backend. func GetSeafdir(repoID string, dirID string) (*SeafDir, error) { var buf bytes.Buffer seafdir := new(SeafDir) if dirID == EmptySha1 { seafdir.DirID = EmptySha1 return seafdir, nil } err := ReadRaw(repoID, dirID, &buf) if err != nil { errors := fmt.Errorf("failed to read seafdir object from storage : %v", err) return nil, errors } err = seafdir.FromData(buf.Bytes()) if err != nil { errors := fmt.Errorf("failed to parse seafdir object %s/%s : %v", repoID, dirID, err) return nil, errors } if seafdir.Version < 1 { errors := fmt.Errorf("seadir object %s/%s version should be > 0", repoID, dirID) return nil, errors } seafdir.DirID = dirID return seafdir, nil } // SaveSeafdir saves seafdir to storage backend. func SaveSeafdir(repoID string, seafdir *SeafDir) error { dirID := seafdir.DirID if dirID == EmptySha1 { return nil } exist, _ := store.Exists(repoID, dirID) if exist { return nil } seafdir.DirType = SeafMetadataTypeDir var buf bytes.Buffer err := seafdir.ToData(&buf) if err != nil { errors := fmt.Errorf("failed to convert seafdir object %s/%s to json", repoID, dirID) return errors } err = WriteRaw(repoID, dirID, &buf) if err != nil { errors := fmt.Errorf("failed to write seafdir object to storage : %v", err) return errors } return nil } // Exists check if fs object is exists. func Exists(repoID string, objID string) (bool, error) { if objID == EmptySha1 { return true, nil } return store.Exists(repoID, objID) } func comp(c rune) bool { return c == '/' } // IsDir check if the mode is dir. func IsDir(m uint32) bool { return (m & syscall.S_IFMT) == syscall.S_IFDIR } // IsRegular Check if the mode is regular. func IsRegular(m uint32) bool { return (m & syscall.S_IFMT) == syscall.S_IFREG } // ErrPathNoExist is an error indicating that the file does not exist var ErrPathNoExist = fmt.Errorf("path does not exist") // GetSeafdirByPath gets the object of seafdir by path. func GetSeafdirByPath(repoID string, rootID string, path string) (*SeafDir, error) { dir, err := GetSeafdir(repoID, rootID) if err != nil { errors := fmt.Errorf("directory is missing") return nil, errors } path = filepath.Join("/", path) parts := strings.FieldsFunc(path, comp) var dirID string for _, name := range parts { entries := dir.Entries for _, v := range entries { if v.Name == name && IsDir(v.Mode) { dirID = v.ID break } } if dirID == `` { return nil, ErrPathNoExist } dir, err = GetSeafdir(repoID, dirID) if err != nil { errors := fmt.Errorf("directory is missing") return nil, errors } } return dir, nil } // GetSeafdirIDByPath gets the dirID of SeafDir by path. func GetSeafdirIDByPath(repoID, rootID, path string) (string, error) { dirID, mode, err := GetObjIDByPath(repoID, rootID, path) if err != nil { err := fmt.Errorf("failed to get dir id by path: %s: %v", path, err) return "", err } if dirID == "" || !IsDir(mode) { return "", nil } return dirID, nil } // GetObjIDByPath gets the obj id by path func GetObjIDByPath(repoID, rootID, path string) (string, uint32, error) { var name string var baseDir *SeafDir formatPath := filepath.Join(path) if len(formatPath) == 0 || formatPath == "/" { return rootID, syscall.S_IFDIR, nil } index := strings.Index(formatPath, "/") if index < 0 { dir, err := GetSeafdir(repoID, rootID) if err != nil { err := fmt.Errorf("failed to find root dir %s: %v", rootID, err) return "", 0, err } name = formatPath baseDir = dir } else { name = filepath.Base(formatPath) dirName := filepath.Dir(formatPath) dir, err := GetSeafdirByPath(repoID, rootID, dirName) if err != nil { if err == ErrPathNoExist { return "", syscall.S_IFDIR, ErrPathNoExist } err := fmt.Errorf("failed to find dir %s in repo %s: %v", dirName, repoID, err) return "", syscall.S_IFDIR, err } baseDir = dir } entries := baseDir.Entries for _, de := range entries { if de.Name == name { return de.ID, de.Mode, nil } } return "", 0, nil } // GetFileCountInfoByPath gets the count info of file by path. func GetFileCountInfoByPath(repoID, rootID, path string) (*FileCountInfo, error) { dirID, err := GetSeafdirIDByPath(repoID, rootID, path) if err != nil { err := fmt.Errorf("failed to get file count info for repo %s path %s: %v", repoID, path, err) return nil, err } info, err := getFileCountInfo(repoID, dirID) if err != nil { err := fmt.Errorf("failed to get file count in repo %s: %v", repoID, err) return nil, err } return info, nil } func getFileCountInfo(repoID, dirID string) (*FileCountInfo, error) { dir, err := GetSeafdir(repoID, dirID) if err != nil { err := fmt.Errorf("failed to get dir: %v", err) return nil, err } info := new(FileCountInfo) entries := dir.Entries for _, de := range entries { if IsDir(de.Mode) { tmpInfo, err := getFileCountInfo(repoID, de.ID) if err != nil { err := fmt.Errorf("failed to get file count: %v", err) return nil, err } info.DirCount = tmpInfo.DirCount + 1 info.FileCount += tmpInfo.FileCount info.Size += tmpInfo.Size } else { info.FileCount++ info.Size += de.Size } } return info, nil }