1
0
mirror of https://github.com/haiwen/seafile-server.git synced 2025-06-29 08:26:51 +00:00
seafile-server/fileserver/fsmgr/fsmgr.go

911 lines
21 KiB
Go
Raw Normal View History

Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// Package fsmgr manages fs objects
package fsmgr
import (
"bytes"
"compress/zlib"
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
"syscall"
"time"
"unsafe"
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
"github.com/haiwen/seafile-server/fileserver/objstore"
"github.com/haiwen/seafile-server/fileserver/utils"
jsoniter "github.com/json-iterator/go"
"github.com/dgraph-io/ristretto"
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// Seafile is a file object
type Seafile struct {
data []byte
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
Version int `json:"version"`
FileType int `json:"type"`
FileID string `json:"-"`
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
FileSize uint64 `json:"size"`
BlkIDs []string `json:"block_ids"`
}
// In the JSON encoding generated by C language, there are spaces after the ',' and ':', and the order of the fields is sorted by the key.
// So it is not compatible with the json library generated by go.
func (file *Seafile) toJSON() ([]byte, error) {
var buf bytes.Buffer
buf.WriteByte('{')
buf.WriteString("\"block_ids\": [")
for i, blkID := range file.BlkIDs {
data, err := json.Marshal(blkID)
if err != nil {
return nil, err
}
buf.Write(data)
if i < len(file.BlkIDs)-1 {
buf.WriteByte(',')
buf.WriteByte(' ')
}
}
buf.WriteByte(']')
buf.WriteByte(',')
buf.WriteByte(' ')
data, err := json.Marshal(file.FileSize)
if err != nil {
return nil, err
}
writeField(&buf, "\"size\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = json.Marshal(SeafMetadataTypeFile)
if err != nil {
return nil, err
}
writeField(&buf, "\"type\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = json.Marshal(file.Version)
if err != nil {
return nil, err
}
writeField(&buf, "\"version\"", data)
buf.WriteByte('}')
return buf.Bytes(), nil
}
func writeField(buf *bytes.Buffer, key string, value []byte) {
buf.WriteString(key)
buf.WriteByte(':')
buf.WriteByte(' ')
buf.Write(value)
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// SeafDirent is a dir entry object
type SeafDirent struct {
Mode uint32 `json:"mode"`
ID string `json:"id"`
Name string `json:"name"`
Mtime int64 `json:"mtime"`
Modifier string `json:"modifier"`
Size int64 `json:"size"`
}
func (dent *SeafDirent) toJSON() ([]byte, error) {
var buf bytes.Buffer
buf.WriteByte('{')
data, err := json.Marshal(dent.ID)
if err != nil {
return nil, err
}
writeField(&buf, "\"id\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = json.Marshal(dent.Mode)
if err != nil {
return nil, err
}
writeField(&buf, "\"mode\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
if IsRegular(dent.Mode) {
data, err = jsonNoEscape(dent.Modifier)
if err != nil {
return nil, err
}
writeField(&buf, "\"modifier\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
}
data, err = json.Marshal(dent.Mtime)
if err != nil {
return nil, err
}
writeField(&buf, "\"mtime\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = jsonNoEscape(dent.Name)
if err != nil {
return nil, err
}
writeField(&buf, "\"name\"", data)
if IsRegular(dent.Mode) {
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = json.Marshal(dent.Size)
if err != nil {
return nil, err
}
writeField(&buf, "\"size\"", data)
}
buf.WriteByte('}')
return buf.Bytes(), nil
}
// In golang json, the string is encoded using HTMLEscape, which replaces "<", ">", "&", U+2028, and U+2029 are escaped to "\u003c","\u003e", "\u0026", "\u2028", and "\u2029".
// So it is not compatible with the json library generated by c. This replacement can be disabled when using an Encoder, by calling SetEscapeHTML(false).
func jsonNoEscape(data interface{}) ([]byte, error) {
var buf bytes.Buffer
encoder := json.NewEncoder(&buf)
encoder.SetEscapeHTML(false)
if err := encoder.Encode(data); err != nil {
return nil, err
}
bytes := buf.Bytes()
// Encode will terminate each value with a newline.
// This makes the output look a little nicer
// when debugging, and some kind of space
// is required if the encoded value was a number,
// so that the reader knows there aren't more
// digits coming.
// The newline at the end needs to be removed for the above reasons.
return bytes[:len(bytes)-1], nil
}
// SeafDir is a dir object
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
type SeafDir struct {
data []byte
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
Version int `json:"version"`
DirType int `json:"type"`
DirID string `json:"-"`
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
Entries []*SeafDirent `json:"dirents"`
}
func (dir *SeafDir) toJSON() ([]byte, error) {
var buf bytes.Buffer
buf.WriteByte('{')
buf.WriteString("\"dirents\": [")
for i, entry := range dir.Entries {
data, err := entry.toJSON()
if err != nil {
return nil, err
}
buf.Write(data)
if i < len(dir.Entries)-1 {
buf.WriteByte(',')
buf.WriteByte(' ')
}
}
buf.WriteByte(']')
buf.WriteByte(',')
buf.WriteByte(' ')
data, err := json.Marshal(SeafMetadataTypeDir)
if err != nil {
return nil, err
}
writeField(&buf, "\"type\"", data)
buf.WriteByte(',')
buf.WriteByte(' ')
data, err = json.Marshal(dir.Version)
if err != nil {
return nil, err
}
writeField(&buf, "\"version\"", data)
buf.WriteByte('}')
return buf.Bytes(), nil
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// FileCountInfo contains information of files
type FileCountInfo struct {
FileCount int64
Size int64
DirCount int64
}
// Meta data type of dir or file
const (
SeafMetadataTypeInvalid = iota
SeafMetadataTypeFile
SeafMetadataTypeLink
SeafMetadataTypeDir
)
var store *objstore.ObjectStore
// Empty value of sha1
const (
EmptySha1 = "0000000000000000000000000000000000000000"
)
// Since zlib library allocates a large amount of memory every time a new reader is created, when the number of calls is too large,
// the GC will be executed frequently, resulting in high CPU usage.
var zlibReaders []io.ReadCloser
var zlibLock sync.Mutex
// Add fs cache, on the one hand to avoid repeated creation and destruction of repeatedly accessed objects,
// on the other hand it will also slow down the speed at which objects are released.
var fsCache *ristretto.Cache
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// Init initializes fs manager and creates underlying object store.
func Init(seafileConfPath string, seafileDataDir string, fsCacheLimit int64) {
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
store = objstore.New(seafileConfPath, seafileDataDir, "fs")
fsCache, _ = ristretto.NewCache(&ristretto.Config{
NumCounters: 1e7, // number of keys to track frequency of (10M).
MaxCost: fsCacheLimit, // maximum cost of cache.
BufferItems: 64, // number of keys per Get buffer.
Cost: calCost,
})
}
func calCost(value interface{}) int64 {
return sizeOf(value)
}
const (
sizeOfString = int64(unsafe.Sizeof(string("")))
sizeOfPointer = int64(unsafe.Sizeof(uintptr(0)))
sizeOfSeafile = int64(unsafe.Sizeof(Seafile{}))
sizeOfSeafDir = int64(unsafe.Sizeof(SeafDir{}))
sizeOfSeafDirent = int64(unsafe.Sizeof(SeafDirent{}))
)
func sizeOf(a interface{}) int64 {
var size int64
switch x := a.(type) {
case string:
return sizeOfString + int64(len(x))
case []string:
for _, s := range x {
size += sizeOf(s)
}
return size
case *Seafile:
size = sizeOfPointer
size += sizeOfSeafile
size += int64(len(x.FileID))
size += sizeOf(x.BlkIDs)
return size
case *SeafDir:
size = sizeOfPointer
size += sizeOfSeafDir
size += int64(len(x.DirID))
for _, dent := range x.Entries {
size += sizeOf(dent)
}
return size
case *SeafDirent:
size = sizeOfPointer
size += sizeOfSeafDirent
size += int64(len(x.ID))
size += int64(len(x.Name))
size += int64(len(x.Modifier))
return size
}
return 0
}
func initZlibReader() (io.ReadCloser, error) {
var buf bytes.Buffer
// Since the corresponding reader has not been obtained when zlib is initialized,
// an io.Reader needs to be built to initialize zlib.
w := zlib.NewWriter(&buf)
w.Close()
r, err := zlib.NewReader(&buf)
if err != nil {
return nil, err
}
return r, nil
}
// GetOneZlibReader gets a zlib reader from zlibReaders.
func GetOneZlibReader() io.ReadCloser {
zlibLock.Lock()
defer zlibLock.Unlock()
var reader io.ReadCloser
if len(zlibReaders) == 0 {
reader, err := initZlibReader()
if err != nil {
return nil
}
return reader
}
reader = zlibReaders[0]
zlibReaders = zlibReaders[1:]
return reader
}
func ReturnOneZlibReader(reader io.ReadCloser) {
if reader == nil {
return
}
zlibLock.Lock()
defer zlibLock.Unlock()
zlibReaders = append(zlibReaders, reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
}
// NewDirent initializes a SeafDirent object
func NewDirent(id string, name string, mode uint32, mtime int64, modifier string, size int64) *SeafDirent {
dent := new(SeafDirent)
dent.ID = id
if id == "" {
dent.ID = EmptySha1
}
dent.Name = name
dent.Mode = mode
dent.Mtime = mtime
if IsRegular(mode) {
dent.Modifier = modifier
dent.Size = size
}
return dent
}
// NewSeafdir initializes a SeafDir object
func NewSeafdir(version int, entries []*SeafDirent) (*SeafDir, error) {
dir := new(SeafDir)
dir.Version = version
dir.Entries = entries
if len(entries) == 0 {
dir.DirID = EmptySha1
return dir, nil
}
jsonstr, err := dir.toJSON()
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
err := fmt.Errorf("failed to convert seafdir to json")
return nil, err
}
dir.data = jsonstr
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
checksum := sha1.Sum(jsonstr)
dir.DirID = hex.EncodeToString(checksum[:])
return dir, nil
}
// NewSeafile initializes a Seafile object
func NewSeafile(version int, fileSize int64, blkIDs []string) (*Seafile, error) {
seafile := new(Seafile)
seafile.Version = version
seafile.FileSize = uint64(fileSize)
seafile.BlkIDs = blkIDs
if len(blkIDs) == 0 {
seafile.FileID = EmptySha1
return seafile, nil
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
jsonstr, err := seafile.toJSON()
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
err := fmt.Errorf("failed to convert seafile to json")
return nil, err
}
seafile.data = jsonstr
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
checkSum := sha1.Sum(jsonstr)
seafile.FileID = hex.EncodeToString(checkSum[:])
return seafile, nil
}
func uncompress(p []byte, reader io.ReadCloser) ([]byte, error) {
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
b := bytes.NewReader(p)
var out bytes.Buffer
if reader == nil {
r, err := zlib.NewReader(b)
if err != nil {
return nil, err
}
_, err = io.Copy(&out, r)
if err != nil {
r.Close()
return nil, err
}
r.Close()
return out.Bytes(), nil
}
// resue the old zlib reader.
resetter, _ := reader.(zlib.Resetter)
err := resetter.Reset(b, nil)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return nil, err
}
_, err = io.Copy(&out, reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return nil, err
}
return out.Bytes(), nil
}
func compress(p []byte) ([]byte, error) {
var out bytes.Buffer
w := zlib.NewWriter(&out)
_, err := w.Write(p)
if err != nil {
w.Close()
return nil, err
}
w.Close()
return out.Bytes(), nil
}
// FromData reads from p and converts JSON-encoded data to Seafile.
func (seafile *Seafile) FromData(p []byte, reader io.ReadCloser) error {
b, err := uncompress(p, reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return err
}
err = json.Unmarshal(b, seafile)
if err != nil {
return err
}
if seafile.FileType != SeafMetadataTypeFile {
return fmt.Errorf("object %s is not a file", seafile.FileID)
}
if seafile.Version < 1 {
return fmt.Errorf("seafile object %s version should be > 0, version is %d", seafile.FileID, seafile.Version)
}
if seafile.BlkIDs == nil {
return fmt.Errorf("no block id array in seafile object %s", seafile.FileID)
}
for _, blkID := range seafile.BlkIDs {
if !utils.IsObjectIDValid(blkID) {
return fmt.Errorf("block id %s is invalid", blkID)
}
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
return nil
}
// ToData converts seafile to JSON-encoded data and writes to w.
func (seafile *Seafile) ToData(w io.Writer) error {
buf, err := compress(seafile.data)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return err
}
_, err = w.Write(buf)
if err != nil {
return err
}
return nil
}
// ToData converts seafdir to JSON-encoded data and writes to w.
func (seafdir *SeafDir) ToData(w io.Writer) error {
buf, err := compress(seafdir.data)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return err
}
_, err = w.Write(buf)
if err != nil {
return err
}
return nil
}
// FromData reads from p and converts JSON-encoded data to SeafDir.
func (seafdir *SeafDir) FromData(p []byte, reader io.ReadCloser) error {
b, err := uncompress(p, reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
return err
}
err = json.Unmarshal(b, seafdir)
if err != nil {
return err
}
if seafdir.DirType != SeafMetadataTypeDir {
return fmt.Errorf("object %s is not a dir", seafdir.DirID)
}
if seafdir.Version < 1 {
return fmt.Errorf("dir object %s version should be > 0, version is %d", seafdir.DirID, seafdir.Version)
}
if seafdir.Entries == nil {
return fmt.Errorf("no dirents in dir object %s", seafdir.DirID)
}
for _, dent := range seafdir.Entries {
if !utils.IsObjectIDValid(dent.ID) {
return fmt.Errorf("dirent id %s is invalid", dent.ID)
}
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
return nil
}
// ReadRaw reads data in binary format from storage backend.
func ReadRaw(repoID string, objID string, w io.Writer) error {
err := store.Read(repoID, objID, w)
if err != nil {
return err
}
return nil
}
// WriteRaw writes data in binary format to storage backend.
func WriteRaw(repoID string, objID string, r io.Reader) error {
err := store.Write(repoID, objID, r, false)
if err != nil {
return err
}
return nil
}
// GetSeafile gets seafile from storage backend.
func GetSeafile(repoID string, fileID string) (*Seafile, error) {
return getSeafile(repoID, fileID, nil)
}
// GetSeafileWithZlibReader gets seafile from storage backend with a zlib reader.
func GetSeafileWithZlibReader(repoID string, fileID string, reader io.ReadCloser) (*Seafile, error) {
return getSeafile(repoID, fileID, reader)
}
func getSeafile(repoID string, fileID string, reader io.ReadCloser) (*Seafile, error) {
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
var buf bytes.Buffer
seafile := new(Seafile)
if fileID == EmptySha1 {
seafile.FileID = EmptySha1
return seafile, nil
}
seafile.FileID = fileID
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
err := ReadRaw(repoID, fileID, &buf)
if err != nil {
errors := fmt.Errorf("failed to read seafile object from storage : %v", err)
return nil, errors
}
err = seafile.FromData(buf.Bytes(), reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
errors := fmt.Errorf("failed to parse seafile object %s/%s : %v", repoID, fileID, err)
return nil, errors
}
if seafile.Version < 1 {
errors := fmt.Errorf("seafile object %s/%s version should be > 0", repoID, fileID)
return nil, errors
}
return seafile, nil
}
// SaveSeafile saves seafile to storage backend.
func SaveSeafile(repoID string, seafile *Seafile) error {
fileID := seafile.FileID
if fileID == EmptySha1 {
return nil
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
exist, _ := store.Exists(repoID, fileID)
if exist {
return nil
}
seafile.FileType = SeafMetadataTypeFile
var buf bytes.Buffer
err := seafile.ToData(&buf)
if err != nil {
errors := fmt.Errorf("failed to convert seafile object %s/%s to json", repoID, fileID)
return errors
}
err = WriteRaw(repoID, fileID, &buf)
if err != nil {
errors := fmt.Errorf("failed to write seafile object to storage : %v", err)
return errors
}
return nil
}
// GetSeafdir gets seafdir from storage backend.
func GetSeafdir(repoID string, dirID string) (*SeafDir, error) {
return getSeafdir(repoID, dirID, nil, false)
}
// GetSeafdir gets seafdir from storage backend with a zlib reader.
func GetSeafdirWithZlibReader(repoID string, dirID string, reader io.ReadCloser) (*SeafDir, error) {
return getSeafdir(repoID, dirID, reader, true)
}
func getSeafdir(repoID string, dirID string, reader io.ReadCloser, useCache bool) (*SeafDir, error) {
var seafdir *SeafDir
if useCache {
seafdir = getSeafdirFromCache(repoID, dirID)
if seafdir != nil {
return seafdir, nil
}
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
var buf bytes.Buffer
seafdir = new(SeafDir)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if dirID == EmptySha1 {
seafdir.DirID = EmptySha1
return seafdir, nil
}
seafdir.DirID = dirID
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
err := ReadRaw(repoID, dirID, &buf)
if err != nil {
errors := fmt.Errorf("failed to read seafdir object from storage : %v", err)
return nil, errors
}
err = seafdir.FromData(buf.Bytes(), reader)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
if err != nil {
errors := fmt.Errorf("failed to parse seafdir object %s/%s : %v", repoID, dirID, err)
return nil, errors
}
if seafdir.Version < 1 {
errors := fmt.Errorf("seadir object %s/%s version should be > 0", repoID, dirID)
return nil, errors
}
if useCache {
setSeafdirToCache(repoID, seafdir)
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
return seafdir, nil
}
func getSeafdirFromCache(repoID string, dirID string) *SeafDir {
key := repoID + dirID
v, ok := fsCache.Get(key)
if !ok {
return nil
}
seafdir, ok := v.(*SeafDir)
if ok {
return seafdir
}
return nil
}
func setSeafdirToCache(repoID string, seafdir *SeafDir) error {
key := repoID + seafdir.DirID
fsCache.SetWithTTL(key, seafdir, 0, time.Duration(1*time.Hour))
return nil
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
// SaveSeafdir saves seafdir to storage backend.
func SaveSeafdir(repoID string, seafdir *SeafDir) error {
dirID := seafdir.DirID
if dirID == EmptySha1 {
return nil
}
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
exist, _ := store.Exists(repoID, dirID)
if exist {
return nil
}
seafdir.DirType = SeafMetadataTypeDir
var buf bytes.Buffer
err := seafdir.ToData(&buf)
if err != nil {
errors := fmt.Errorf("failed to convert seafdir object %s/%s to json", repoID, dirID)
return errors
}
err = WriteRaw(repoID, dirID, &buf)
if err != nil {
errors := fmt.Errorf("failed to write seafdir object to storage : %v", err)
return errors
}
return nil
}
// Exists check if fs object is exists.
func Exists(repoID string, objID string) (bool, error) {
if objID == EmptySha1 {
return true, nil
}
return store.Exists(repoID, objID)
}
func comp(c rune) bool {
return c == '/'
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
}
// IsDir check if the mode is dir.
func IsDir(m uint32) bool {
return (m & syscall.S_IFMT) == syscall.S_IFDIR
}
// IsRegular Check if the mode is regular.
func IsRegular(m uint32) bool {
return (m & syscall.S_IFMT) == syscall.S_IFREG
}
// ErrPathNoExist is an error indicating that the file does not exist
var ErrPathNoExist = fmt.Errorf("path does not exist")
// GetSeafdirByPath gets the object of seafdir by path.
func GetSeafdirByPath(repoID string, rootID string, path string) (*SeafDir, error) {
dir, err := GetSeafdir(repoID, rootID)
if err != nil {
errors := fmt.Errorf("directory is missing")
return nil, errors
}
path = filepath.Join("/", path)
parts := strings.FieldsFunc(path, comp)
var dirID string
for _, name := range parts {
entries := dir.Entries
for _, v := range entries {
if v.Name == name && IsDir(v.Mode) {
dirID = v.ID
break
}
}
if dirID == `` {
return nil, ErrPathNoExist
}
dir, err = GetSeafdir(repoID, dirID)
if err != nil {
errors := fmt.Errorf("directory is missing")
return nil, errors
}
}
return dir, nil
}
// GetSeafdirIDByPath gets the dirID of SeafDir by path.
func GetSeafdirIDByPath(repoID, rootID, path string) (string, error) {
dirID, mode, err := GetObjIDByPath(repoID, rootID, path)
if err != nil {
err := fmt.Errorf("failed to get dir id by path: %s: %w", path, err)
Go fileserver (#437) * Initial commit for fileserver written in golang. [gofileserver] Fix some syntaxt errors. Add fs backend and objstore test (#352) * Add fs backend and objstore test * modify test case and optimize fs backend * Modify function name and first write temporary files * Don't need to reopen the temp files Add comment for objstore (#354) * Add comment for objstore * Modify comment Add commitmgr and test case (#356) * Add commitmgr and test case * Redefine the interface * Modify comment and interface * Modify parameter and del unused method * Add comment for FromData and ToData Add blockmgr and test case (#357) * Add blockmgr and test case * Modify comment and interface Add fsmgr and test case (#358) * Add fsmgr and test case * Add save interface and error details * Modify errors and comments Add searpc package and test case (#360) * Add searpc package * Add searpc test case * Add return error and add Request struct * Modify returned error * Modify comments add checkPerm (#369) Add file and block download (#363) * Add file and block download * Modify init and use aes algorithm * Get block by offset and add stat method * Modify objID's type * Fix reset pos after add start * Add http error handing and record log when failed to read block or write block to response * Modify http return code and value names * Modify http return code and add log info * Block read add comment and only repeat once load ccnetdb and support sqlite (#371) Add zip download (#372) * Add zip download * Modify pack dir and log info * Modify http return code and use Deflate zip compression methods add /repo/<repo-id>/permission-check (#375) add /<repo-id>/commit/HEAD (#377) add /repo/<repo-id>/commit/<id> (#379) add /repo/<repo-id>/block/<id> (#380) add /repo/<repo-id>/fs-id-list (#383) add /repo/head-commits-multi (#388) Add file upload api (#378) * Add file upload api * Upload api implements post multi files and create relative path * Modify handle error and save files directly * Fix rebase conflict * index block use channel and optimize mkdir with parents * Handle jobs and results in a loop * Mkdir with parents use postMultiFiles and use pointer of SeafDirent * Del diff_simple size_sched virtual_repo * Need to check the path with and without slash * Modify merge trees and add merge test case * Del postFile and don't close results channel * Close the file and remove multipart temp file * Modify merge test case and compare the first name of path * Use pointer of Entries for SeafDir * Add test cases for different situations add /repo/<repo-id>/pack-fs (#389) add POST /<repo-id>/check-fs and /<repo-id>/check-blocks (#396) Merge compute repo (#397) * Add update repo size and merge virtual repo * Eliminate lint warnings * Uncomment merge virtual repo and compute repo size * Need init the dents * Use interface{} param and modify removeElems * Move update dir to file.go and modify logs * Del sync pkg add PUT /<repo-id>/commit/<commit-id> (#400) add PUT /<repo-id>/block/<id> (#401) add POST /<repo-id>/recv-fs (#398) add PUT /<repo-id>/commit/HEAD (#402) Add http return code (#403) Add file update API (#399) * Add file update API * Add GetObjIDByPath and fix change size error * Add traffic statistics for update api add diffTrees unit test (#391) add GET /accessible-repos (#406) add GET /<repo-id>/block-map/<file-id> (#405) Add test update repo size and merge virtual repo (#409) * Update dir need update repo size * Add test update repo size and merge virtual repo * Add delay for test ajax * Add delay before get repo size and modify comment Use go fileserver for unit test (#410) * Use go fileserver for unit test * Blocking scheduling update repo size * Add delay because of sqlite doesn't support concurrency * Post use multipart form encode * Del mysql database when test finished * Fix merge virtual repo failed when use sqlite3 Add upload block API (#412) fixed error Add quota-check API (#426) use diff package * Use central conf for go fileserver (#428) * Use central conf for go fileserver * Fix log error * use store id and remove share get repo owner (#430) * Fix permission error (#432) Co-authored-by: feiniks <36756310+feiniks@users.noreply.github.com> Co-authored-by: Xiangyue Cai <caixiangyue007@gmail.com>
2021-01-04 03:41:53 +00:00
return "", err
}
if dirID == "" || !IsDir(mode) {
return "", nil
}
return dirID, nil
}
// GetObjIDByPath gets the obj id by path
func GetObjIDByPath(repoID, rootID, path string) (string, uint32, error) {
var name string
var baseDir *SeafDir
formatPath := filepath.Join(path)
if len(formatPath) == 0 || formatPath == "/" {
return rootID, syscall.S_IFDIR, nil
}
index := strings.Index(formatPath, "/")
if index < 0 {
dir, err := GetSeafdir(repoID, rootID)
if err != nil {
err := fmt.Errorf("failed to find root dir %s: %v", rootID, err)
return "", 0, err
}
name = formatPath
baseDir = dir
} else {
name = filepath.Base(formatPath)
dirName := filepath.Dir(formatPath)
dir, err := GetSeafdirByPath(repoID, rootID, dirName)
if err != nil {
if err == ErrPathNoExist {
return "", syscall.S_IFDIR, ErrPathNoExist
}
err := fmt.Errorf("failed to find dir %s in repo %s: %v", dirName, repoID, err)
return "", syscall.S_IFDIR, err
}
baseDir = dir
}
entries := baseDir.Entries
for _, de := range entries {
if de.Name == name {
return de.ID, de.Mode, nil
}
}
return "", 0, nil
}
// GetFileCountInfoByPath gets the count info of file by path.
func GetFileCountInfoByPath(repoID, rootID, path string) (*FileCountInfo, error) {
dirID, err := GetSeafdirIDByPath(repoID, rootID, path)
if err != nil {
err := fmt.Errorf("failed to get file count info for repo %s path %s: %v", repoID, path, err)
return nil, err
}
info, err := getFileCountInfo(repoID, dirID)
if err != nil {
err := fmt.Errorf("failed to get file count in repo %s: %v", repoID, err)
return nil, err
}
return info, nil
}
func getFileCountInfo(repoID, dirID string) (*FileCountInfo, error) {
dir, err := GetSeafdir(repoID, dirID)
if err != nil {
err := fmt.Errorf("failed to get dir: %v", err)
return nil, err
}
info := new(FileCountInfo)
entries := dir.Entries
for _, de := range entries {
if IsDir(de.Mode) {
tmpInfo, err := getFileCountInfo(repoID, de.ID)
if err != nil {
err := fmt.Errorf("failed to get file count: %v", err)
return nil, err
}
info.DirCount = tmpInfo.DirCount + 1
info.FileCount += tmpInfo.FileCount
info.Size += tmpInfo.Size
} else {
info.FileCount++
info.Size += de.Size
}
}
return info, nil
}