diff --git a/fileserver/fileop.go b/fileserver/fileop.go index e0ba093..660fddc 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -7,12 +7,13 @@ import ( "crypto/sha1" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/ioutil" - "log" "mime" "mime/multipart" + "net" "net/http" "net/url" "os" @@ -34,6 +35,7 @@ import ( "github.com/haiwen/seafile-server/fileserver/diff" "github.com/haiwen/seafile-server/fileserver/fsmgr" "github.com/haiwen/seafile-server/fileserver/repomgr" + log "github.com/sirupsen/logrus" ) const ( @@ -294,7 +296,9 @@ func doFile(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, fileID for _, blkID := range file.BlkIDs { err := blockmgr.Read(repo.StoreID, blkID, rsp) if err != nil { - log.Printf("fatild to write block %s to response: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } return nil } } @@ -310,6 +314,14 @@ func doFile(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, fileID return nil } +func isNetworkErr(err error) bool { + _, ok := err.(net.Error) + if ok { + return true + } + return false +} + type blockMap struct { blkSize []uint64 expireTime int64 @@ -402,7 +414,9 @@ func doFileRange(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, f if end-start+1 <= blkSize[i]-pos { err := blockmgr.Read(repo.StoreID, blkID, &buf) if err != nil { - log.Printf("failed to read block %s: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } return nil } recvBuf := buf.Bytes() @@ -415,7 +429,9 @@ func doFileRange(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, f err := blockmgr.Read(repo.StoreID, blkID, &buf) if err != nil { - log.Printf("failed to read block %s: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } return nil } recvBuf := buf.Bytes() @@ -436,7 +452,9 @@ func doFileRange(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, f if end-start+1 <= blkSize[i] { err := blockmgr.Read(repo.StoreID, blkID, &buf) if err != nil { - log.Printf("failed to read block %s: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } return nil } recvBuf := buf.Bytes() @@ -449,7 +467,9 @@ func doFileRange(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, f } else { err := blockmgr.Read(repo.StoreID, blkID, rsp) if err != nil { - log.Printf("failed to write block %s to response: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } return nil } start += blkSize[i] @@ -645,7 +665,9 @@ func doBlock(rsp http.ResponseWriter, r *http.Request, repo *repomgr.Repo, fileI err = blockmgr.Read(repo.StoreID, blkID, rsp) if err != nil { - log.Printf("fatild to write block %s to response: %v", blkID, err) + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blkID, err) + } } sendStatisticMsg(repo.StoreID, user, "web-file-download", uint64(size)) @@ -744,12 +766,16 @@ func downloadZipFile(rsp http.ResponseWriter, r *http.Request, data, repoID, use for _, v := range dirList { if fsmgr.IsDir(v.Mode) { if err := packDir(ar, repo, v.ID, v.Name); err != nil { - log.Printf("failed to pack dir %s: %v", v.Name, err) + if !isNetworkErr(err) { + log.Printf("failed to pack dir %s: %v", v.Name, err) + } return nil } } else { if err := packFiles(ar, &v, repo, ""); err != nil { - log.Printf("failed to pack file %s: %v", v.Name, err) + if !isNetworkErr(err) { + log.Printf("failed to pack file %s: %v", v.Name, err) + } return nil } } @@ -1467,8 +1493,11 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, for _, filePath := range files { id, size, err := indexBlocks(r.Context(), repo.StoreID, repo.Version, filePath, nil, cryptKey) if err != nil { - err := fmt.Errorf("failed to index blocks: %v", err) - return &appError{err, "", http.StatusInternalServerError} + if !errors.Is(err, context.Canceled) { + err := fmt.Errorf("failed to index blocks: %v", err) + return &appError{err, "", http.StatusInternalServerError} + } + return &appError{nil, "", http.StatusInternalServerError} } ids = append(ids, id) sizes = append(sizes, size) @@ -1477,8 +1506,11 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, for _, handler := range fsm.fileHeaders { id, size, err := indexBlocks(r.Context(), repo.StoreID, repo.Version, "", handler, cryptKey) if err != nil { - err := fmt.Errorf("failed to index blocks: %v", err) - return &appError{err, "", http.StatusInternalServerError} + if !errors.Is(err, context.Canceled) { + err := fmt.Errorf("failed to index blocks: %v", err) + return &appError{err, "", http.StatusInternalServerError} + } + return &appError{nil, "", http.StatusInternalServerError} } ids = append(ids, id) sizes = append(sizes, size) @@ -2112,7 +2144,7 @@ func chunkingWorker(ctx context.Context, wg *sync.WaitGroup, chunkJobs chan chun for job := range chunkJobs { select { case <-ctx.Done(): - err := fmt.Errorf("chunk work canceled") + err := context.Canceled result := chunkingResult{-1, "", err} res <- result wg.Done() @@ -2769,8 +2801,11 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user, filePath := files[0] id, fileSize, err := indexBlocks(r.Context(), repo.StoreID, repo.Version, filePath, nil, cryptKey) if err != nil { - err := fmt.Errorf("failed to index blocks: %v", err) - return &appError{err, "", http.StatusInternalServerError} + if !errors.Is(err, context.Canceled) { + err := fmt.Errorf("failed to index blocks: %w", err) + return &appError{err, "", http.StatusInternalServerError} + } + return &appError{nil, "", http.StatusInternalServerError} } fileID = id size = fileSize @@ -2778,8 +2813,11 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user, handler := fsm.fileHeaders[0] id, fileSize, err := indexBlocks(r.Context(), repo.StoreID, repo.Version, "", handler, cryptKey) if err != nil { - err := fmt.Errorf("failed to index blocks: %v", err) - return &appError{err, "", http.StatusInternalServerError} + if !errors.Is(err, context.Canceled) { + err := fmt.Errorf("failed to index blocks: %w", err) + return &appError{err, "", http.StatusInternalServerError} + } + return &appError{nil, "", http.StatusInternalServerError} } fileID = id size = fileSize diff --git a/fileserver/fileserver.go b/fileserver/fileserver.go index 3658e3b..a78846c 100644 --- a/fileserver/fileserver.go +++ b/fileserver/fileserver.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "io" - "log" "net/http" "os" "os/signal" @@ -25,6 +24,7 @@ import ( "github.com/haiwen/seafile-server/fileserver/searpc" "github.com/haiwen/seafile-server/fileserver/share" _ "github.com/mattn/go-sqlite3" + log "github.com/sirupsen/logrus" "gopkg.in/ini.v1" ) @@ -77,6 +77,22 @@ func init() { flag.StringVar(&logFile, "l", "", "log file path") flag.StringVar(&rpcPipePath, "p", "", "rpc pipe path") flag.StringVar(&pidFilePath, "P", "", "pid file path") + + log.SetFormatter(&LogFormatter{}) +} + +const ( + timestampFormat = "[2006-01-02 15:04:05] " +) + +type LogFormatter struct{} + +func (f *LogFormatter) Format(entry *log.Entry) ([]byte, error) { + buf := make([]byte, 0, len(timestampFormat)+len(entry.Message)+1) + buf = entry.Time.AppendFormat(buf, timestampFormat) + buf = append(buf, entry.Message...) + buf = append(buf, '\n') + return buf, nil } func loadCcnetDB() { @@ -435,7 +451,7 @@ func main() { } // When logFile is "-", use default output (StdOut) - log.SetFlags(log.Ldate | log.Ltime) + log.SetLevel(log.InfoLevel) if absLogFile != "" { errorLogFile := filepath.Join(filepath.Dir(absLogFile), "fileserver-error.log") diff --git a/fileserver/go.mod b/fileserver/go.mod index b90c647..eec1d38 100644 --- a/fileserver/go.mod +++ b/fileserver/go.mod @@ -7,6 +7,7 @@ require ( github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/mattn/go-sqlite3 v1.14.0 + github.com/sirupsen/logrus v1.8.1 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect gopkg.in/ini.v1 v1.55.0 ) diff --git a/fileserver/go.sum b/fileserver/go.sum index 64a4a29..3a86228 100644 --- a/fileserver/go.sum +++ b/fileserver/go.sum @@ -1,5 +1,6 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -11,17 +12,23 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.3 h1:QdmJJYlDQhMDFrFP8IvVnx66D8mCbaQM4TsxKf7BXzo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/fileserver/repomgr/repomgr.go b/fileserver/repomgr/repomgr.go index 93b0a48..db59fa6 100644 --- a/fileserver/repomgr/repomgr.go +++ b/fileserver/repomgr/repomgr.go @@ -4,12 +4,12 @@ package repomgr import ( "database/sql" "fmt" - "log" "time" // Change to non-blank imports when use _ "github.com/haiwen/seafile-server/fileserver/blockmgr" "github.com/haiwen/seafile-server/fileserver/commitmgr" + log "github.com/sirupsen/logrus" ) // Repo status diff --git a/fileserver/share/share.go b/fileserver/share/share.go index 333d25d..0d088ad 100644 --- a/fileserver/share/share.go +++ b/fileserver/share/share.go @@ -5,12 +5,12 @@ package share import ( "database/sql" "fmt" - "log" "path/filepath" "strconv" "strings" "github.com/haiwen/seafile-server/fileserver/repomgr" + log "github.com/sirupsen/logrus" ) type group struct { diff --git a/fileserver/size_sched.go b/fileserver/size_sched.go index 28e2fa3..8926224 100644 --- a/fileserver/size_sched.go +++ b/fileserver/size_sched.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "path/filepath" "gopkg.in/ini.v1" @@ -14,6 +13,7 @@ import ( "github.com/haiwen/seafile-server/fileserver/fsmgr" "github.com/haiwen/seafile-server/fileserver/repomgr" "github.com/haiwen/seafile-server/fileserver/workerpool" + log "github.com/sirupsen/logrus" ) var updateSizePool *workerpool.WorkPool diff --git a/fileserver/sync_api.go b/fileserver/sync_api.go index ba6c1b9..fb3bbe8 100644 --- a/fileserver/sync_api.go +++ b/fileserver/sync_api.go @@ -9,7 +9,6 @@ import ( "fmt" "html" "io/ioutil" - "log" "net" "net/http" "strconv" @@ -25,6 +24,7 @@ import ( "github.com/haiwen/seafile-server/fileserver/fsmgr" "github.com/haiwen/seafile-server/fileserver/repomgr" "github.com/haiwen/seafile-server/fileserver/share" + log "github.com/sirupsen/logrus" ) type checkExistType int32 @@ -730,8 +730,6 @@ func putSendBlockCB(rsp http.ResponseWriter, r *http.Request) *appError { return &appError{err, "", http.StatusInternalServerError} } - rsp.WriteHeader(http.StatusOK) - sendStatisticMsg(storeID, user, "sync-file-upload", uint64(r.ContentLength)) return nil @@ -770,7 +768,10 @@ func getBlockInfo(rsp http.ResponseWriter, r *http.Request) *appError { blockLen := fmt.Sprintf("%d", blockSize) rsp.Header().Set("Content-Length", blockLen) if err := blockmgr.Read(storeID, blockID, rsp); err != nil { - return &appError{err, "", http.StatusInternalServerError} + if !isNetworkErr(err) { + log.Printf("failed to read block %s: %v", blockID, err) + } + return nil } sendStatisticMsg(storeID, user, "sync-file-download", uint64(blockSize)) diff --git a/fileserver/virtual_repo.go b/fileserver/virtual_repo.go index be4c044..2030913 100644 --- a/fileserver/virtual_repo.go +++ b/fileserver/virtual_repo.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "log" "path/filepath" "strings" "time" @@ -14,6 +13,7 @@ import ( "github.com/haiwen/seafile-server/fileserver/fsmgr" "github.com/haiwen/seafile-server/fileserver/repomgr" "github.com/haiwen/seafile-server/fileserver/workerpool" + log "github.com/sirupsen/logrus" ) const mergeVirtualRepoWorkerNumber = 5 diff --git a/fileserver/workerpool/workerpool.go b/fileserver/workerpool/workerpool.go index 61f13c1..fc61d10 100644 --- a/fileserver/workerpool/workerpool.go +++ b/fileserver/workerpool/workerpool.go @@ -1,7 +1,7 @@ package workerpool import ( - "log" + log "github.com/sirupsen/logrus" "runtime/debug" )