haiwen/seahub
1
0
Fork 0
mirror of https://github.com/haiwen/seahub.git synced 2025-08-02 07:47:32 +00:00
Code Issues Releases Wiki Activity

Put task every 1000000 objs (#6015)

Browse Source 
Co-authored-by: 杨赫然 <heran.yang@seafile.com>
This commit is contained in:
feiniks 2024-04-23 10:34:29 +08:00 committed by GitHub
parent 50be4b437c
commit e39459c55f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/migrate.py
View File

@ -173,6 +173,14 @@ class ObjMigrateWorker(Thread):
logging.warning('[%s] Failed to write object %s from repo %s: %s' % (self.dtype, task.obj_id, task.repo_id, e))
raise
def put_task(self, objs):
if self.dest_store.get_name() != "filesystem storage backend":
random.shuffle(objs)
for obj in objs:
repo_id,obj_id=obj.split('/')
task = Task(repo_id, 1, obj_id)
self.thread_pool.put_task(task)
def migrate(self):
try:
obj_list = self.orig_store.list_objs(self.repo_id)
@ -187,14 +195,11 @@ class ObjMigrateWorker(Thread):
repo_id = obj[0]
obj_id = obj[1]
objs.append(repo_id+"/"+obj_id)
if len(objs) >= 1000000:
self.put_task(objs)
objs = []
if self.dest_store.get_name() != "filesystem storage backend":
random.shuffle(objs)
for obj in objs:
repo_id,obj_id=obj.split('/')
task = Task(repo_id, 1, obj_id)
self.thread_pool.put_task(task)
self.put_task(objs)
def invalid_obj(self, obj):
if len(obj) < 2: