fxreader.online/freelance-project-34-marketing-blog
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

[~] Refactor

Browse Source
This commit is contained in:
Siarhei Siniak 2024-07-07 12:19:59 +03:00
parent 4d2e9a3fd7
commit 5db573c025
1 changed files with 30 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
python/tasks/tiktok/tasks.py
View File

@ -204,41 +204,37 @@ def tiktok_videos_fetch(
error=0,
)
with multiprocessing.Pool(processes=1) as pool:
for o in tqdm.tqdm(meta):
stats['total'] += 1
path = os.path.join(
o['result_dir'],
o['fname'],
)
for o in tqdm.tqdm(meta):
stats['total'] += 1
path = os.path.join(
o['result_dir'],
o['fname'],
)
if (
not os.path.exists(path) or
file_mime_type(path) in ['empty'] or
force
):
try:
pool.apply(
tiktok_video_fetch,
kwds=dict(
id=o['id'],
url=o['url'],
fname=o['fname'],
method=method,
method_str=method_str,
result_dir=o['result_dir'],
),
)
stats['saved'] += 1
except KeyboardInterrupt:
break
except:
logger.error(json.dumps(dict(
msg=traceback.format_exc(),
)))
stats['error'] += 1
else:
stats['skipped'] += 1
if (
not os.path.exists(path) or
file_mime_type(path) in ['empty'] or
force
):
try:
tiktok_video_fetch.apply_async(
id=o['id'],
url=o['url'],
fname=o['fname'],
method=method,
method_str=method_str,
result_dir=o['result_dir'],
).wait()
stats['saved'] += 1
except KeyboardInterrupt:
break
except:
logger.error(json.dumps(dict(
msg=traceback.format_exc(),
)))
stats['error'] += 1
else:
stats['skipped'] += 1
return stats