fxreader.online/freelance-project-34-marketing-blog
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

[~] Refactor

Browse Source
This commit is contained in:
Siarhei Siniak 2024-07-06 19:17:56 +03:00
parent b234330471
commit cffbe712c6
3 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docker/tiktok/Makefile
View File

@ -20,6 +20,12 @@ celery-up:
celery-stop: celery-stop:
$(MAKE) c ARGS="stop redis celery" $(MAKE) c ARGS="stop redis celery"
logs:
$(MAKE) c ARGS="logs --tail=100 -f"
celery-restart:
$(MAKE) c ARGS="restart celery"
run: run:
cd ${PROJECT_ROOT} && \ cd ${PROJECT_ROOT} && \
sudo docker-compose \ sudo docker-compose \

11
python/tasks/tiktok/__init__.py
View File

@ -13,6 +13,7 @@ from typing import (
Optional, Optional,
Iterable, Iterable,
) )
import celery
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -26,6 +27,8 @@ class tiktok_config_t:
videos: str='' videos: str=''
audios: str='' audios: str=''
celery_broker: str='' celery_broker: str=''
celery_result_backend: str=''
celery_imports: Iterable[str]=tuple()
def tiktok_config() -> tiktok_config_t.res_t: def tiktok_config() -> tiktok_config_t.res_t:
res = tiktok_config_t.res_t( res = tiktok_config_t.res_t(
@ -38,6 +41,8 @@ def tiktok_config() -> tiktok_config_t.res_t:
) )
res.celery_broker = 'redis://@redis/1' res.celery_broker = 'redis://@redis/1'
res.celery_result_backend = 'redis://@redis/2'
res.celery_imports = ['python.tasks.tiktok']
res.cache = os.path.join( res.cache = os.path.join(
res.project_root, res.project_root,
'tmp/cache/tiktok', 'tmp/cache/tiktok',
@ -64,6 +69,7 @@ def logger_setup():
logger_setup() logger_setup()
@celery.shared_task()
async def tiktok_videos_links_get( async def tiktok_videos_links_get(
query: Optional[str]=None, query: Optional[str]=None,
screenshot_path: Optional[str]=None, screenshot_path: Optional[str]=None,
@ -136,6 +142,7 @@ async def tiktok_videos_links_get(
return list(links)[:max_links] return list(links)[:max_links]
@celery.shared_task()
def tiktok_videos_meta(links: Iterable[str]) -> Iterable[dict[str, Any]]: def tiktok_videos_meta(links: Iterable[str]) -> Iterable[dict[str, Any]]:
res = [] res = []
for o in links: for o in links:
@ -156,6 +163,7 @@ class tiktok_video_fetch_t:
tikcdn_io_curl = 'tikcdn.io-curl' tikcdn_io_curl = 'tikcdn.io-curl'
tikcdn_io_wget = 'tikcdn.io-wget' tikcdn_io_wget = 'tikcdn.io-wget'
@celery.shared_task()
def tiktok_video_fetch( def tiktok_video_fetch(
id: int, id: int,
url: str, url: str,
@ -222,6 +230,7 @@ async def playwright_save(url: str):
download.save_as(path) download.save_as(path)
print(path) print(path)
@celery.shared_task()
def tiktok_videos_fetch( def tiktok_videos_fetch(
meta: Iterable[dict[str, Any]], meta: Iterable[dict[str, Any]],
method: Optional[tiktok_video_fetch_t.method_t]=None, method: Optional[tiktok_video_fetch_t.method_t]=None,
@ -278,6 +287,7 @@ def tiktok_videos_fetch(
return stats return stats
@celery.shared_task()
def tiktok_videos_process(meta: Iterable[dict[str, Any]]) -> dict[str, Any]: def tiktok_videos_process(meta: Iterable[dict[str, Any]]) -> dict[str, Any]:
import tqdm import tqdm
stats = dict( stats = dict(
@ -355,6 +365,7 @@ class audio_get_t:
path_mp3: str path_mp3: str
url: str url: str
@celery.shared_task()
def audio_get() -> audio_get_t.res_t: def audio_get() -> audio_get_t.res_t:
c = tiktok_config() c = tiktok_config()
url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'

4
python/tasks/tiktok/celery.py
View File

@ -4,7 +4,11 @@ import celery
c = tiktok_config() c = tiktok_config()
CELERY_IMPORTS = c.celery_imports
app = celery.Celery( app = celery.Celery(
__name__, __name__,
broker=c.celery_broker, broker=c.celery_broker,
result_backend=c.celery_result_backend,
) )