Commit de2648e
Changed files (7)
src/preview/bilibili.py
@@ -15,7 +15,6 @@ from bilibili_api import ApiException, Credential, comment, opus, video
from glom import Coalesce, flatten, glom
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from config import DB, READING_SPEED, TZ, cache
@@ -24,7 +23,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import summay_media
+from messages.utils import blockquote, summay_media
from networking import download_file, download_media, hx_req
from others.emoji import emojify
from utils import av2bv, count_subtitles, https_url, number_to_emoji, readable_count, seconds_to_hms, ts_to_dt
@@ -265,8 +264,9 @@ async def get_bilibili_comments(url_or_vid: int | str) -> list[str]:
location = f"({location})" if location else ""
if cmt := glom(x, "content.message", default=""):
if idx == 0:
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:")
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{name}**{location}: {emojify(cmt)}")
+ comments.append(f"\n{blockquote('💬**点此展开评论区**:')}")
+ cmt = f"💬**{name}**{location}: {emojify(cmt)}"
+ comments.append(f"\n{blockquote(cmt)}")
except Exception as e:
logger.error(f"Failed to get Bilibili comments: {e}")
return []
src/preview/douyin.py
@@ -11,7 +11,6 @@ from glom import Coalesce, glom
from glom import Path as GlomPath
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from bridge.social import send_to_social_media_bridge
@@ -20,7 +19,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import summay_media
+from messages.utils import blockquote, summay_media
from networking import download_file, download_first_success_urls, download_media, hx_req
from others.emoji import emojify
from utils import rand_number, readable_count, true
@@ -90,12 +89,8 @@ async def preview_douyin(
if true(show_description) and data.get("desc"):
texts += f"\n{data['desc']}"
- comments = []
- if comments_list := await get_comments(data["aweme_id"], platform, douyin_comments_provider):
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:")
- comments.extend(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{cmt['name']}**{cmt['region']}: {cmt['text']}" for cmt in comments_list)
-
- sent_messages = await send2tg(client, message, texts=emojify(texts), media=data.get("media", []), comments=comments, **kwargs)
+ comments = await get_comments(data["aweme_id"], platform, douyin_comments_provider)
+ sent_messages = await send2tg(client, message, texts=emojify(texts) + comments, media=data.get("media", []), **kwargs)
await modify_progress(del_status=True, **kwargs)
await save_messages(messages=sent_messages, key=db_key)
@@ -243,7 +238,7 @@ def prefer_jpg_urls(url_list: list[str] | None = None) -> list[str]:
return urls
-async def get_comments(aweme_id: str = "", platform: str = "douyin", douyin_comments_provider: str = PROVIDER.DOUYIN_COMMENTS) -> list[dict]:
+async def get_comments(aweme_id: str = "", platform: str = "douyin", douyin_comments_provider: str = PROVIDER.DOUYIN_COMMENTS) -> str:
"""Fetch douyin or tiktok comments.
Args:
@@ -252,11 +247,11 @@ async def get_comments(aweme_id: str = "", platform: str = "douyin", douyin_comm
douyin_comments_provider (str, optional): The douyin comments extractor: "free" or "tikhub".
Returns:
- list[dict]: comments list.
+ str: comments string.
"""
if not true(douyin_comments_provider):
- return []
- comments = []
+ return ""
+
api_urls = {
"douyin_tikhub": f"{API.TIKHUB}/api/v1/douyin/app/v3/fetch_video_comments?aweme_id={aweme_id}",
"douyin_free": f"{API.TIKHUB_FREE}/api/douyin/web/fetch_video_comments?aweme_id={aweme_id}",
@@ -282,7 +277,8 @@ async def get_comments(aweme_id: str = "", platform: str = "douyin", douyin_comm
data = resp["data"].get("comments", [])
except Exception:
logger.warning(f"{platform} comments API [tikhub] failed")
- return []
+ return ""
+ comments = []
try:
for node in data:
name = glom(node, "user.nickname", default="")
@@ -294,5 +290,12 @@ async def get_comments(aweme_id: str = "", platform: str = "douyin", douyin_comm
comments.append({"name": name, "text": emojify(text.strip()), "region": region})
except Exception as e:
logger.error(e)
- return []
- return comments
+ return ""
+
+ comments_str = ""
+ for idx, cmt in enumerate(comments):
+ if idx == 0:
+ comments_str += f"\n{blockquote('💬**点此展开评论区**:')}"
+ cmt_str = f"💬**{cmt['name']}**{cmt['region']}: {cmt['text']}"
+ comments_str += f"\n{blockquote(cmt_str)}"
+ return comments_str
src/preview/instagram.py
@@ -8,7 +8,6 @@ from bs4 import BeautifulSoup
from glom import glom
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from bridge.social import send_to_social_media_bridge
@@ -17,7 +16,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import summay_media
+from messages.utils import blockquote, summay_media
from multimedia import is_valid_video_or_audio, validate_img
from networking import download_file, download_media, hx_req
from utils import readable_count, true
@@ -103,18 +102,21 @@ async def preview_instagram(
if true(show_description) and (description := glom(metadata_node, "node.text", default="")):
texts += f"{description}\n"
# parse comments
- comments: list[str] = []
+ comments = ""
if true(instagram_comments):
comment_nodes = glom(data, "edge_media_to_parent_comment.edges", default=[])
comment_nodes = sorted(comment_nodes, key=lambda x: glom(x, "node.created_at", default=0))
- comment_list = [{"author": glom(node, "node.owner.username", default="user"), "text": glom(node, "node.text", default="")} for node in comment_nodes]
- if comment_list := [x for x in comment_list if x["text"]]:
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:")
- comments.extend(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**[{cmt['author']}](https://www.instagram.com/{cmt['author']})**: {cmt['text']}" for cmt in comment_list)
+ for idx, node in enumerate(comment_nodes):
+ if idx == 0:
+ comments += f"\n{blockquote('💬**点此展开评论区**:')}"
+ author = glom(node, "node.owner.username", default="user")
+ cmt = glom(node, "node.text", default="")
+ comment = f"💬**[{author}](https://www.instagram.com/{author})**: {cmt}"
+ comments += f"\n{blockquote(comment)}"
await modify_progress(text=f"⏬正在下载:\n{summay_media(media)}", force_update=True, **kwargs)
media = await download_media(media, **kwargs)
- sent_messages = await send2tg(client, message, texts=texts.strip(), media=media, comments=comments, **kwargs)
+ sent_messages = await send2tg(client, message, texts=texts.strip() + comments, media=media, **kwargs)
await modify_progress(del_status=True, **kwargs)
await save_messages(messages=sent_messages, key=db_key)
src/preview/reddit.py
@@ -8,7 +8,6 @@ from zoneinfo import ZoneInfo
from glom import glom
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from config import DB, PROXY, TZ
@@ -16,7 +15,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import summay_media
+from messages.utils import blockquote, summay_media
from networking import download_file, download_media, hx_req
from preview.utils import has_markdown_img
from utils import nowstr
@@ -77,7 +76,7 @@ async def get_reddit_info(url: str, **kwargs) -> dict:
media.append({"photo": download_file(data["url"], proxy=PROXY.REDDIT, **kwargs)})
if video_url := glom(data, "secure_media.reddit_video.fallback_url", default=""):
media.append({"video": download_file(video_url, proxy=PROXY.REDDIT, **kwargs)})
- comments = []
+ comments = ""
for reply in glom(resp, "1.data.children.*.data"):
author = reply.get("author", "author")
author_url = f"https://www.reddit.com/user/{author}"
@@ -86,15 +85,16 @@ async def get_reddit_info(url: str, **kwargs) -> dict:
continue
if comment == "[removed]" or has_markdown_img(comment):
continue
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**[{author}]({author_url})**: {comment}")
+ cmt = f"💬**[{author}]({author_url})**: {comment}"
+ comments += f"\n{blockquote(cmt)}"
if comments:
- comments.insert(0, f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:")
+ comments = f"\n{blockquote('💬**点此展开评论区**:')}{comments}"
await modify_progress(text=f"⏬正在下载:\n{summay_media(media)}", force_update=True, **kwargs)
media = await download_media(media, **kwargs)
except Exception as e:
logger.error(e)
return {"error": str(e)}
- return {"texts": texts, "media": media, "comments": comments}
+ return {"texts": texts + comments, "media": media}
def remove_preview_links(text: str) -> str:
src/preview/twitter.py
@@ -8,7 +8,6 @@ from zoneinfo import ZoneInfo
from glom import glom
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from bridge.social import send_to_social_media_bridge
@@ -17,7 +16,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import remove_img_tag, summay_media
+from messages.utils import blockquote, remove_img_tag, summay_media
from networking import download_file, download_media, flatten_rediercts, hx_req
from utils import convert_html, readable_count, remove_consecutive_newlines, remove_none_values, split_parts, true
@@ -165,11 +164,12 @@ async def preview_twitter(
if texts := master_info.get("texts"):
msg += f"\n{texts}"
if true(twitter_comments) and (comments := master_info.get("comments")):
- msg += f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:"
+ msg += f"\n{blockquote('💬**点此展开评论区**:')}"
for cmt in comments:
if str(cmt["post_id"]) == str(this_info["post_id"]):
continue
- msg += f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{cmt['author']}**: {cmt['text']}"
+ full_cmt = f"💬**{cmt['author']}**: {cmt['text']}"
+ msg += f"\n{blockquote(full_cmt)}"
media.extend(master_media)
# 本条推文
@@ -193,10 +193,11 @@ async def preview_twitter(
msg += f"\n{texts}"
if true(twitter_comments) and (comments := this_info.get("comments")):
- msg += f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:"
+ msg += f"\n{blockquote('💬**点此展开评论区**:')}"
for cmt in comments:
cmt_texts = cmt["text"].strip().removeprefix(f"@{master_handle}").strip() # 有时回推的comment前会附带被回推的handle, 这里去掉
- msg += f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{cmt['author']}**: {cmt_texts}"
+ full_cmt = f"💬**{cmt['author']}**: {cmt_texts}"
+ msg += f"\n{blockquote(full_cmt)}"
# 引用推文
if quote_info:
@@ -550,9 +551,8 @@ def parse_article(article: dict) -> dict:
e_type = entity.get("type", "").upper()
if e_type == "MEDIA":
media_id = glom(entity, "data.mediaItems.0.mediaId", default="")
- if media_dict.get(str(media_id)):
- # texts += f""
- texts += "[IMAGE]"
+ if img_url := media_dict.get(str(media_id)):
+ texts += f""
elif e_type == "DIVIDER":
texts += "\n"
elif e_type == "TWEET":
src/preview/weibo.py
@@ -11,7 +11,6 @@ from bs4 import BeautifulSoup
from glom import glom
from loguru import logger
from pyrogram.client import Client
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from pyrogram.types import Message
from bridge.social import send_to_social_media_bridge
@@ -21,7 +20,7 @@ from database.database import get_db
from messages.database import copy_messages_from_db, save_messages
from messages.progress import modify_progress
from messages.sender import send2tg
-from messages.utils import summay_media
+from messages.utils import blockquote, summay_media
from networking import download_file, download_first_success_urls, download_media, hx_req
from others.emoji import emojify
from utils import rand_string, readable_count, soup_to_text, split_parts, true
@@ -136,9 +135,7 @@ async def preview_weibo(
comments = []
if true(weibo_comments):
- comments = this_info.get("comments", [])
- if not comments:
- comments = await parse_weibo_comments(post_id)
+ comments = await parse_weibo_comments(post_id)
sent_messages = await send2tg(client, message, texts=emojify(msg.strip()), media=media, comments=comments, **kwargs)
await modify_progress(del_status=True, **kwargs)
await save_messages(messages=sent_messages, key=db_key)
@@ -246,9 +243,9 @@ async def weibo_vid_to_postid(post_id: str) -> str:
@cache.memoize(ttl=30)
-async def parse_weibo_comments(post_id: str) -> list[str]:
+async def parse_weibo_comments(post_id: str) -> str:
if not post_id:
- return []
+ return ""
headers = {
"cookie": await get_weibo_cookies(),
"accept": "application/json, text/plain, */*",
@@ -282,9 +279,9 @@ async def parse_weibo_comments(post_id: str) -> list[str]:
resp = await hx_req(api, headers=headers, params=params, proxy=PROXY.WEIBO, check_kv={"ok": 1}, max_retry=1)
if resp.get("hx_error"):
logger.error(f"Weibo Comments API failed: {resp}")
- return []
+ return ""
- comments = [f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:"]
+ comments = ""
for info in resp.get("data", []):
if not info.get("text"):
continue
@@ -292,18 +289,18 @@ async def parse_weibo_comments(post_id: str) -> list[str]:
uid = glom(info, "user.id", default="")
author = glom(info, "user.screen_name", default="")
if author and uid:
- cmt += f"{BLOCKQUOTE_EXPANDABLE_DELIM}💬**[{author}](https://weibo.com/u/{uid})**"
+ cmt += f"💬**[{author}](https://weibo.com/u/{uid})**"
elif author:
- cmt += f"{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{author}**"
+ cmt += f"💬**{author}**"
if region := info.get("source", "").removeprefix("来自"):
cmt += f"({region})"
cmt += ":"
if text := info.get("text"):
cmt += f" {soup_to_text(BeautifulSoup(text, 'html.parser'))}"
cmt = emojify(cmt)
- comments.append(f"\n{cmt}")
- if len(comments) == 1:
- return []
+ comments += f"\n{blockquote(cmt)}"
+ if comments:
+ comments = f"\n{blockquote('💬**点此展开评论区**:')}{comments}"
return comments
src/preview/youtube.py
@@ -12,9 +12,9 @@ from zoneinfo import ZoneInfo
from glom import Coalesce, glom
from loguru import logger
-from pyrogram.parser.markdown import BLOCKQUOTE_EXPANDABLE_DELIM
from config import PROXY, TOKEN, TZ, cache
+from messages.utils import blockquote
from networking import hx_req
from utils import nowstr, readable_count, true
@@ -39,8 +39,9 @@ async def get_youtube_comments(vid: str | None) -> list[str]:
name = f"[{name}]({author_url})"
if cmt := glom(x, "snippet.topLevelComment.snippet.textDisplay", default=""):
if idx == 0:
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**点此展开评论区**:")
- comments.append(f"\n{BLOCKQUOTE_EXPANDABLE_DELIM}💬**{name}**: {cmt}")
+ comments.append(f"\n{blockquote('💬**点此展开评论区**:')}")
+ cmt = f"💬**{name}**: {cmt}"
+ comments.append(f"\n{blockquote(cmt)}")
except Exception as e:
logger.error(f"Failed to get YouTube comments: {e}")
return []