Commit 4bdf69b
src/history/d1.py
@@ -92,7 +92,7 @@ async def backup_chat_history_to_d1(
saved_mids = glom(resp, "result.0.results.*.mid", default=[])
saved_mids = {int(x) for x in saved_mids}
offset_id = 0 # retrieve from latest message
- logger.info(f"Found {len(saved_mids)} messages of {table_name} in D1")
+ logger.info(f"Found {len(saved_mids)} messages of {table_name} in D1, Time >= {begin_time}")
concurrency = 100
num_sync = 0
tasks = []
@@ -101,12 +101,12 @@ async def backup_chat_history_to_d1(
if not isinstance(message, Message) or message.empty or message.service or message.id in saved_mids:
continue
info = parse_msg(message, silent=True, use_cache=False)
- if not fine_grained_check(info):
- continue
if info["time"] < begin_time:
break
if num_sync >= max_sync:
break
+ if not fine_grained_check(info):
+ continue
num_sync += 1
records = {
"mid": info["mid"],
src/history/turso.py
@@ -93,7 +93,7 @@ async def backup_chat_history_to_turso(
saved_mids = flatten(glom(resp, "results.0.response.result.rows.*.*.value", default=[]))
saved_mids = {int(x) for x in saved_mids}
offset_id = 0 # retrieve from latest message
- logger.info(f"Found {len(saved_mids)} messages in Turso. Rows read: {glom(resp, 'results.0.response.result.rows_read', default=1)}")
+ logger.info(f"Found {len(saved_mids)} messages of {table_name} in Turso. Time >= {begin_time}, Rows read: {glom(resp, 'results.0.response.result.rows_read', default=1)}")
concurrency = 1000
num_sync = 0
statements = []
@@ -102,12 +102,12 @@ async def backup_chat_history_to_turso(
if not isinstance(message, Message) or message.empty or message.service or message.id in saved_mids:
continue
info = parse_msg(message, silent=True, use_cache=False)
- if not fine_grained_check(info):
- continue
if info["time"] < begin_time:
break
if num_sync >= max_sync:
break
+ if not fine_grained_check(info):
+ continue
num_sync += 1
records = {
"mid": info["mid"],