Rank channel search results by relevance, run search off the event loop
list_channel previously matched every playlist with one unranked ".*keyword.*" regex, so a specific channel name could be buried pages behind unrelated names that merely contained the keyword mid-word. Split matches into two tiers (word-boundary match first, then anywhere-else) using a cheap count+skip/limit per tier so pagination still avoids pulling every match into memory. Also found while testing against a 1.19M-channel playlist: searching scans an unindexed regex and can take several seconds, and it was running synchronously inside the async handler, freezing every other request/socket for that whole time. Moved it to run_in_executor; verified a concurrent request now returns in ~2ms while a 3.6s search is in flight. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
685bbb1683
commit
4e596114c1
@@ -892,31 +892,44 @@ async def get_source(request):
|
||||
|
||||
|
||||
# List Channels
|
||||
async def list_channel(request):
|
||||
def _search_channels_sync(channel_to_search, current_page):
|
||||
MAX_RECORDS = 30
|
||||
data = await request.post()
|
||||
|
||||
channel_to_search = unquote(data['keyword'])
|
||||
current_page = int(data['num_page'])
|
||||
escaped_keyword = re.escape(channel_to_search)
|
||||
# Channel names are typically prefixed with a country/format tag (e.g.
|
||||
# "AR - BEIN SPORTS 1 FHD"), so match at word boundaries rather than the
|
||||
# start of the string to rank those above unrelated names that merely
|
||||
# contain the keyword mid-word.
|
||||
prefix_pattern = re.compile(r'\b' + escaped_keyword, re.IGNORECASE)
|
||||
contains_pattern = re.compile(escaped_keyword, re.IGNORECASE)
|
||||
|
||||
logger.info("Search keyword='%s' page=%s", channel_to_search, current_page)
|
||||
regex_pattern = ".*" + re.escape(channel_to_search) + ".*"
|
||||
|
||||
# First pass: compute totals without pulling all docs into memory
|
||||
total_records = 0
|
||||
active_playlists = list(playlists.find({"activation": 1}))
|
||||
per_playlist_counts = []
|
||||
|
||||
# Two relevance tiers, most relevant first: names starting with the keyword,
|
||||
# then everything else that merely contains it. Counted per tier/playlist so
|
||||
# pagination stays a cheap count+skip/limit instead of pulling every match
|
||||
# into memory to sort it.
|
||||
tiers = []
|
||||
for playlist in active_playlists:
|
||||
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||
count = collection_channels.count_documents({
|
||||
prefix_filter = {'id_playlist': playlist['id_playlist'], 'Name': prefix_pattern}
|
||||
count = collection_channels.count_documents(prefix_filter)
|
||||
if count:
|
||||
tiers.append((playlist, prefix_filter, count))
|
||||
for playlist in active_playlists:
|
||||
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||
contains_only_filter = {
|
||||
'id_playlist': playlist['id_playlist'],
|
||||
'Name': {'$regex': regex_pattern, '$options': "i"}
|
||||
})
|
||||
per_playlist_counts.append((playlist, count))
|
||||
total_records += count
|
||||
'$and': [{'Name': contains_pattern}, {'Name': {'$not': prefix_pattern}}],
|
||||
}
|
||||
count = collection_channels.count_documents(contains_only_filter)
|
||||
if count:
|
||||
tiers.append((playlist, contains_only_filter, count))
|
||||
|
||||
total_records = sum(count for _, _, count in tiers)
|
||||
|
||||
if total_records == 0:
|
||||
return web.json_response({'num_total_page': 0, 'current_page': 1, 'data': []})
|
||||
return {'num_total_page': 0, 'current_page': 1, 'data': []}
|
||||
|
||||
num_total_page = max(1, (total_records + MAX_RECORDS - 1) // MAX_RECORDS)
|
||||
current_page = max(1, min(current_page, num_total_page))
|
||||
@@ -926,7 +939,7 @@ async def list_channel(request):
|
||||
# Second pass: fetch only the page slice
|
||||
serialized_data = []
|
||||
offset = 0
|
||||
for playlist, count in per_playlist_counts:
|
||||
for playlist, mongo_filter, count in tiers:
|
||||
if remaining == 0:
|
||||
break
|
||||
if offset + count <= begin:
|
||||
@@ -936,16 +949,30 @@ async def list_channel(request):
|
||||
local_skip = max(0, begin - offset)
|
||||
fetch_limit = min(remaining, count - local_skip)
|
||||
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||
cursor = collection_channels.find({
|
||||
'id_playlist': playlist['id_playlist'],
|
||||
'Name': {'$regex': regex_pattern, '$options': "i"}
|
||||
}).skip(local_skip).limit(fetch_limit)
|
||||
cursor = collection_channels.find(mongo_filter).skip(local_skip).limit(fetch_limit)
|
||||
|
||||
serialized_data.extend(serialize_doc(doc) for doc in cursor)
|
||||
remaining = MAX_RECORDS - len(serialized_data)
|
||||
offset += count
|
||||
|
||||
return web.json_response({'num_total_page': num_total_page, 'current_page': current_page, 'data': serialized_data})
|
||||
return {'num_total_page': num_total_page, 'current_page': current_page, 'data': serialized_data}
|
||||
|
||||
|
||||
async def list_channel(request):
|
||||
data = await request.post()
|
||||
|
||||
channel_to_search = unquote(data['keyword'])
|
||||
current_page = int(data['num_page'])
|
||||
|
||||
logger.info("Search keyword='%s' page=%s", channel_to_search, current_page)
|
||||
|
||||
# Searching can mean scanning a multi-hundred-thousand-channel collection
|
||||
# with an unindexed regex (several seconds); run it off the event loop so
|
||||
# it doesn't stall every other request/socket while it works.
|
||||
result = await asyncio.get_running_loop().run_in_executor(
|
||||
None, _search_channels_sync, channel_to_search, current_page
|
||||
)
|
||||
return web.json_response(result)
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user