Rank channel search results by relevance, run search off the event loop
list_channel previously matched every playlist with one unranked ".*keyword.*" regex, so a specific channel name could be buried pages behind unrelated names that merely contained the keyword mid-word. Split matches into two tiers (word-boundary match first, then anywhere-else) using a cheap count+skip/limit per tier so pagination still avoids pulling every match into memory. Also found while testing against a 1.19M-channel playlist: searching scans an unindexed regex and can take several seconds, and it was running synchronously inside the async handler, freezing every other request/socket for that whole time. Moved it to run_in_executor; verified a concurrent request now returns in ~2ms while a 3.6s search is in flight. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
685bbb1683
commit
4e596114c1
@@ -892,31 +892,44 @@ async def get_source(request):
|
|||||||
|
|
||||||
|
|
||||||
# List Channels
|
# List Channels
|
||||||
async def list_channel(request):
|
def _search_channels_sync(channel_to_search, current_page):
|
||||||
MAX_RECORDS = 30
|
MAX_RECORDS = 30
|
||||||
data = await request.post()
|
|
||||||
|
|
||||||
channel_to_search = unquote(data['keyword'])
|
escaped_keyword = re.escape(channel_to_search)
|
||||||
current_page = int(data['num_page'])
|
# Channel names are typically prefixed with a country/format tag (e.g.
|
||||||
|
# "AR - BEIN SPORTS 1 FHD"), so match at word boundaries rather than the
|
||||||
|
# start of the string to rank those above unrelated names that merely
|
||||||
|
# contain the keyword mid-word.
|
||||||
|
prefix_pattern = re.compile(r'\b' + escaped_keyword, re.IGNORECASE)
|
||||||
|
contains_pattern = re.compile(escaped_keyword, re.IGNORECASE)
|
||||||
|
|
||||||
logger.info("Search keyword='%s' page=%s", channel_to_search, current_page)
|
|
||||||
regex_pattern = ".*" + re.escape(channel_to_search) + ".*"
|
|
||||||
|
|
||||||
# First pass: compute totals without pulling all docs into memory
|
|
||||||
total_records = 0
|
|
||||||
active_playlists = list(playlists.find({"activation": 1}))
|
active_playlists = list(playlists.find({"activation": 1}))
|
||||||
per_playlist_counts = []
|
|
||||||
|
# Two relevance tiers, most relevant first: names starting with the keyword,
|
||||||
|
# then everything else that merely contains it. Counted per tier/playlist so
|
||||||
|
# pagination stays a cheap count+skip/limit instead of pulling every match
|
||||||
|
# into memory to sort it.
|
||||||
|
tiers = []
|
||||||
for playlist in active_playlists:
|
for playlist in active_playlists:
|
||||||
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||||
count = collection_channels.count_documents({
|
prefix_filter = {'id_playlist': playlist['id_playlist'], 'Name': prefix_pattern}
|
||||||
|
count = collection_channels.count_documents(prefix_filter)
|
||||||
|
if count:
|
||||||
|
tiers.append((playlist, prefix_filter, count))
|
||||||
|
for playlist in active_playlists:
|
||||||
|
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||||
|
contains_only_filter = {
|
||||||
'id_playlist': playlist['id_playlist'],
|
'id_playlist': playlist['id_playlist'],
|
||||||
'Name': {'$regex': regex_pattern, '$options': "i"}
|
'$and': [{'Name': contains_pattern}, {'Name': {'$not': prefix_pattern}}],
|
||||||
})
|
}
|
||||||
per_playlist_counts.append((playlist, count))
|
count = collection_channels.count_documents(contains_only_filter)
|
||||||
total_records += count
|
if count:
|
||||||
|
tiers.append((playlist, contains_only_filter, count))
|
||||||
|
|
||||||
|
total_records = sum(count for _, _, count in tiers)
|
||||||
|
|
||||||
if total_records == 0:
|
if total_records == 0:
|
||||||
return web.json_response({'num_total_page': 0, 'current_page': 1, 'data': []})
|
return {'num_total_page': 0, 'current_page': 1, 'data': []}
|
||||||
|
|
||||||
num_total_page = max(1, (total_records + MAX_RECORDS - 1) // MAX_RECORDS)
|
num_total_page = max(1, (total_records + MAX_RECORDS - 1) // MAX_RECORDS)
|
||||||
current_page = max(1, min(current_page, num_total_page))
|
current_page = max(1, min(current_page, num_total_page))
|
||||||
@@ -926,7 +939,7 @@ async def list_channel(request):
|
|||||||
# Second pass: fetch only the page slice
|
# Second pass: fetch only the page slice
|
||||||
serialized_data = []
|
serialized_data = []
|
||||||
offset = 0
|
offset = 0
|
||||||
for playlist, count in per_playlist_counts:
|
for playlist, mongo_filter, count in tiers:
|
||||||
if remaining == 0:
|
if remaining == 0:
|
||||||
break
|
break
|
||||||
if offset + count <= begin:
|
if offset + count <= begin:
|
||||||
@@ -936,16 +949,30 @@ async def list_channel(request):
|
|||||||
local_skip = max(0, begin - offset)
|
local_skip = max(0, begin - offset)
|
||||||
fetch_limit = min(remaining, count - local_skip)
|
fetch_limit = min(remaining, count - local_skip)
|
||||||
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
collection_channels = db_iptv['channels_' + playlist['id_playlist']]
|
||||||
cursor = collection_channels.find({
|
cursor = collection_channels.find(mongo_filter).skip(local_skip).limit(fetch_limit)
|
||||||
'id_playlist': playlist['id_playlist'],
|
|
||||||
'Name': {'$regex': regex_pattern, '$options': "i"}
|
|
||||||
}).skip(local_skip).limit(fetch_limit)
|
|
||||||
|
|
||||||
serialized_data.extend(serialize_doc(doc) for doc in cursor)
|
serialized_data.extend(serialize_doc(doc) for doc in cursor)
|
||||||
remaining = MAX_RECORDS - len(serialized_data)
|
remaining = MAX_RECORDS - len(serialized_data)
|
||||||
offset += count
|
offset += count
|
||||||
|
|
||||||
return web.json_response({'num_total_page': num_total_page, 'current_page': current_page, 'data': serialized_data})
|
return {'num_total_page': num_total_page, 'current_page': current_page, 'data': serialized_data}
|
||||||
|
|
||||||
|
|
||||||
|
async def list_channel(request):
|
||||||
|
data = await request.post()
|
||||||
|
|
||||||
|
channel_to_search = unquote(data['keyword'])
|
||||||
|
current_page = int(data['num_page'])
|
||||||
|
|
||||||
|
logger.info("Search keyword='%s' page=%s", channel_to_search, current_page)
|
||||||
|
|
||||||
|
# Searching can mean scanning a multi-hundred-thousand-channel collection
|
||||||
|
# with an unindexed regex (several seconds); run it off the event loop so
|
||||||
|
# it doesn't stall every other request/socket while it works.
|
||||||
|
result = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None, _search_channels_sync, channel_to_search, current_page
|
||||||
|
)
|
||||||
|
return web.json_response(result)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user