Stream M3U playlists line by line during analysis
readlines() loaded whole playlists into memory; a 336 MB file needs roughly 500-600 MB of RAM on a VPS with ~800 MB available. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4fa53e7a64
commit
ad82a96440
@@ -561,8 +561,12 @@ async def analyse_m3u_file(request):
|
||||
# Process the m3u file
|
||||
|
||||
new_channels =[]
|
||||
f = open(new_path_filename, "r", encoding="utf8")
|
||||
Lines = f.readlines()
|
||||
# Lecture ligne à ligne : une playlist de plusieurs centaines de Mo chargée d'un
|
||||
# bloc (readlines) ne tient pas dans la mémoire du VPS.
|
||||
with open(new_path_filename, "r", encoding="utf8", errors="replace") as count_file:
|
||||
total_lines = max(1, sum(1 for _ in count_file))
|
||||
f = open(new_path_filename, "r", encoding="utf8", errors="replace")
|
||||
Lines = f
|
||||
print('ouverture fichier', flush=True)
|
||||
channel = {}
|
||||
separator = '--------'
|
||||
@@ -570,7 +574,6 @@ async def analyse_m3u_file(request):
|
||||
num_channel = 0
|
||||
TypeLangChannel = ['EN','FR','US','CA', 'UK' ]
|
||||
|
||||
total_lines = len(Lines)
|
||||
number_line = 0
|
||||
|
||||
for line in Lines:
|
||||
@@ -622,7 +625,8 @@ async def analyse_m3u_file(request):
|
||||
channel_exist = True
|
||||
num_channel += 1
|
||||
|
||||
|
||||
|
||||
f.close()
|
||||
bulk_insert_channels(collection_channels, new_channels, num_channel)
|
||||
print("Save final channels ", num_channel, flush=True)
|
||||
number_channels = collection_channels.count_documents({"id_playlist":id_playlist})
|
||||
|
||||
Reference in New Issue
Block a user