Update app.py

app.py

@@ -37,6 +37,11 @@ if uploaded_file is not None:
         "unknown_params": []
     }
 
+    # قوائم لتحليل الصيغ وأنواع الروابط
+    extensions = {}
+    internal_links = []
+    external_links = []
+
     # متغيرات للتحكم في العملية
     progress_bar = st.progress(0)
     pause_button = st.button("Pause")
@@ -44,36 +49,42 @@ if uploaded_file is not None:
     continue_button = st.button("Continue")
     paused = False
     stopped = False
-    current_index = 0
+    current_index = 0  # مؤشر للكلمة الحالية
+    batch_size = 10  # عدد العناصر التي يتم معالجتها في الدفعة الواحدة
 
-    # دالة تصنيف العناصر (للكلمات المفتاحية)
-    def classify_keywords(items, categories, start_index=0):
+    # دالة تصنيف الكلمات باستخدام الدفعات
+    def classify_keywords_batch(items, categories, start_index=0):
         global paused, stopped, current_index
         total_items = len(items)
-        for i, item in enumerate(items[start_index:], start=start_index):
-            current_index = i
+        for i in range(start_index, total_items, batch_size):
             if stopped:
                 break
             if paused:
                 time.sleep(0.5)
                 continue
 
-            # تصنيف الكلمة باستخدام zero-shot-classification
-            result = classifier(item, categories)
-            best_category = result['labels'][0]
-            score = result['scores'][0]
+            # معالجة دفعة من العناصر
+            batch = items[i:i + batch_size]
+            results = classifier(batch, categories)
+
+            for j, result in enumerate(results):
+                best_category = result['labels'][0]
+                score = result['scores'][0]
+
+                if best_category == "shop" and score > 0.5:
+                    shopping_items.append(batch[j])
+                elif best_category == "game" and score > 0.5:
+                    gaming_items.append(batch[j])
+                elif best_category == "stream" and score > 0.5:
+                    streaming_items.append(batch[j])
+                else:
+                    unknown_items.append(batch[j])
 
-            if best_category == "shop" and score > 0.5:
-                shopping_items.append(item)
-            elif best_category == "game" and score > 0.5:
-                gaming_items.append(item)
-            elif best_category == "stream" and score > 0.5:
-                streaming_items.append(item)
-            else:
-                unknown_items.append(item)
+            # تحديث المؤشر الحالي
+            current_index = i + batch_size
 
             # تحديث شريط التقدم
-            progress = (current_index + 1) / total_items
+            progress = (current_index) / total_items
             progress_bar.progress(progress)
 
             # تحديث النتائج في الوقت الحقيقي
@@ -86,33 +97,52 @@ if uploaded_file is not None:
     def classify_parameters(items, categories, start_index=0):
         global paused, stopped, current_index
         total_items = len(items)
-        for i, url in enumerate(items[start_index:], start=start_index):
-            current_index = i
+        for i in range(start_index, total_items, batch_size):
             if stopped:
                 break
             if paused:
                 time.sleep(0.5)
                 continue
 
-            # استخراج الباراميترات من الرابط باستخدام RegEx
-            params = re.findall(r'(\w+)=\w+', url)
-            for param in params:
-                # تصنيف الباراميتر باستخدام zero-shot-classification
-                result = classifier(param, categories)
-                best_category = result['labels'][0]
-                score = result['scores'][0]
-
-                if best_category == "shop" and score > 0.5:
-                    param_categories["shop_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
-                elif best_category == "game" and score > 0.5:
-                    param_categories["game_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
-                elif best_category == "stream" and score > 0.5:
-                    param_categories["stream_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+            # معالجة دفعة من الروابط
+            batch = items[i:i + batch_size]
+            for url in batch:
+                # استخراج الباراميترات من الرابط باستخدام RegEx
+                params = re.findall(r'(\w+)=\w+', url)
+                for param in params:
+                    # تصنيف الباراميتر باستخدام zero-shot-classification
+                    result = classifier(param, categories)
+                    best_category = result['labels'][0]
+                    score = result['scores'][0]
+
+                    if best_category == "shop" and score > 0.5:
+                        param_categories["shop_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+                    elif best_category == "game" and score > 0.5:
+                        param_categories["game_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+                    elif best_category == "stream" and score > 0.5:
+                        param_categories["stream_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+                    else:
+                        param_categories["unknown_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+
+                # تصنيف الرابط بناءً على نوعه (داخلي/خارجي)
+                if re.match(r'^https?://[^/]+\.example\.com', url):
+                    internal_links.append(url)
                 else:
-                    param_categories["unknown_params"].append(f"{param}={re.search(param + r'=([^&]*)', url).group(1)}")
+                    external_links.append(url)
+
+                # استخراج الصيغ (Extensions) من الروابط
+                match = re.search(r'\.([a-zA-Z0-9]+)$', url)
+                if match:
+                    ext = match.group(1)
+                    if ext not in extensions:
+                        extensions[ext] = []
+                    extensions[ext].append(url)
+
+            # تحديث المؤشر الحالي
+            current_index = i + batch_size
 
             # تحديث شريط التقدم
-            progress = (current_index + 1) / total_items
+            progress = (current_index) / total_items
             progress_bar.progress(progress)
 
             # تحديث النتائج في الوقت الحقيقي
@@ -135,6 +165,13 @@ if uploaded_file is not None:
         st.session_state.stream_params = "\n".join(param_categories["stream_params"])
         st.session_state.unknown_params = "\n".join(param_categories["unknown_params"])
 
+        # تحديث محتوى المربعات الخاصة بأنواع الروابط
+        st.session_state.internal_links = "\n".join(internal_links)
+        st.session_state.external_links = "\n".join(external_links)
+
+        # تحديث محتوى المربعات الخاصة بالصيغ
+        st.session_state.extensions_text = "\n\n".join([f"{k}: {', '.join(v)}" for k, v in extensions.items()])
+
     # زر البدء
     if st.button("Start"):
         stopped = False
@@ -142,7 +179,7 @@ if uploaded_file is not None:
         current_index = 0
 
         if operation == "Filter Keywords":
-            classify_keywords(items, categories, start_index=current_index)
+            classify_keywords_batch(items, categories, start_index=current_index)
         elif operation == "Extra & Filter Param (URLs)":
             classify_parameters(items, categories, start_index=current_index)
 
@@ -156,7 +193,7 @@ if uploaded_file is not None:
         paused = False
         st.write("Classification resumed.")
         if operation == "Filter Keywords":
-            classify_keywords(items, categories, start_index=current_index)
+            classify_keywords_batch(items, categories, start_index=current_index)
         elif operation == "Extra & Filter Param (URLs)":
             classify_parameters(items, categories, start_index=current_index)
 
@@ -210,5 +247,22 @@ if uploaded_file is not None:
             st.session_state.unknown_params = ""
         st.text_area("Copy the unknown parameters here:", value=st.session_state.unknown_params, height=200, key="unknown_params")
 
+        # عرض أنواع الروابط
+        st.header("Internal Links")
+        if 'internal_links' not in st.session_state:
+            st.session_state.internal_links = ""
+        st.text_area("Copy the internal links here:", value=st.session_state.internal_links, height=200, key="internal_links")
+
+        st.header("External Links")
+        if 'external_links' not in st.session_state:
+            st.session_state.external_links = ""
+        st.text_area("Copy the external links here:", value=st.session_state.external_links, height=200, key="external_links")
+
+        # عرض الصيغ
+        st.header("File Extensions")
+        if 'extensions_text' not in st.session_state:
+            st.session_state.extensions_text = ""
+        st.text_area("Copy the file extensions here:", value=st.session_state.extensions_text, height=200, key="extensions")
+
 else:
     st.warning("Please upload a text file to start classification.")