Update crawler.py

crawler.py

@@ -93,17 +93,17 @@ def get_answers(results_a_elements, query):
     print("Result links extracted: ", len(results_a_elements))
     
     # Limit the number of parallel processes for better resource management
-    max_processes = max(1, int(multiprocessing.cpu_count() * 0.5))
+    # max_processes = max(1, int(multiprocessing.cpu_count() * 0.5))
     
-    with multiprocessing.Pool(processes=max_processes) as pool:
-        results = pool.map(process_url, results_a_elements)
-
-    # results = []
-    # # answer_count = 0
-    # for url in tqdm.tqdm(results_a_elements):
-    #     res = process_url(url)
-    #     results.append(res)
-        # answer_count += len(res['answers'])
+    # with multiprocessing.Pool(processes=max_processes) as pool:
+    #     results = pool.map(process_url, results_a_elements)
+
+    results = []
+    # answer_count = 0
+    for url in tqdm.tqdm(results_a_elements):
+        res = process_url(url)
+        results.append(res)
+        answer_count += len(res['answers'])
     return results
 
 def get_search_results(query, num_pages):