add saliency options, add image downsampling

app.py

@@ -50,7 +50,7 @@ if __name__ == "__main__":
                             file_types=["image"],
                             file_count="multiple"
                         )
-                uploaded_files = gr.Gallery(label="Input images", visible=False, columns=7, rows=1, height=200)
+                uploaded_files = gr.Gallery(label="Input images", visible=False, columns=7, rows=1, height=250)
 
 
                 with gr.Column(visible=False) as clear_button:
@@ -61,27 +61,36 @@ if __name__ == "__main__":
                 remove_and_reupload.click(fn=remove_back_to_files, outputs=[uploaded_files, clear_button, image_input])
 
                 gr.Markdown("### Select the parameters for recoloring")
-                with gr.Row():
+                with gr.Group():  
                     with gr.Group(): 
                         gr.Markdown("### Recoloring without other techiques")
-                        num_center_grp = gr.Dropdown(choices=[1, 2, 3, 4, 5], value=3, label="Number of group palettes")
-                with gr.Group():         
-                    gr.Markdown("### Recoloring with other techiques")
-                    with gr.Row():
+                        num_center_grp = gr.Dropdown(choices=[1, 2, 3, 4, 5, 6, 7], value=3, label="Number of group palettes")
+                        # num_center_grp = gr.Slider(label="Number of group palettes", minimum=1, maximum=7, value=3, step=1)
+                    with gr.Group():         
+                        gr.Markdown("### Recoloring with other techiques")
+                        gr.Markdown("")
                         with gr.Group(): 
-                            # gr.Markdown("### white balance")
+                            gr.Markdown("<center>white balance<center>")
                             checkbox_input_wb = gr.Checkbox(value=False, label="Apply white balance correction")
                         with gr.Group(): 
-                            # gr.Markdown("### saliency detection")
-                            checkbox_input_sal = gr.Checkbox(value=False, label="Apply saliency")
-                            checkbox_input_recolor_sal = gr.Checkbox(value=False, label="Recolor salient part only")
-                            checkbox_input_recolor_nonsal = gr.Checkbox(value=False, label="Recolor non-salient part only")
-                            num_center_sal = gr.Dropdown(choices=[1, 2, 3], value=1, label="Number of salient palettes")
-                            num_center_nonsal = gr.Dropdown(choices=[1, 2, 3], value=1, label="Number of non-salient palettes")
+                            gr.Markdown("<center>saliency detection<center>")
+                            with gr.Row():
+                                checkbox_input_sal = gr.Checkbox(value=False, label="Apply saliency")
+                                sal_method = gr.Dropdown(choices=['LDF (accurate)','ft (fast)','rbd (fast)'], value='LDF', label="saliency detection method")
+                            with gr.Row():
+                                num_center_sal = gr.Dropdown(choices=[1, 2, 3, 4], value=1, label="Number of salient palettes")
+                                num_center_nonsal = gr.Dropdown(choices=[1, 2, 3, 4], value=1, label="Number of non-salient palettes")
+                            # num_center_sal = gr.Slider(label="Number of salient palettes", minimum=1, maximum=4, value=1, step=1)
+                            # num_center_nonsal = gr.Slider(label="Number of non-salient palettes", minimum=1, maximum=4, value=1, step=1)
+                            with gr.Row():
+                                checkbox_input_recolor_sal = gr.Checkbox(value=False, label="Recolor salient colors only")
+                                checkbox_input_recolor_nonsal = gr.Checkbox(value=False, label="Recolor non-salient colors only")
+
                         with gr.Group(): 
-                            # gr.Markdown("### color naming")
-                            checkbox_input_cn = gr.Checkbox(value=False, label="Apply color naming")
-                            naming_thres = gr.Textbox(value=0.8, label="Threshold of color naming", placeholder=0.8)
+                            gr.Markdown("<center>color naming<center>")
+                            with gr.Row():
+                                checkbox_input_cn = gr.Checkbox(value=False, label="Apply color naming")
+                                naming_thres = gr.Textbox(value=0.8, label="Threshold of color naming", placeholder=0.8)
                 
             with gr.Column():
                 gr.Markdown("### Outputs") 
@@ -110,7 +119,7 @@ if __name__ == "__main__":
                 recolor_group_images, 
                 inputs=[image_input, num_center_grp, num_center_sal, num_center_nonsal,
                         checkbox_input_wb, 
-                        checkbox_input_sal, checkbox_input_recolor_sal, checkbox_input_recolor_nonsal,
+                        checkbox_input_sal, sal_method, checkbox_input_recolor_sal, checkbox_input_recolor_nonsal,
                         checkbox_input_cn, naming_thres],
                 outputs=[output_gallery_recolor, output_gallery_palette_in, output_gallery_palette_out, output_gallery_palette_group]
             )

image.py

@@ -1,5 +1,6 @@
 # import cv2
 import os
+import cv2
 import numpy as np
 
 from PIL import Image
@@ -16,6 +17,18 @@ class BaseImage:
         self.filename = os.path.basename(filepath.name)
         self.image = Image.open(filepath)
         self.img_rgb = np.asarray(self.image).astype(dtype=np.uint8)
+
+        anchor = 256
+        width = self.img_rgb.shape[1]
+        height = self.img_rgb.shape[0]
+        if width > 512 or height > 512:
+            self.if_downsample = True
+            if width >= height:
+                dim = (np.floor(width/height*anchor).astype(int), anchor)
+            else:
+                dim = (anchor, np.floor(height/width*anchor).astype(int))
+            self.img_rgb = cv2.resize(self.img_rgb, dim, interpolation=cv2.INTER_LINEAR)
+
         self.img_lab = rgb2lab(self.img_rgb)
 
         self.bin_size = 16
@@ -28,36 +41,23 @@ class BaseImage:
         # self.if_correct_wb = if_correct_wb
         # self.if_saliency = if_saliency
         # self.saliency_threshold = sal_thres
-        self.cdiff_threshold = 30
-        self.sal_threshold = 0.9
+        # self.cdiff_threshold = 30
+        # self.sal_threshold = 0.9
         self.applied_wb = False
         # self.valid_class = [0,1]
 
-        
-        # self.hist_value, self.hist_count, \
-        # self.c_center, self.c_density, \
-        # self.c_img_label = self.extract_palette(if_wb=self.if_correct_wb, 
-        #                                         if_saliency=self.if_saliency, 
-        #                                         sal_thres=self.saliency_threshold)
-        
-        # self.inital_info(self.if_correct_wb, 
-        #                  self.if_saliency, 
-        #                  self.saliency_threshold)
-        
-        
-        # self.hist_value, self.hist_count, self.c_center, self.c_density, self.c_img_label = self.extract_palette(if_wb=self.if_correct_wb, if_saliency=False)
-        # self.hist_value_sal, self.hist_count_sal, self.c_center_sal, self.c_density_sal, self.c_img_label_sal = self.extract_palette(if_wb=self.if_correct_wb, if_saliency=True, sal_thres=self.saliency_threshold)
 
-    def inital_info(self, if_correct_wb, if_saliency, wb_thres, sal_thres, valid_class):
+    def inital_info(self, if_correct_wb, if_saliency, wb_thres, sal_method, sal_thres, valid_class):
         self.hist_value, self.hist_count, \
         self.c_center, self.c_density, \
         self.c_img_label, self.sal_links = self.extract_salient_palette(if_wb=if_correct_wb, 
                                                         if_saliency=if_saliency,
                                                         wb_thres=wb_thres, 
+                                                        sal_method=sal_method,
                                                         sal_thres=sal_thres, 
                                                         valid_class=valid_class)
         
-        self.label_colored = self.cal_color_segment()
+        # self.label_colored = self.cal_color_segment()
 
     def get_rgb_image(self):
         return self.img_rgb
@@ -131,6 +131,7 @@ class BaseImage:
             sal_map = get_saliency_ft(img_rgb)
         elif method == 'rbd':
             sal_map = get_saliency_mbd(img_rgb)
+        
         return sal_map
 
 
@@ -159,7 +160,7 @@ class BaseImage:
 
 
 
-    def extract_salient_palette(self, if_wb=False, if_saliency=False, wb_thres=5, sal_thres=0.9, valid_class=[0,1]):
+    def extract_salient_palette(self, if_wb=False, if_saliency=False, wb_thres=5, sal_method='LDF', sal_thres=0.9, valid_class=[0,1]):
 
         img_rgb = self.img_rgb.copy()
         if if_wb:
@@ -182,8 +183,8 @@ class BaseImage:
             return hist_value, hist_count, center, density, colorlabel, sal_links
         
         else:
-            self.sal_map = self.saliency_detection(self.img_rgb)
-            label_sem = np.zeros_like(self.img_rgb[:,:,0])
+            self.sal_map = self.saliency_detection(img_rgb, method=sal_method)
+            label_sem = np.zeros_like(img_rgb[:,:,0])
             # print(label_sem.shape, self.sal_map.shape)
             label_sem[self.sal_map > sal_thres]=1
             

multi_image_process.py

@@ -92,6 +92,7 @@ def solve_grp_palette(images, mode, bin_size,
         image.inital_info(if_wb, 
                           False,
                           wb_thres,
+                          'LDF',
                           0.9,
                           [0, 1])
         
@@ -144,7 +145,7 @@ def solve_grp_palette_wsal(images, mode, bin_size,
                             lightness=70., eta=1e10, gamma=0, iteration=10,
                             if_wb=False, wb_thres=30, 
                             if_saliency=True, sal_thres=0.9, valid_class=[0,1], 
-                            sal_center=1, nonsal_center=1, 
+                            sal_center=1, nonsal_center=1, sal_method='LDF',
                             recolor_nonsal_only=False, recolor_sal_only=False,
                             if_cn=False, naming_thres=0.5):
     
@@ -175,6 +176,7 @@ def solve_grp_palette_wsal(images, mode, bin_size,
         image.inital_info(if_wb,
                           if_saliency,
                           wb_thres,
+                          sal_method,
                           sal_thres,
                           valid_class)
         
@@ -288,7 +290,7 @@ def solve_grp_palette_wsal(images, mode, bin_size,
 def recolor_group_images(inp_images, 
                          num_center_grp, num_center_sal, num_center_nonsal,
                          if_wb, 
-                         if_sal, recolor_nonsal_only, recolor_sal_only,
+                         if_sal, sal_method, recolor_nonsal_only, recolor_sal_only,
                          if_cn, naming_thres, 
                          save_dir='./results/testing'):
 
@@ -309,10 +311,18 @@ def recolor_group_images(inp_images,
         images[img_id] = BaseImage(image)
 
     if if_sal:
+        if sal_method == 'LDF (accurate)':
+            sal_method = 'LDF'
+        elif sal_method == 'ft (fast)':
+            sal_method = 'ft'
+        elif sal_method == 'rbd (fast)':
+            sal_method = 'rbd'
+        else:
+            raise ValueError('saliency method not supported!')
         palette_src, palette_grp, palette_tgt, links, src_sal_size, grp_sal_size = solve_grp_palette_wsal(images, mode, bin_size,
                                                                                     lightness=70., eta=1e10, gamma=0, iteration=10,
                                                                                     if_wb=if_wb, wb_thres=30, 
-                                                                                    if_saliency=True, sal_thres=0.5, valid_class=[0,1], 
+                                                                                    if_saliency=True, sal_thres=0.5, valid_class=[0,1], sal_method=sal_method,
                                                                                     sal_center=num_center_sal, nonsal_center=num_center_nonsal, 
                                                                                     recolor_nonsal_only=recolor_nonsal_only, recolor_sal_only=recolor_sal_only,
                                                                                     if_cn=if_cn, naming_thres=float(naming_thres))