Spaces:
Running
on
Zero
Running
on
Zero
update parameters
Browse files
app.py
CHANGED
|
@@ -75,8 +75,10 @@ def compute_ncut(
|
|
| 75 |
perplexity=150,
|
| 76 |
n_neighbors=150,
|
| 77 |
min_dist=0.1,
|
| 78 |
-
sampling_method="
|
| 79 |
metric="cosine",
|
|
|
|
|
|
|
| 80 |
progess_start=0.4,
|
| 81 |
):
|
| 82 |
progress = gr.Progress()
|
|
@@ -100,6 +102,8 @@ def compute_ncut(
|
|
| 100 |
sample_method=sampling_method,
|
| 101 |
distance=metric,
|
| 102 |
normalize_features=False,
|
|
|
|
|
|
|
| 103 |
).fit_transform(features.reshape(-1, features.shape[-1]))
|
| 104 |
# print(f"NCUT time: {time.time() - start:.2f}s")
|
| 105 |
logging_str += f"NCUT time: {time.time() - start:.2f}s\n"
|
|
@@ -384,7 +388,10 @@ def ncut_run(
|
|
| 384 |
perplexity=500,
|
| 385 |
n_neighbors=500,
|
| 386 |
min_dist=0.1,
|
| 387 |
-
sampling_method="
|
|
|
|
|
|
|
|
|
|
| 388 |
old_school_ncut=False,
|
| 389 |
recursion=False,
|
| 390 |
recursion_l2_n_eigs=50,
|
|
@@ -475,7 +482,9 @@ def ncut_run(
|
|
| 475 |
n_neighbors=n_neighbors,
|
| 476 |
min_dist=min_dist,
|
| 477 |
sampling_method=sampling_method,
|
| 478 |
-
metric=
|
|
|
|
|
|
|
| 479 |
progess_start=progress_start,
|
| 480 |
)
|
| 481 |
logging_str += _logging_str
|
|
@@ -522,6 +531,9 @@ def ncut_run(
|
|
| 522 |
n_neighbors=n_neighbors,
|
| 523 |
min_dist=min_dist,
|
| 524 |
sampling_method=sampling_method,
|
|
|
|
|
|
|
|
|
|
| 525 |
progess_start=progress_start+step_float*i_image,
|
| 526 |
)
|
| 527 |
logging_str += _logging_str
|
|
@@ -544,6 +556,9 @@ def ncut_run(
|
|
| 544 |
n_neighbors=n_neighbors,
|
| 545 |
min_dist=min_dist,
|
| 546 |
sampling_method=sampling_method,
|
|
|
|
|
|
|
|
|
|
| 547 |
)
|
| 548 |
logging_str += _logging_str
|
| 549 |
|
|
@@ -792,6 +807,8 @@ def run_fn(
|
|
| 792 |
affinity_focal_gamma=0.5,
|
| 793 |
num_sample_ncut=10000,
|
| 794 |
knn_ncut=10,
|
|
|
|
|
|
|
| 795 |
embedding_method="tsne_3d",
|
| 796 |
embedding_metric='euclidean',
|
| 797 |
num_sample_tsne=300,
|
|
@@ -799,7 +816,8 @@ def run_fn(
|
|
| 799 |
perplexity=150,
|
| 800 |
n_neighbors=150,
|
| 801 |
min_dist=0.1,
|
| 802 |
-
sampling_method="
|
|
|
|
| 803 |
old_school_ncut=False,
|
| 804 |
max_frames=100,
|
| 805 |
recursion=False,
|
|
@@ -826,7 +844,7 @@ def run_fn(
|
|
| 826 |
images = extract_video_frames(images, max_frames=max_frames)
|
| 827 |
video_output = True
|
| 828 |
|
| 829 |
-
if sampling_method == "
|
| 830 |
sampling_method = "farthest"
|
| 831 |
|
| 832 |
# resize the images before acquiring GPU
|
|
@@ -923,6 +941,9 @@ def run_fn(
|
|
| 923 |
"n_neighbors": n_neighbors,
|
| 924 |
"min_dist": min_dist,
|
| 925 |
"sampling_method": sampling_method,
|
|
|
|
|
|
|
|
|
|
| 926 |
"old_school_ncut": old_school_ncut,
|
| 927 |
"recursion": recursion,
|
| 928 |
"recursion_l2_n_eigs": recursion_l2_n_eigs,
|
|
@@ -1001,7 +1022,8 @@ def make_dataset_images_section(advanced=False, is_random=False):
|
|
| 1001 |
dataset_classes = DATASET_CLASSES
|
| 1002 |
with gr.Row():
|
| 1003 |
dataset_dropdown = gr.Dropdown(dataset_names, label="Dataset name", value="mrm8488/ImageNet1K-val", elem_id="dataset", min_width=300)
|
| 1004 |
-
num_images_slider = gr.Number(10, label="Number of images", elem_id="num_images")
|
|
|
|
| 1005 |
if not is_random:
|
| 1006 |
filter_by_class_checkbox = gr.Checkbox(label="Filter by class", value=True, elem_id="filter_by_class_checkbox")
|
| 1007 |
filter_by_class_text = gr.Textbox(label="Class to select", value="0,33,99", elem_id="filter_by_class_text", info=f"e.g. `0,1,2`. (1000 classes)", visible=True)
|
|
@@ -1241,24 +1263,32 @@ def make_parameters_section(is_lisa=False, model_ratio=True):
|
|
| 1241 |
gr.Textbox(label="Prompt (Negative)", elem_id="prompt", placeholder="e.g. 'a photo from egocentric view'", visible=False))
|
| 1242 |
model_dropdown.change(fn=change_prompt_text, inputs=model_dropdown, outputs=[positive_prompt, negative_prompt])
|
| 1243 |
|
| 1244 |
-
with gr.Accordion("
|
| 1245 |
gr.Markdown("<a href='https://ncut-pytorch.readthedocs.io/en/latest/how_to_get_better_segmentation/' target='_blank'>Docs: How to Get Better Segmentation</a>")
|
| 1246 |
affinity_focal_gamma_slider = gr.Slider(0.01, 1, step=0.01, label="NCUT: Affinity focal gamma", value=0.5, elem_id="affinity_focal_gamma", info="decrease for shaper segmentation")
|
| 1247 |
num_sample_ncut_slider = gr.Slider(100, 50000, step=100, label="NCUT: num_sample", value=10000, elem_id="num_sample_ncut", info="Nyström approximation")
|
| 1248 |
-
sampling_method_dropdown = gr.Dropdown(["
|
| 1249 |
-
|
| 1250 |
-
|
| 1251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1252 |
num_sample_tsne_slider = gr.Slider(100, 10000, step=100, label="t-SNE/UMAP: num_sample", value=300, elem_id="num_sample_tsne", info="Nyström approximation")
|
| 1253 |
knn_tsne_slider = gr.Slider(1, 100, step=1, label="t-SNE/UMAP: KNN", value=10, elem_id="knn_tsne", info="Nyström approximation")
|
| 1254 |
perplexity_slider = gr.Slider(10, 1000, step=10, label="t-SNE: perplexity", value=150, elem_id="perplexity")
|
| 1255 |
n_neighbors_slider = gr.Slider(10, 1000, step=10, label="UMAP: n_neighbors", value=150, elem_id="n_neighbors")
|
| 1256 |
min_dist_slider = gr.Slider(0.1, 1, step=0.1, label="UMAP: min_dist", value=0.1, elem_id="min_dist")
|
| 1257 |
return [model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1258 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1259 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1260 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1261 |
-
sampling_method_dropdown, positive_prompt, negative_prompt]
|
| 1262 |
|
| 1263 |
demo = gr.Blocks(
|
| 1264 |
theme=gr.themes.Base(spacing_size='md', text_size='lg', primary_hue='blue', neutral_hue='slate', secondary_hue='pink'),
|
|
@@ -1280,10 +1310,10 @@ with demo:
|
|
| 1280 |
cluster_gallery = gr.Gallery(value=[], label="Clusters", show_label=True, elem_id="clusters", columns=[5], rows=[2], object_fit="contain", height="auto", show_share_button=True, preview=True, interactive=False)
|
| 1281 |
[
|
| 1282 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1283 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1284 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1285 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1286 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1287 |
] = make_parameters_section()
|
| 1288 |
num_eig_slider.value = 30
|
| 1289 |
|
|
@@ -1298,9 +1328,9 @@ with demo:
|
|
| 1298 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1299 |
positive_prompt, negative_prompt,
|
| 1300 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1301 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1302 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1303 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
|
| 1304 |
],
|
| 1305 |
outputs=[output_gallery, cluster_gallery, logging_text],
|
| 1306 |
api_name="API_AlignedCut",
|
|
@@ -1341,14 +1371,14 @@ with demo:
|
|
| 1341 |
output_gallery = make_output_images_section()
|
| 1342 |
[
|
| 1343 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1344 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1345 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1346 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1347 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1348 |
] = make_parameters_section()
|
| 1349 |
old_school_ncut_checkbox = gr.Checkbox(label="Old school NCut", value=True, elem_id="old_school_ncut")
|
| 1350 |
-
invisible_list = [old_school_ncut_checkbox, num_sample_ncut_slider,
|
| 1351 |
-
num_sample_tsne_slider, knn_tsne_slider, sampling_method_dropdown]
|
| 1352 |
for item in invisible_list:
|
| 1353 |
item.visible = False
|
| 1354 |
# logging text box
|
|
@@ -1364,9 +1394,9 @@ with demo:
|
|
| 1364 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1365 |
positive_prompt, negative_prompt,
|
| 1366 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1367 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1368 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1369 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown,
|
| 1370 |
old_school_ncut_checkbox
|
| 1371 |
],
|
| 1372 |
outputs=[output_gallery, logging_text],
|
|
@@ -1411,10 +1441,10 @@ with demo:
|
|
| 1411 |
l3_affinity_focal_gamma_slider = gr.Slider(0.01, 1, step=0.01, label="Recursion #3: Affinity focal gamma", value=0.5, elem_id="recursion_l3_gamma")
|
| 1412 |
[
|
| 1413 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1414 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1415 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1416 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1417 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1418 |
] = make_parameters_section()
|
| 1419 |
num_eig_slider.visible = False
|
| 1420 |
affinity_focal_gamma_slider.visible = False
|
|
@@ -1433,9 +1463,9 @@ with demo:
|
|
| 1433 |
input_gallery, model_dropdown, layer_slider, l1_num_eig_slider, node_type_dropdown,
|
| 1434 |
positive_prompt, negative_prompt,
|
| 1435 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1436 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1437 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1438 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown,
|
| 1439 |
false_placeholder, number_placeholder, true_placeholder,
|
| 1440 |
l2_num_eig_slider, l3_num_eig_slider, metric_dropdown,
|
| 1441 |
l1_affinity_focal_gamma_slider, l2_affinity_focal_gamma_slider, l3_affinity_focal_gamma_slider
|
|
@@ -1453,10 +1483,10 @@ with demo:
|
|
| 1453 |
video_output_gallery = gr.Video(value=None, label="NCUT Embedding", elem_id="ncut", height="auto", show_share_button=False)
|
| 1454 |
[
|
| 1455 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1456 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1457 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1458 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1459 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1460 |
] = make_parameters_section()
|
| 1461 |
num_sample_tsne_slider.value = 1000
|
| 1462 |
perplexity_slider.value = 500
|
|
@@ -1476,9 +1506,9 @@ with demo:
|
|
| 1476 |
video_input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1477 |
positive_prompt, negative_prompt,
|
| 1478 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1479 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1480 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1481 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown,
|
| 1482 |
place_holder_false, max_frame_number
|
| 1483 |
],
|
| 1484 |
outputs=[video_output_gallery, logging_text],
|
|
@@ -1522,10 +1552,10 @@ with demo:
|
|
| 1522 |
with gr.Column(scale=5, min_width=200):
|
| 1523 |
[
|
| 1524 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1525 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1526 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1527 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1528 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1529 |
] = make_parameters_section(is_lisa=True)
|
| 1530 |
logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
|
| 1531 |
|
|
@@ -1537,9 +1567,9 @@ with demo:
|
|
| 1537 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1538 |
positive_prompt, negative_prompt,
|
| 1539 |
true_placeholder, prompt1, prompt2, prompt3,
|
| 1540 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1541 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1542 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
|
| 1543 |
],
|
| 1544 |
outputs=galleries + [logging_text],
|
| 1545 |
)
|
|
@@ -1569,10 +1599,10 @@ with demo:
|
|
| 1569 |
gr.Markdown('---')
|
| 1570 |
[
|
| 1571 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1572 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1573 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1574 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1575 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1576 |
] = make_parameters_section(model_ratio=False)
|
| 1577 |
model_dropdown.value = "AlignedThreeModelAttnNodes"
|
| 1578 |
model_dropdown.visible = False
|
|
@@ -1594,9 +1624,9 @@ with demo:
|
|
| 1594 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1595 |
positive_prompt, negative_prompt,
|
| 1596 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1597 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1598 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1599 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
|
| 1600 |
],
|
| 1601 |
# outputs=galleries + [logging_text],
|
| 1602 |
outputs=[output_gallery, logging_text],
|
|
@@ -1655,10 +1685,10 @@ with demo:
|
|
| 1655 |
gr.Markdown('Layer type: attention output (attn), without sum of residual')
|
| 1656 |
[
|
| 1657 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1658 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1659 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1660 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1661 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1662 |
] = make_parameters_section(model_ratio=False)
|
| 1663 |
num_eig_slider.visible = False
|
| 1664 |
affinity_focal_gamma_slider.visible = False
|
|
@@ -1687,9 +1717,9 @@ with demo:
|
|
| 1687 |
input_gallery, model_dropdown, layer_slider, l1_num_eig_slider, node_type_dropdown,
|
| 1688 |
positive_prompt, negative_prompt,
|
| 1689 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1690 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1691 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1692 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown,
|
| 1693 |
false_placeholder, number_placeholder, true_placeholder,
|
| 1694 |
l2_num_eig_slider, l3_num_eig_slider, metric_dropdown,
|
| 1695 |
l1_affinity_focal_gamma_slider, l2_affinity_focal_gamma_slider, l3_affinity_focal_gamma_slider
|
|
@@ -1707,10 +1737,10 @@ with demo:
|
|
| 1707 |
add_output_images_buttons(output_gallery)
|
| 1708 |
[
|
| 1709 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1710 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1711 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1712 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1713 |
-
sampling_method_dropdown, positive_prompt, negative_prompt
|
| 1714 |
] = make_parameters_section()
|
| 1715 |
# logging text box
|
| 1716 |
logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
|
|
@@ -1723,9 +1753,9 @@ with demo:
|
|
| 1723 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1724 |
positive_prompt, negative_prompt,
|
| 1725 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1726 |
-
affinity_focal_gamma_slider, num_sample_ncut_slider,
|
| 1727 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1728 |
-
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
|
| 1729 |
],
|
| 1730 |
outputs=[output_gallery, logging_text]
|
| 1731 |
)
|
|
|
|
| 75 |
perplexity=150,
|
| 76 |
n_neighbors=150,
|
| 77 |
min_dist=0.1,
|
| 78 |
+
sampling_method="QuickFPS",
|
| 79 |
metric="cosine",
|
| 80 |
+
indirect_connection=True,
|
| 81 |
+
make_orthogonal=False,
|
| 82 |
progess_start=0.4,
|
| 83 |
):
|
| 84 |
progress = gr.Progress()
|
|
|
|
| 102 |
sample_method=sampling_method,
|
| 103 |
distance=metric,
|
| 104 |
normalize_features=False,
|
| 105 |
+
indirect_connection=indirect_connection,
|
| 106 |
+
make_orthogonal=make_orthogonal,
|
| 107 |
).fit_transform(features.reshape(-1, features.shape[-1]))
|
| 108 |
# print(f"NCUT time: {time.time() - start:.2f}s")
|
| 109 |
logging_str += f"NCUT time: {time.time() - start:.2f}s\n"
|
|
|
|
| 388 |
perplexity=500,
|
| 389 |
n_neighbors=500,
|
| 390 |
min_dist=0.1,
|
| 391 |
+
sampling_method="QuickFPS",
|
| 392 |
+
ncut_metric="cosine",
|
| 393 |
+
indirect_connection=True,
|
| 394 |
+
make_orthogonal=False,
|
| 395 |
old_school_ncut=False,
|
| 396 |
recursion=False,
|
| 397 |
recursion_l2_n_eigs=50,
|
|
|
|
| 482 |
n_neighbors=n_neighbors,
|
| 483 |
min_dist=min_dist,
|
| 484 |
sampling_method=sampling_method,
|
| 485 |
+
metric=ncut_metric if i == 0 else recursion_metric,
|
| 486 |
+
indirect_connection=indirect_connection,
|
| 487 |
+
make_orthogonal=make_orthogonal,
|
| 488 |
progess_start=progress_start,
|
| 489 |
)
|
| 490 |
logging_str += _logging_str
|
|
|
|
| 531 |
n_neighbors=n_neighbors,
|
| 532 |
min_dist=min_dist,
|
| 533 |
sampling_method=sampling_method,
|
| 534 |
+
metric=ncut_metric,
|
| 535 |
+
indirect_connection=indirect_connection,
|
| 536 |
+
make_orthogonal=make_orthogonal,
|
| 537 |
progess_start=progress_start+step_float*i_image,
|
| 538 |
)
|
| 539 |
logging_str += _logging_str
|
|
|
|
| 556 |
n_neighbors=n_neighbors,
|
| 557 |
min_dist=min_dist,
|
| 558 |
sampling_method=sampling_method,
|
| 559 |
+
indirect_connection=indirect_connection,
|
| 560 |
+
make_orthogonal=make_orthogonal,
|
| 561 |
+
metric=ncut_metric,
|
| 562 |
)
|
| 563 |
logging_str += _logging_str
|
| 564 |
|
|
|
|
| 807 |
affinity_focal_gamma=0.5,
|
| 808 |
num_sample_ncut=10000,
|
| 809 |
knn_ncut=10,
|
| 810 |
+
ncut_indirect_connection=True,
|
| 811 |
+
ncut_make_orthogonal=False,
|
| 812 |
embedding_method="tsne_3d",
|
| 813 |
embedding_metric='euclidean',
|
| 814 |
num_sample_tsne=300,
|
|
|
|
| 816 |
perplexity=150,
|
| 817 |
n_neighbors=150,
|
| 818 |
min_dist=0.1,
|
| 819 |
+
sampling_method="QuickFPS",
|
| 820 |
+
ncut_metric="cosine",
|
| 821 |
old_school_ncut=False,
|
| 822 |
max_frames=100,
|
| 823 |
recursion=False,
|
|
|
|
| 844 |
images = extract_video_frames(images, max_frames=max_frames)
|
| 845 |
video_output = True
|
| 846 |
|
| 847 |
+
if sampling_method == "QuickFPS":
|
| 848 |
sampling_method = "farthest"
|
| 849 |
|
| 850 |
# resize the images before acquiring GPU
|
|
|
|
| 941 |
"n_neighbors": n_neighbors,
|
| 942 |
"min_dist": min_dist,
|
| 943 |
"sampling_method": sampling_method,
|
| 944 |
+
"ncut_metric": ncut_metric,
|
| 945 |
+
"indirect_connection": ncut_indirect_connection,
|
| 946 |
+
"make_orthogonal": ncut_make_orthogonal,
|
| 947 |
"old_school_ncut": old_school_ncut,
|
| 948 |
"recursion": recursion,
|
| 949 |
"recursion_l2_n_eigs": recursion_l2_n_eigs,
|
|
|
|
| 1022 |
dataset_classes = DATASET_CLASSES
|
| 1023 |
with gr.Row():
|
| 1024 |
dataset_dropdown = gr.Dropdown(dataset_names, label="Dataset name", value="mrm8488/ImageNet1K-val", elem_id="dataset", min_width=300)
|
| 1025 |
+
# num_images_slider = gr.Number(10, label="Number of images", elem_id="num_images")
|
| 1026 |
+
num_images_slider = gr.Slider(1, 1000, step=1, label="Number of images", value=10, elem_id="num_images")
|
| 1027 |
if not is_random:
|
| 1028 |
filter_by_class_checkbox = gr.Checkbox(label="Filter by class", value=True, elem_id="filter_by_class_checkbox")
|
| 1029 |
filter_by_class_text = gr.Textbox(label="Class to select", value="0,33,99", elem_id="filter_by_class_text", info=f"e.g. `0,1,2`. (1000 classes)", visible=True)
|
|
|
|
| 1263 |
gr.Textbox(label="Prompt (Negative)", elem_id="prompt", placeholder="e.g. 'a photo from egocentric view'", visible=False))
|
| 1264 |
model_dropdown.change(fn=change_prompt_text, inputs=model_dropdown, outputs=[positive_prompt, negative_prompt])
|
| 1265 |
|
| 1266 |
+
with gr.Accordion("Advanced Parameters: NCUT", open=False):
|
| 1267 |
gr.Markdown("<a href='https://ncut-pytorch.readthedocs.io/en/latest/how_to_get_better_segmentation/' target='_blank'>Docs: How to Get Better Segmentation</a>")
|
| 1268 |
affinity_focal_gamma_slider = gr.Slider(0.01, 1, step=0.01, label="NCUT: Affinity focal gamma", value=0.5, elem_id="affinity_focal_gamma", info="decrease for shaper segmentation")
|
| 1269 |
num_sample_ncut_slider = gr.Slider(100, 50000, step=100, label="NCUT: num_sample", value=10000, elem_id="num_sample_ncut", info="Nyström approximation")
|
| 1270 |
+
# sampling_method_dropdown = gr.Dropdown(["QuickFPS", "random"], label="NCUT: Sampling method", value="QuickFPS", elem_id="sampling_method", info="Nyström approximation")
|
| 1271 |
+
sampling_method_dropdown = gr.Radio(["QuickFPS", "random"], label="NCUT: Sampling method", value="QuickFPS", elem_id="sampling_method")
|
| 1272 |
+
# ncut_metric_dropdown = gr.Dropdown(["euclidean", "cosine"], label="NCUT: Distance metric", value="cosine", elem_id="ncut_metric")
|
| 1273 |
+
ncut_metric_dropdown = gr.Radio(["euclidean", "cosine"], label="NCUT: Distance metric", value="cosine", elem_id="ncut_metric")
|
| 1274 |
+
ncut_knn_slider = gr.Slider(1, 100, step=1, label="NCUT: KNN", value=10, elem_id="knn_ncut", info="Nyström approximation")
|
| 1275 |
+
ncut_indirect_connection = gr.Checkbox(label="indirect_connection", value=True, elem_id="ncut_indirect_connection", info="Add indirect connection to the sub-sampled graph")
|
| 1276 |
+
ncut_make_orthogonal = gr.Checkbox(label="make_orthogonal", value=False, elem_id="ncut_make_orthogonal", info="Apply post-hoc eigenvectors orthogonalization")
|
| 1277 |
+
with gr.Accordion("Advanced Parameters: t-SNE/UMAP", open=False):
|
| 1278 |
+
# embedding_method_dropdown = gr.Dropdown(["tsne_3d", "umap_3d", "umap_shpere", "tsne_2d", "umap_2d"], label="Coloring method", value="tsne_3d", elem_id="embedding_method")
|
| 1279 |
+
embedding_method_dropdown = gr.Radio(["tsne_3d", "umap_3d", "umap_shpere", "tsne_2d", "umap_2d"], label="Coloring method", value="tsne_3d", elem_id="embedding_method")
|
| 1280 |
+
# embedding_metric_dropdown = gr.Dropdown(["euclidean", "cosine"], label="t-SNE/UMAP metric", value="euclidean", elem_id="embedding_metric")
|
| 1281 |
+
embedding_metric_dropdown = gr.Radio(["euclidean", "cosine"], label="t-SNE/UMAP: metric", value="euclidean", elem_id="embedding_metric")
|
| 1282 |
num_sample_tsne_slider = gr.Slider(100, 10000, step=100, label="t-SNE/UMAP: num_sample", value=300, elem_id="num_sample_tsne", info="Nyström approximation")
|
| 1283 |
knn_tsne_slider = gr.Slider(1, 100, step=1, label="t-SNE/UMAP: KNN", value=10, elem_id="knn_tsne", info="Nyström approximation")
|
| 1284 |
perplexity_slider = gr.Slider(10, 1000, step=10, label="t-SNE: perplexity", value=150, elem_id="perplexity")
|
| 1285 |
n_neighbors_slider = gr.Slider(10, 1000, step=10, label="UMAP: n_neighbors", value=150, elem_id="n_neighbors")
|
| 1286 |
min_dist_slider = gr.Slider(0.1, 1, step=0.1, label="UMAP: min_dist", value=0.1, elem_id="min_dist")
|
| 1287 |
return [model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1288 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1289 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1290 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1291 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt]
|
| 1292 |
|
| 1293 |
demo = gr.Blocks(
|
| 1294 |
theme=gr.themes.Base(spacing_size='md', text_size='lg', primary_hue='blue', neutral_hue='slate', secondary_hue='pink'),
|
|
|
|
| 1310 |
cluster_gallery = gr.Gallery(value=[], label="Clusters", show_label=True, elem_id="clusters", columns=[5], rows=[2], object_fit="contain", height="auto", show_share_button=True, preview=True, interactive=False)
|
| 1311 |
[
|
| 1312 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1313 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1314 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1315 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1316 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1317 |
] = make_parameters_section()
|
| 1318 |
num_eig_slider.value = 30
|
| 1319 |
|
|
|
|
| 1328 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1329 |
positive_prompt, negative_prompt,
|
| 1330 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1331 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1332 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1333 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown
|
| 1334 |
],
|
| 1335 |
outputs=[output_gallery, cluster_gallery, logging_text],
|
| 1336 |
api_name="API_AlignedCut",
|
|
|
|
| 1371 |
output_gallery = make_output_images_section()
|
| 1372 |
[
|
| 1373 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1374 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1375 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1376 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1377 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1378 |
] = make_parameters_section()
|
| 1379 |
old_school_ncut_checkbox = gr.Checkbox(label="Old school NCut", value=True, elem_id="old_school_ncut")
|
| 1380 |
+
invisible_list = [old_school_ncut_checkbox, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1381 |
+
num_sample_tsne_slider, knn_tsne_slider, sampling_method_dropdown, ncut_metric_dropdown]
|
| 1382 |
for item in invisible_list:
|
| 1383 |
item.visible = False
|
| 1384 |
# logging text box
|
|
|
|
| 1394 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1395 |
positive_prompt, negative_prompt,
|
| 1396 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1397 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1398 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1399 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown,
|
| 1400 |
old_school_ncut_checkbox
|
| 1401 |
],
|
| 1402 |
outputs=[output_gallery, logging_text],
|
|
|
|
| 1441 |
l3_affinity_focal_gamma_slider = gr.Slider(0.01, 1, step=0.01, label="Recursion #3: Affinity focal gamma", value=0.5, elem_id="recursion_l3_gamma")
|
| 1442 |
[
|
| 1443 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1444 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1445 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1446 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1447 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1448 |
] = make_parameters_section()
|
| 1449 |
num_eig_slider.visible = False
|
| 1450 |
affinity_focal_gamma_slider.visible = False
|
|
|
|
| 1463 |
input_gallery, model_dropdown, layer_slider, l1_num_eig_slider, node_type_dropdown,
|
| 1464 |
positive_prompt, negative_prompt,
|
| 1465 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1466 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1467 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1468 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown,
|
| 1469 |
false_placeholder, number_placeholder, true_placeholder,
|
| 1470 |
l2_num_eig_slider, l3_num_eig_slider, metric_dropdown,
|
| 1471 |
l1_affinity_focal_gamma_slider, l2_affinity_focal_gamma_slider, l3_affinity_focal_gamma_slider
|
|
|
|
| 1483 |
video_output_gallery = gr.Video(value=None, label="NCUT Embedding", elem_id="ncut", height="auto", show_share_button=False)
|
| 1484 |
[
|
| 1485 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1486 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1487 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1488 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1489 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1490 |
] = make_parameters_section()
|
| 1491 |
num_sample_tsne_slider.value = 1000
|
| 1492 |
perplexity_slider.value = 500
|
|
|
|
| 1506 |
video_input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1507 |
positive_prompt, negative_prompt,
|
| 1508 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1509 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1510 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1511 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown,
|
| 1512 |
place_holder_false, max_frame_number
|
| 1513 |
],
|
| 1514 |
outputs=[video_output_gallery, logging_text],
|
|
|
|
| 1552 |
with gr.Column(scale=5, min_width=200):
|
| 1553 |
[
|
| 1554 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1555 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1556 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1557 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1558 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1559 |
] = make_parameters_section(is_lisa=True)
|
| 1560 |
logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
|
| 1561 |
|
|
|
|
| 1567 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1568 |
positive_prompt, negative_prompt,
|
| 1569 |
true_placeholder, prompt1, prompt2, prompt3,
|
| 1570 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1571 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1572 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown
|
| 1573 |
],
|
| 1574 |
outputs=galleries + [logging_text],
|
| 1575 |
)
|
|
|
|
| 1599 |
gr.Markdown('---')
|
| 1600 |
[
|
| 1601 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1602 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1603 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1604 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1605 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1606 |
] = make_parameters_section(model_ratio=False)
|
| 1607 |
model_dropdown.value = "AlignedThreeModelAttnNodes"
|
| 1608 |
model_dropdown.visible = False
|
|
|
|
| 1624 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1625 |
positive_prompt, negative_prompt,
|
| 1626 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1627 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1628 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1629 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown
|
| 1630 |
],
|
| 1631 |
# outputs=galleries + [logging_text],
|
| 1632 |
outputs=[output_gallery, logging_text],
|
|
|
|
| 1685 |
gr.Markdown('Layer type: attention output (attn), without sum of residual')
|
| 1686 |
[
|
| 1687 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1688 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1689 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1690 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1691 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1692 |
] = make_parameters_section(model_ratio=False)
|
| 1693 |
num_eig_slider.visible = False
|
| 1694 |
affinity_focal_gamma_slider.visible = False
|
|
|
|
| 1717 |
input_gallery, model_dropdown, layer_slider, l1_num_eig_slider, node_type_dropdown,
|
| 1718 |
positive_prompt, negative_prompt,
|
| 1719 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1720 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1721 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1722 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown,
|
| 1723 |
false_placeholder, number_placeholder, true_placeholder,
|
| 1724 |
l2_num_eig_slider, l3_num_eig_slider, metric_dropdown,
|
| 1725 |
l1_affinity_focal_gamma_slider, l2_affinity_focal_gamma_slider, l3_affinity_focal_gamma_slider
|
|
|
|
| 1737 |
add_output_images_buttons(output_gallery)
|
| 1738 |
[
|
| 1739 |
model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
|
| 1740 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1741 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1742 |
perplexity_slider, n_neighbors_slider, min_dist_slider,
|
| 1743 |
+
sampling_method_dropdown, ncut_metric_dropdown, positive_prompt, negative_prompt
|
| 1744 |
] = make_parameters_section()
|
| 1745 |
# logging text box
|
| 1746 |
logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
|
|
|
|
| 1753 |
input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
|
| 1754 |
positive_prompt, negative_prompt,
|
| 1755 |
false_placeholder, no_prompt, no_prompt, no_prompt,
|
| 1756 |
+
affinity_focal_gamma_slider, num_sample_ncut_slider, ncut_knn_slider, ncut_indirect_connection, ncut_make_orthogonal,
|
| 1757 |
embedding_method_dropdown, embedding_metric_dropdown, num_sample_tsne_slider, knn_tsne_slider,
|
| 1758 |
+
perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown, ncut_metric_dropdown
|
| 1759 |
],
|
| 1760 |
outputs=[output_gallery, logging_text]
|
| 1761 |
)
|