from ..torch_core import *
from ..basic_train import *
from ..basic_data import *
from ..vision.data import *
from ..vision.transform import *
from ..vision.image import *
from ..callbacks.hooks import *
from ..layers import *
from ipywidgets import widgets, Layout
from IPython.display import clear_output, display
__all__ = ['DatasetFormatter', 'ImageCleaner']
class DatasetFormatter():
"Returns a dataset with the appropriate format and file indices to be displayed."
@classmethod
def from_toplosses(cls, learn, n_imgs=None, **kwargs):
"Gets indices with top losses."
train_ds, train_idxs = cls.get_toplosses_idxs(learn, n_imgs, **kwargs)
return train_ds, train_idxs
@classmethod
def get_toplosses_idxs(cls, learn, n_imgs, **kwargs):
"Sorts `ds_type` dataset by top losses and returns dataset and sorted indices."
dl = learn.data.fix_dl
if not n_imgs: n_imgs = len(dl.dataset)
_,_,top_losses = learn.get_preds(ds_type=DatasetType.Fix, with_loss=True)
idxs = torch.topk(top_losses, n_imgs)[1]
return cls.padded_ds(dl.dataset, **kwargs), idxs
def padded_ds(ll_input, size=(250, 300), resize_method=ResizeMethod.CROP, padding_mode='zeros', **kwargs):
"For a LabelList `ll_input`, resize each image to `size` using `resize_method` and `padding_mode`."
return ll_input.transform(tfms=crop_pad(), size=size, resize_method=resize_method, padding_mode=padding_mode)
@classmethod
def from_similars(cls, learn, layer_ls:list=[0, 7, 2], **kwargs):
"Gets the indices for the most similar images."
train_ds, train_idxs = cls.get_similars_idxs(learn, layer_ls, **kwargs)
return train_ds, train_idxs
@classmethod
def get_similars_idxs(cls, learn, layer_ls, **kwargs):
"Gets the indices for the most similar images in `ds_type` dataset"
hook = hook_output(learn.model[layer_ls[0]][layer_ls[1]][layer_ls[2]])
dl = learn.data.fix_dl
ds_actns = cls.get_actns(learn, hook=hook, dl=dl, **kwargs)
similarities = cls.comb_similarity(ds_actns, ds_actns, **kwargs)
idxs = cls.sort_idxs(similarities)
return cls.padded_ds(dl, **kwargs), idxs
@staticmethod
def get_actns(learn, hook:Hook, dl:DataLoader, pool=AdaptiveConcatPool2d, pool_dim:int=4, **kwargs):
"Gets activations at the layer specified by `hook`, applies `pool` of dim `pool_dim` and concatenates"
print('Getting activations...')
actns = []
learn.model.eval()
with torch.no_grad():
for (xb,yb) in progress_bar(dl):
learn.model(xb)
actns.append((hook.stored).cpu())
if pool:
pool = pool(pool_dim)
return pool(torch.cat(actns)).view(len(dl.x),-1)
else: return torch.cat(actns).view(len(dl.x),-1)
@staticmethod
def comb_similarity(t1: torch.Tensor, t2: torch.Tensor, **kwargs):
# https://github.com/pytorch/pytorch/issues/11202
"Computes the similarity function between each embedding of `t1` and `t2` matrices."
print('Computing similarities...')
w1 = t1.norm(p=2, dim=1, keepdim=True)
w2 = w1 if t2 is t1 else t2.norm(p=2, dim=1, keepdim=True)
t = torch.mm(t1, t2.t()) / (w1 * w2.t()).clamp(min=1e-8)
return torch.tril(t, diagonal=-1)
def largest_indices(arr, n):
"Returns the `n` largest indices from a numpy array `arr`."
#https://stackoverflow.com/questions/6910641/how-do-i-get-indices-of-n-maximum-values-in-a-numpy-array
flat = arr.flatten()
indices = np.argpartition(flat, -n)[-n:]
indices = indices[np.argsort(-flat[indices])]
return np.unravel_index(indices, arr.shape)
@classmethod
def sort_idxs(cls, similarities):
"Sorts `similarities` and return the indexes in pairs ordered by highest similarity."
idxs = cls.largest_indices(similarities, len(similarities))
idxs = [(idxs[0][i], idxs[1][i]) for i in range(len(idxs[0]))]
return [e for l in idxs for e in l]
class ImageCleaner():
"Displays images for relabeling or deletion and saves changes in `path` as 'cleaned.csv'."
def __init__(self, dataset, fns_idxs, path, batch_size:int=5, duplicates=False):
self._all_images,self._batch = [],[]
self._path = Path(path)
self._batch_size = batch_size
if duplicates: self._batch_size = 2
self._duplicates = duplicates
self._labels = dataset.classes
self._all_images = self.create_image_list(dataset, fns_idxs)
self._csv_dict = {dataset.x.items[i]: dataset.y[i] for i in range(len(dataset))}
self._deleted_fns = []
self._skipped = 0
self.render()
@classmethod
def make_img_widget(cls, img, layout=Layout(), format='jpg'):
"Returns an image widget for specified file name `img`."
return widgets.Image(value=img, format=format, layout=layout)
@classmethod
def make_button_widget(cls, label, file_path=None, handler=None, style=None, layout=Layout(width='auto')):
"Return a Button widget with specified `handler`."
btn = widgets.Button(description=label, layout=layout)
if handler is not None: btn.on_click(handler)
if style is not None: btn.button_style = style
btn.file_path = file_path
btn.flagged_for_delete = False
return btn
@classmethod
def make_dropdown_widget(cls, description='Description', options=['Label 1', 'Label 2'], value='Label 1',
file_path=None, layout=Layout(), handler=None):
"Return a Dropdown widget with specified `handler`."
dd = widgets.Dropdown(description=description, options=options, value=value, layout=layout)
if file_path is not None: dd.file_path = file_path
if handler is not None: dd.observe(handler, names=['value'])
return dd
@classmethod
def make_horizontal_box(cls, children, layout=Layout()):
"Make a horizontal box with `children` and `layout`."
return widgets.HBox(children, layout=layout)
@classmethod
def make_vertical_box(cls, children, layout=Layout(), duplicates=False):
"Make a vertical box with `children` and `layout`."
if not duplicates: return widgets.VBox(children, layout=layout)
else: return widgets.VBox([children[0], children[2]], layout=layout)
def create_image_list(self, dataset, fns_idxs):
"Create a list of images, filenames and labels but first removing files that are not supposed to be displayed."
items = dataset.x.items
if self._duplicates:
chunked_idxs = chunks(fns_idxs, 2)
chunked_idxs = [chunk for chunk in chunked_idxs if Path(items[chunk[0]]).is_file() and Path(items[chunk[1]]).is_file()]
return [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for chunk in chunked_idxs for i in chunk]
else:
return [(dataset.x[i]._repr_jpeg_(), items[i], self._labels[dataset.y[i].data]) for i in fns_idxs if
Path(items[i]).is_file()]
def relabel(self, change):
"Relabel images by moving from parent dir with old label `class_old` to parent dir with new label `class_new`."
class_new,class_old,file_path = change.new,change.old,change.owner.file_path
fp = Path(file_path)
parent = fp.parents[1]
self._csv_dict[fp] = class_new
def next_batch(self, _):
"Handler for 'Next Batch' button click. Delete all flagged images and renders next batch."
for img_widget, delete_btn, fp, in self._batch:
fp = delete_btn.file_path
if (delete_btn.flagged_for_delete == True):
self.delete_image(fp)
self._deleted_fns.append(fp)
self._all_images = self._all_images[self._batch_size:]
self.empty_batch()
self.render()
def on_delete(self, btn):
"Flag this image as delete or keep."
btn.button_style = "" if btn.flagged_for_delete else "danger"
btn.flagged_for_delete = not btn.flagged_for_delete
def empty_batch(self): self._batch[:] = []
def delete_image(self, file_path):
del self._csv_dict[file_path]
def empty(self):
return len(self._all_images) == 0
def get_widgets(self, duplicates):
"Create and format widget set."
widgets = []
for (img,fp,human_readable_label) in self._all_images[:self._batch_size]:
img_widget = self.make_img_widget(img, layout=Layout(height='250px', width='300px'))
dropdown = self.make_dropdown_widget(description='', options=self._labels, value=human_readable_label,
file_path=fp, handler=self.relabel, layout=Layout(width='auto'))
delete_btn = self.make_button_widget('Delete', file_path=fp, handler=self.on_delete)
widgets.append(self.make_vertical_box([img_widget, dropdown, delete_btn],
layout=Layout(width='auto', height='300px',
overflow_x="hidden"), duplicates=duplicates))
self._batch.append((img_widget, delete_btn, fp))
return widgets
def batch_contains_deleted(self):
"Check if current batch contains already deleted images."
if not self._duplicates: return False
imgs = [self._all_images[:self._batch_size][0][1], self._all_images[:self._batch_size][1][1]]
return any(img in self._deleted_fns for img in imgs)
def write_csv(self):
# Get first element's file path so we write CSV to same directory as our data
csv_path = self._path/'cleaned.csv'
with open(csv_path, 'w') as f:
csv_writer = csv.writer(f)
csv_writer.writerow(['name','label'])
for pair in self._csv_dict.items():
pair = [os.path.relpath(pair[0], self._path), pair[1]]
csv_writer.writerow(pair)
return csv_path
def render(self):
"Re-render Jupyter cell for batch of images."
clear_output()
self.write_csv()
if self.empty() and self._skipped>0:
return display(f'No images to show :). {self._skipped} pairs were '
f'skipped since at least one of the images was deleted by the user.')
elif self.empty():
return display('No images to show :)')
if self.batch_contains_deleted():
self.next_batch(None)
self._skipped += 1
else:
display(self.make_horizontal_box(self.get_widgets(self._duplicates)))
display(self.make_button_widget('Next Batch', handler=self.next_batch, style="primary"))