import numpy as np import numpy.typing as npt from matplotlib import pyplot as plt import cv2 import uz_framework.image as uz_image import uz_framework.text as uz_text import os ############################################## # EXCERCISE 1: Exercise 1: Image derivatives # ############################################## def ex1(): one_a() one_b() #one_d() def one_a() -> None: img = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64) sigmas = [3, 6, 9, 12] # Plot the points fig, axs = plt.subplots(2, len(sigmas)) fig.suptitle("Hessian corner detection") for i, sigma in enumerate(sigmas): determinant, hessian_points = uz_image.hessian_points(img, sigma, 0.004) # Plot determinant axs[0, i].imshow(determinant) axs[0, i].set_title(f"Sigma: {sigma}") # Plot grayscale image axs[1, i].imshow(img, cmap="gray") # Plot scatter hessian points (x, y) axs[1, i].scatter(hessian_points[:, 1], hessian_points[:, 0], s=20, c="r", marker="x") plt.show() def one_b() -> None: img = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64) sigmas = [3, 6, 9] # Plot the points fig, axs = plt.subplots(2, len(sigmas)) fig.suptitle("Harris corner detection") for i, sigma in enumerate(sigmas): determinant, harris_points = uz_image.harris_detector(img, sigma, treshold=1e-6) # Plot determinant axs[0, i].imshow(determinant) axs[0, i].set_title(f"Sigma: {sigma}") # Plot grayscale image axs[1, i].imshow(img, cmap="gray") # Plot scatter hessian points axs[1, i].scatter(harris_points[:, 1], harris_points[:, 0], s=20, c="r", marker="x") plt.show() def one_d(): """ mplement a local feature detector of your choice (e.g. SIFT, SURF, BRIEF, HOG). Test it on other assignments and report on the changes (increased robustness etc.). """ # Show how the descriptors do not work image_a = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64) image_b = uz_image.imread_gray("data/graf/graf_b.jpg", uz_image.ImageType.float64) # Get the keypoints keypoints_a, descriptors_a = uz_image.sift(image_a.astype(np.float64), plot=True, get_descriptors=True) keypoints_b, descriptors_b = uz_image.sift(image_b.astype(np.float64), plot=False, get_descriptors=True) # Find correspondences correspondences_a = uz_image.find_correspondences(descriptors_a, descriptors_b) correspondences_b = uz_image.find_correspondences(descriptors_b, descriptors_a) def check_reciprocacvbility(c_a, c_b): for match in c_a: if np.flip(match) not in c_b: index = np.where((c_a == match).all(axis=1))[0][0] c_a = np.delete(c_a, index, axis=0) return c_a def remove_duplicates(c_a): unique, counts = np.unique(c_a[:, 1], return_counts=True) duplicates = unique[counts > 1] for duplicate in duplicates: index = np.where(c_a[:, 1] == duplicate)[0] c_a = np.delete(c_a, index, axis=0) return c_a correspondences_a = remove_duplicates(correspondences_a) correspondences_b = remove_duplicates(correspondences_b) correspondences_a = check_reciprocacvbility(correspondences_a, correspondences_b) correspondences_b = check_reciprocacvbility(correspondences_b, correspondences_a) # Now map correspondences to the keypoints def map_indexes_to_points(a_points, b_points, corrs_a, corrs_b, img_a_k, img_b_k): for correspondence in corrs_a: ix = np.argwhere(correspondence[0] == corrs_b[:, 1]) if ix.size > 0: a_points.append(np.flip(img_a_k[correspondence[0]])) b_points.append(np.flip(img_b_k[correspondence[1]])) image_a_points = [] image_b_points = [] map_indexes_to_points(image_a_points, image_b_points, correspondences_a, correspondences_b, keypoints_a, keypoints_b) map_indexes_to_points(image_b_points, image_a_points, correspondences_b, correspondences_a, keypoints_b, keypoints_a) # Plot the matches uz_image.display_matches(image_a, np.array(image_a_points), image_b, np.array(image_b_points)) # But what if we compute simple descriptors? descriptors_a = uz_image.simple_descriptors(image_a, keypoints_a[:,0], keypoints_a[:,1]) descriptors_b = uz_image.simple_descriptors(image_b, keypoints_b[:,0], keypoints_b[:,1]) # Find correspondences correspondences_a = uz_image.find_correspondences(descriptors_a, descriptors_b) correspondences_b = uz_image.find_correspondences(descriptors_b, descriptors_a) correspondences_a = remove_duplicates(correspondences_a) correspondences_b = remove_duplicates(correspondences_b) correspondences_a = check_reciprocacvbility(correspondences_a, correspondences_b) correspondences_b = check_reciprocacvbility(correspondences_b, correspondences_a) # Now map correspondences to the keypoints image_a_points = [] image_b_points = [] map_indexes_to_points(image_a_points, image_b_points, correspondences_a, correspondences_b, keypoints_a, keypoints_b) map_indexes_to_points(image_b_points, image_a_points, correspondences_b, correspondences_a, keypoints_b, keypoints_a) # Plot the matches uz_image.display_matches(image_a, np.array(image_a_points), image_b, np.array(image_b_points)) def ex2(): two_a() two_b() def two_a() -> None: """ Hello """ graph_a_small = uz_image.imread_gray("data/graf/graf_a_small.jpg", uz_image.ImageType.float64) graph_b_small = uz_image.imread_gray("data/graf/graf_b_small.jpg", uz_image.ImageType.float64) # Get the keypoints _, graph_a_keypoints = uz_image.harris_detector(graph_a_small, 3, treshold=1e-6) _, graph_b_keypoints = uz_image.harris_detector(graph_b_small, 3, treshold=1e-6) # Get the descriptors graph_a_descriptors = uz_image.simple_descriptors(graph_a_small, graph_a_keypoints[:,0], graph_a_keypoints[:,1]) graph_b_descriptors = uz_image.simple_descriptors(graph_b_small, graph_b_keypoints[:,0], graph_b_keypoints[:,1]) # Find the correspondences matches_a = uz_image.find_correspondences(graph_a_descriptors, graph_b_descriptors) matches_b = uz_image.find_correspondences(graph_b_descriptors, graph_a_descriptors) matches_a_coordinates = [] matches_b_coordinates = [] for i, match in enumerate(matches_a): matches_a_coordinates.append(np.flip(graph_a_keypoints[match[0]])) matches_b_coordinates.append(np.flip(graph_b_keypoints[match[1]])) # Plot the matches uz_image.display_matches(graph_a_small, matches_a_coordinates, graph_b_small, matches_b_coordinates) def two_b() -> None: """ also as two_c as it has improved method included """ #graph_a_small = uz_image.imread_gray("datam/img1.jpg", uz_image.ImageType.float64) #graph_b_small = uz_image.imread_gray("datam/img2.jpg", uz_image.ImageType.float64) graph_a_small = uz_image.imread_gray("data/graf/graf_a_small.jpg", uz_image.ImageType.float64) graph_b_small = uz_image.imread_gray("data/graf/graf_b_small.jpg", uz_image.ImageType.float64) a, b = uz_image.find_matches(graph_a_small, graph_b_small) uz_image.display_matches(graph_a_small, a, graph_b_small, b) def ex3(): three_a() three_b() def three_a() -> None: """ hello """ keypoints_path = ["data/newyork/newyork.txt", "data/graf/graf.txt"] images_a_path = ["data/newyork/newyork_a.jpg", "data/graf/graf_a.jpg"] images_b_path = ["data/newyork/newyork_b.jpg", "data/graf/graf_b.jpg"] fig, axs = plt.subplots(4, 2) fig.suptitle("Transformation and rotation using homography") for i in range(len(keypoints_path)): keypoints = np.loadtxt(keypoints_path[i], dtype=np.float64) image_a = uz_image.imread_gray(images_a_path[i], uz_image.ImageType.float64) image_b = uz_image.imread_gray(images_b_path[i], uz_image.ImageType.float64) axs[i*2, 0].imshow(image_a, cmap="gray") axs[i*2, 1].imshow(image_b, cmap="gray") axs[i*2, 0].set_title("A") axs[i*2, 1].set_title("B") homography_matrix = uz_image.estimate_homography(keypoints) img_output = cv2.warpPerspective(image_a, homography_matrix, (image_a.shape[1], image_a.shape[0])) axs[i*2+1, 1].imshow(img_output, cmap="gray") axs[i*2+1, 1].set_title("A transformed") # invert keypoints keypoints[:,[0, 1, 2, 3]] = keypoints[:,[2, 3, 0, 1]] homography_matrix = uz_image.estimate_homography(keypoints) img_output = cv2.warpPerspective(image_b, homography_matrix, (image_b.shape[1], image_b.shape[0])) axs[i*2+1, 0].imshow(img_output, cmap="gray") axs[i*2+1, 0].set_title("B transformed") plt.show() def three_b() -> None: """ Hi """ image_a = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64) image_b = uz_image.imread_gray("data/graf/graf_b.jpg", uz_image.ImageType.float64) #image_a = uz_image.imread_gray("datam/img1.jpg", uz_image.ImageType.float64) #image_b = uz_image.imread_gray("datam/img2.jpg", uz_image.ImageType.float64) #image_a = uz_image.imread_gray("data/newyork/newyork_a.jpg", uz_image.ImageType.float64) #image_b = uz_image.imread_gray("data/newyork/newyork_b.jpg", uz_image.ImageType.float64) # Does not work for newyork dataset, becouse the keypoints are not reciprocal # Get the keypoints keypoints_a, keypoints_b = uz_image.find_matches(image_a, image_b, sigma=3) hm, best_inliers = uz_image.ransac(image_a,keypoints_a, image_b, keypoints_b, iterations=100, threshold=5) # Plot the matches def map_keypoints(keypoints): # Map the keypoints a_points =[] b_points = [] for row in keypoints: a_points.append((row[0], row[1])) b_points.append((row[2], row[3])) return np.array(a_points), np.array(b_points) a,b = map_keypoints(best_inliers) uz_image.display_matches(image_a, a, image_b, b) img_output = cv2.warpPerspective(image_a, hm, (image_b.shape[1], image_b.shape[0])) fig, axs = plt.subplots(1, 2) fig.suptitle("Transformation and rotation using homography") axs[0].imshow(image_b, cmap="gray") axs[1].imshow(img_output, cmap="gray") plt.show() # ######## # # SOLUTION # # ######## # def main(): ex1() ex2() ex3() if __name__ == '__main__': main()