uz_assignments/assignment4/solution.py

import numpy as np
import numpy.typing as npt
from matplotlib import pyplot as plt
import cv2
import uz_framework.image as uz_image
import uz_framework.text as uz_text
import os

##############################################
# EXCERCISE 1: Exercise 1: Image derivatives #
##############################################

def ex1():
	#one_a()
	#one_b()
	one_d()

def one_a() -> None:
	img = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64)
	sigmas = [3, 6, 9, 12]
	# Plot the points
	fig, axs = plt.subplots(2, len(sigmas))
	fig.suptitle("Hessian corner detection")

	for i, sigma in enumerate(sigmas):
		determinant, hessian_points  = uz_image.hessian_points(img, sigma, 0.004)
		# Plot determinant
		axs[0, i].imshow(determinant)
		axs[0, i].set_title(f"Sigma: {sigma}")
		# Plot grayscale image
		axs[1, i].imshow(img, cmap="gray")
		# Plot scatter hessian points (x, y)
		axs[1, i].scatter(hessian_points[:, 1], hessian_points[:, 0], s=20, c="r", marker="x")

	plt.show()

def one_b() -> None:
	img = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64)
	sigmas = [3, 6, 9]
	# Plot the points
	fig, axs = plt.subplots(2, len(sigmas))
	fig.suptitle("Harris corner detection")

	for i, sigma in enumerate(sigmas):
		determinant, harris_points  = uz_image.harris_detector(img, sigma, treshold=1e-6)
		# Plot determinant
		axs[0, i].imshow(determinant)
		axs[0, i].set_title(f"Sigma: {sigma}")
		# Plot grayscale image
		axs[1, i].imshow(img, cmap="gray")
		# Plot scatter hessian points
		axs[1, i].scatter(harris_points[:, 1], harris_points[:, 0], s=20, c="r", marker="x")

	plt.show()

def one_d():
	"""
	mplement a local feature detector of your choice (e.g. SIFT, SURF,
	BRIEF, HOG). Test it on other assignments and report on the changes (increased
	robustness etc.).
	"""
	# Show how the descriptors do not work
	image_a = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64)
	image_b = uz_image.imread_gray("data/graf/graf_b.jpg", uz_image.ImageType.float64)

	# Get the keypoints
	keypoints_a, descriptors_a = uz_image.sift(image_a.astype(np.float64), plot=True, get_descriptors=True)
	keypoints_b, descriptors_b = uz_image.sift(image_b.astype(np.float64), plot=False, get_descriptors=True)

	# Find correspondences
	correspondences_a = uz_image.find_correspondences(descriptors_a, descriptors_b)
	correspondences_b = uz_image.find_correspondences(descriptors_b, descriptors_a)


	def check_reciprocacvbility(c_a, c_b):
		for match in c_a:
			if np.flip(match) not in c_b:
				index = np.where((c_a == match).all(axis=1))[0][0]
				c_a = np.delete(c_a, index, axis=0)
		return c_a

	def remove_duplicates(c_a):
		unique, counts = np.unique(c_a[:, 1], return_counts=True)
		duplicates = unique[counts > 1]
		for duplicate in duplicates:
			index = np.where(c_a[:, 1] == duplicate)[0]
			c_a = np.delete(c_a, index, axis=0)

		return c_a
	correspondences_a = remove_duplicates(correspondences_a) 
	correspondences_b = remove_duplicates(correspondences_b)
	
	correspondences_a = check_reciprocacvbility(correspondences_a, correspondences_b)
	correspondences_b = check_reciprocacvbility(correspondences_b, correspondences_a)

	# Now map correspondences to the keypoints 
	def map_indexes_to_points(a_points, b_points, corrs_a, corrs_b, img_a_k, img_b_k):
		for correspondence in corrs_a:
			ix = np.argwhere(correspondence[0] == corrs_b[:, 1])
			if ix.size > 0:
				a_points.append(np.flip(img_a_k[correspondence[0]]))
			b_points.append(np.flip(img_b_k[correspondence[1]]))	

	image_a_points = []
	image_b_points = []

	map_indexes_to_points(image_a_points, image_b_points, correspondences_a, correspondences_b, keypoints_a, keypoints_b)
	map_indexes_to_points(image_b_points, image_a_points, correspondences_b, correspondences_a, keypoints_b, keypoints_a)

	# Plot the matches
	uz_image.display_matches(image_a, np.array(image_a_points), image_b, np.array(image_b_points))

	# But what if we compute simple descriptors?
	descriptors_a = uz_image.simple_descriptors(image_a, keypoints_a[:,0], keypoints_a[:,1])
	descriptors_b = uz_image.simple_descriptors(image_b, keypoints_b[:,0], keypoints_b[:,1])

	# Find correspondences
	correspondences_a = uz_image.find_correspondences(descriptors_a, descriptors_b)
	correspondences_b = uz_image.find_correspondences(descriptors_b, descriptors_a)

	correspondences_a = remove_duplicates(correspondences_a) 
	correspondences_b = remove_duplicates(correspondences_b)
	
	correspondences_a = check_reciprocacvbility(correspondences_a, correspondences_b)
	correspondences_b = check_reciprocacvbility(correspondences_b, correspondences_a)

	# Now map correspondences to the keypoints
	image_a_points = []
	image_b_points = []

	map_indexes_to_points(image_a_points, image_b_points, correspondences_a, correspondences_b, keypoints_a, keypoints_b)
	map_indexes_to_points(image_b_points, image_a_points, correspondences_b, correspondences_a, keypoints_b, keypoints_a)

	# Plot the matches
	uz_image.display_matches(image_a, np.array(image_a_points), image_b, np.array(image_b_points))


def ex2():
	two_a()
	two_b()

def two_a() -> None:
	"""
	Hello
	"""
	graph_a_small = uz_image.imread_gray("data/graf/graf_a_small.jpg", uz_image.ImageType.float64)
	graph_b_small = uz_image.imread_gray("data/graf/graf_b_small.jpg", uz_image.ImageType.float64)

	# Get the keypoints
	_, graph_a_keypoints = uz_image.harris_detector(graph_a_small, 3, treshold=1e-6)
	_, graph_b_keypoints = uz_image.harris_detector(graph_b_small, 3, treshold=1e-6)

	# Get the descriptors
	graph_a_descriptors = uz_image.simple_descriptors(graph_a_small, graph_a_keypoints[:,0], graph_a_keypoints[:,1])
	graph_b_descriptors = uz_image.simple_descriptors(graph_b_small, graph_b_keypoints[:,0], graph_b_keypoints[:,1])

	# Find the correspondences
	matches_a = uz_image.find_correspondences(graph_a_descriptors, graph_b_descriptors)
	matches_b = uz_image.find_correspondences(graph_b_descriptors, graph_a_descriptors)

	matches_a_coordinates = []
	matches_b_coordinates = []
	for i, match in enumerate(matches_a):
		matches_a_coordinates.append(np.flip(graph_a_keypoints[match[0]]))
		matches_b_coordinates.append(np.flip(graph_b_keypoints[match[1]]))

	# Plot the matches
	uz_image.display_matches(graph_a_small, matches_a_coordinates, graph_b_small, matches_b_coordinates)

def two_b() -> None:
	"""
	also as two_c as it has improved method included
	"""
	#graph_a_small = uz_image.imread_gray("datam/img1.jpg", uz_image.ImageType.float64)
	#graph_b_small = uz_image.imread_gray("datam/img2.jpg", uz_image.ImageType.float64)
	graph_a_small = uz_image.imread_gray("data/graf/graf_a_small.jpg", uz_image.ImageType.float64)
	graph_b_small = uz_image.imread_gray("data/graf/graf_b_small.jpg", uz_image.ImageType.float64)

	a, b = uz_image.find_matches(graph_a_small, graph_b_small)

	uz_image.display_matches(graph_a_small, a, graph_b_small, b)


def ex3():
	#three_a()
	three_b()

def three_a() -> None:
	"""
	hello
	"""
	keypoints_path = ["data/newyork/newyork.txt", "data/graf/graf.txt"]
	images_a_path = ["data/newyork/newyork_a.jpg", "data/graf/graf_a.jpg"]
	images_b_path = ["data/newyork/newyork_b.jpg", "data/graf/graf_b.jpg"]

	fig, axs = plt.subplots(4, 2)
	fig.suptitle("Transformation and rotation using homography")

	for i in range(len(keypoints_path)):

		keypoints = np.loadtxt(keypoints_path[i], dtype=np.float64)
		image_a = uz_image.imread_gray(images_a_path[i], uz_image.ImageType.float64)
		image_b = uz_image.imread_gray(images_b_path[i], uz_image.ImageType.float64)

		axs[i*2, 0].imshow(image_a, cmap="gray")
		axs[i*2, 1].imshow(image_b, cmap="gray")
		axs[i*2, 0].set_title("A")
		axs[i*2, 1].set_title("B")

		homography_matrix = uz_image.estimate_homography(keypoints)
		img_output = cv2.warpPerspective(image_a, homography_matrix, (image_a.shape[1], image_a.shape[0]))
		axs[i*2+1, 1].imshow(img_output, cmap="gray")
		axs[i*2+1, 1].set_title("A transformed")

		# invert keypoints
		keypoints[:,[0, 1, 2, 3]] = keypoints[:,[2, 3, 0, 1]]	
		homography_matrix = uz_image.estimate_homography(keypoints)
		img_output = cv2.warpPerspective(image_b, homography_matrix, (image_b.shape[1], image_b.shape[0]))
		axs[i*2+1, 0].imshow(img_output, cmap="gray")
		axs[i*2+1, 0].set_title("B transformed")
	plt.show()


def three_b() -> None:
	"""
	Hi
	"""
	image_a = uz_image.imread_gray("data/graf/graf_a.jpg", uz_image.ImageType.float64)
	image_b = uz_image.imread_gray("data/graf/graf_b.jpg", uz_image.ImageType.float64)
	#image_a = uz_image.imread_gray("datam/img1.jpg", uz_image.ImageType.float64)
	#image_b = uz_image.imread_gray("datam/img2.jpg", uz_image.ImageType.float64)
	#image_a = uz_image.imread_gray("data/newyork/newyork_a.jpg", uz_image.ImageType.float64)
	#image_b = uz_image.imread_gray("data/newyork/newyork_b.jpg", uz_image.ImageType.float64)
	# Does not work for newyork dataset, becouse the keypoints are not reciprocal

	# Get the keypoints
	keypoints_a, keypoints_b = uz_image.find_matches(image_a, image_b, sigma=3)

	hm, best_inliers = uz_image.ransac(image_a,keypoints_a, image_b, keypoints_b, iterations=100, threshold=5)

	# Plot the matches

	def map_keypoints(keypoints):
		# Map the keypoints
		a_points =[]
		b_points = []
		for row in keypoints:
			a_points.append((row[0], row[1]))
			b_points.append((row[2], row[3]))
		return np.array(a_points), np.array(b_points)

	a,b = map_keypoints(best_inliers)

	uz_image.display_matches(image_a, a, image_b, b)

	img_output = cv2.warpPerspective(image_a, hm, (image_b.shape[1], image_b.shape[0]))

	fig, axs = plt.subplots(1, 2)
	fig.suptitle("Transformation and rotation using homography")
	axs[0].imshow(image_b, cmap="gray")
	axs[1].imshow(img_output, cmap="gray")
	plt.show()	

# ######## #
# SOLUTION #
# ######## #

def main():
	#ex1() 
	#ex2()
	ex3()

if __name__ == '__main__':
	main()