import os import sys import cv2 import numpy as np import easyocr ''' This class uses the easyOCR character recognition technology to find the digits in a Sudoku puzzle. It does some overall image processing, then finds edges and lines in the overall image. It uses the line information to crop the image and find the individual cells that may or may not contain a digit. It uses the edge information to determine if a cell contains a digit or not, and then uses easyOCR to recognize the digit. It returns a string with a 0 representing an empty cell and 1-9 representing a digit in a cell. ''' class SudokuReader: # class variables easyocr_reader = "" ''' Class image processing constants ''' # Canny edge detector parameters canny_low_threshold = 38 # low threshold; middle of a decent range for test image canny_ratio = 3 # ratio of high to low threshold; suggested 3-4 # Hough line detector parameters (all determined by testing hough_threshold = 90 # "votes" required for line detection hough_min_length = 64 # minimum length of line in pixels hough_max_gap = 10 # maximum gap between segments to join segments in pixels # crop factor for individual cells cell_crop_factor_horizontal = 0.20 cell_crop_factor_vertical = 0.15 def __init__(self): """ Acquires an easyOCR "reader", which takes a long time. """ # get an easyOCR Reader; using GPU makes no difference self.easyocr_reader = easyocr.Reader(['en'], gpu=False) def _clean_image(self, image): """ Finds the Canny edges and Hough lines. Determines the extent of the actual puzzle using the lines. Crops the input image and the edges images per the puzzle extent, and returns the resulting images. :param image: the image to process :return: image cropped to puzzle boundary and image containing edges """ # Use the HoughLinesP function to detect lines gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) gray = cv2.blur(gray, (3,3)) edges = cv2.Canny(gray, self.canny_low_threshold, self.canny_low_threshold*self.canny_ratio, apertureSize=3) lines = cv2.HoughLinesP(edges, 1, np.pi / 180, self.hough_threshold, minLineLength=self.hough_min_length, maxLineGap=self.hough_max_gap) # find the leftmost and rightmost and topmost and bottommost line (edges of puzzle) x_min = 6000 x_max = 0 y_min = 6000 y_max = 0 for line in lines: x1, y1, x2, y2 = line[0] if x1 < x_min: x_min = x1 if x2 < x_min: x_min = x2 if y1 < y_min: y_min = y1 if y2 < y_min: y_min = y2 if x1 > x_max: x_max = x1 if x2 > x_max: x_max = x2 if y1 < y_min: y_min = y1 elif y1 > y_max: y_max = y1 if y2 < y_min: y_min = y2 elif y2 > y_max: y_max = y2 # crop the image to remove extraneous garbage cropped_image = gray[y_min:y_max, x_min:x_max] cropped_edges = edges[y_min:y_max, x_min:x_max] return cropped_image, cropped_edges def _get_cells(self, image): """ Splits an image into 81 cell images. Crops each cell image to strip off "waste" pixels, and return an array of images. :param image: image to split into cells :return: a list of image cells """ # get the rows first (axis=0) rows = np.array_split(image, 9, 0) cells = [] index = 0 # split the image for row in rows: #print('row size ', r.shape) #cv2.imshow("row ", r) cols = np.array_split(row, 9, 1) for cell in cols: # get size of cell h, w = cell.shape # print('height, width: ', h, w) # crop cell crop_x = int(w * self.cell_crop_factor_horizontal) crop_y = int(h * self.cell_crop_factor_vertical) cell = cell[crop_y:h - crop_y, crop_x:w - crop_x] cells.append(cell) #print('cell size' , cell.shape) #cv2.imshow("Split cell " + str(index), cell) index = index + 1 return cells # print as 9x9 from 1x81 @staticmethod def _print_cells(cells): k = 0 for i in range(9): output = "" for j in range(9): output = output + cells[k] + " " k = k + 1 print(output) @staticmethod def _print_digit_string(digit_string): k = 0 for i in range(9): output = "" for j in range(9): output = output + digit_string[k] + " " k = k + 1 print(output) @staticmethod def _is_cell_empty(cell_edges): """ Detects if a cell is empty, i.e., no digit. The theory is that since the cell contents are binary, i.e., black or white, in cells that are empty, the minimum and maximum values will be the same. In cells that have edges, that will not be the case. :param cell_edges: :return: True if empty, False if not """ # find min and max value in edge (min_val, max_val, min_loc, max_loc) = cv2.minMaxLoc(cell_edges) # check if above threshold if min_val != max_val: return False else: return True def find_puzzle(self, image) -> str: """ Finds the digits and empty cells in an image of a Sudoku puzzle. First crops an image and finds the Canny edges. Uses the edge information to detect an empty cell rather than a cell containing a digit. Uses an easyOCR detector to find the digit in a non-empty cell. :param image: the image representing the Sudoku puzzle :return: a string representing the puzzle """ # rotate image -90 degrees image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE) image_digits, image_edges = self._clean_image(image) # cv2.imshow('Edges', image_edges) # cv2.waitKey(0) # cv2.destroyAllWindows() # find the individual digit images digit_cells = self._get_cells(image_digits) # find the individual edge images edge_cells = self._get_cells(image_edges) # find digit in each cell digit_string = "" for i in range(81): # cell = cells[i] if self._is_cell_empty(edge_cells[i]): digit_string += '0' else: current = cv2.blur(digit_cells[i], (3, 3)) texts = self.easyocr_reader.readtext(current, allowlist='123456789', mag_ratio=2, low_text=0.3) if len(texts) == 0: # THIS SHOULD NOT HAPPEN!!! print('BAD interpretation, no text string') digit_string += '0' elif len(texts) > 1: print("BAD interpretation, multiple text strings") digit_string += 'X' else: text_info = texts[0] bbox, text, score = text_info digit_string += text return digit_string ####################### MAIN ############################ def main(image_path): # load the image image = cv2.imread(image_path) # get an instance reader = SudokuReader() digit_string = reader.find_puzzle(image) reader._print_digit_string(digit_string) if __name__ == '__main__': print('through main(); parms=', len(sys.argv)) if len(sys.argv) == 2: print("using input file") image_path = sys.argv[1] else: print("using hardcoded file") # path = 'from_v2_2-edit.jpg' # path = 'cropped_image_rotated.jpg' path = 'cropped_image.jpg' home_dir = os.path.expanduser("~") image_path = os.path.join(home_dir, path) main(image_path)