# System

import json

import multiprocessing

import os

import warnings

# Advanced

import xml.etree.ElementTree as ET

from argparse import ArgumentParser

from pathlib import Path

import pandas as pd

import cv2

import matplotlib.pyplot as plt

# Numpy

import numpy as np

# Image Processing

from PIL import Image

# # Fix to get the dlls to load properly under python >= 3.8 and windows

script_dir = os.path.dirname(os.path.realpath(__file__))

try:

    openslide_dll_path = os.path.join(script_dir, "..", "openslide-win64-20171122", "bin")

    os.add_dll_directory(openslide_dll_path)

    # print(openslide_dll_path)

except Exception as e:

    pass

# noinspection PyPep8

import openslide

# Custom

# noinspection PyPep8

import tissue_detection

_MULTIPROCESS = True

global lock

class WSIHandler:

    def __init__(self, config_path="resources/config.json"):

        self.slide = None

        self.output_path = None

        self.total_width = 0

        self.total_height = 0

        self.levels = 0

        self.current_level = 0

        self.annotation_list = None

        self.annotation_dict = None

        self.config = self.load_config(config_path)

        assert "save_annotated_only" in self.config.keys()

        self.annotated_only = self.config["save_annotated_only"]

        self.scanner = None

        self.res_x = None

        self.res_y = None

        self.validate_label_dict()

    def validate_label_dict(self):

        self.check_at_most_one_unannotated_label()

        self.check_unannotated_label_first()

    def check_at_most_one_unannotated_label(self):

        label_dict = self.config["label_dict"]

        unannotated_labels = []

        for label, label_config in label_dict.items():

            if not label_config["annotated"]:

                unannotated_labels.append(label)

        assert len(unannotated_labels) < 2, (f"More than one label (=tissue type) is marked as unannotated in the "

                                             f"config.label_dict. Please make sure that at most one type (usually "

                                             f"non-tumor) is marked as unannotated. The labels in question are "

                                             f"{unannotated_labels}.")

    def check_unannotated_label_first(self):

        label_dict = self.config["label_dict"]

        for label in list(label_dict)[1:]:

            assert label_dict[label]["annotated"], (f"WSIHandler requires the unannotated label to be located in the "

                                                    f"first position in config.label_dict. Please move the unannotated "

                                                    f"tissue type '{label}' to the first position.")

    def print_and_log_slide_error(self, slide_name, error_msg, method_name):

        print(f"Error in slide {slide_name}. The error is: {type(error_msg).__name__}: {error_msg} in method: "

              f"{method_name}.")

        with lock:

            with open(os.path.join(self.config["output_path"], "error_log.txt"), "a") as f:

                f.write(f"Error in slide {slide_name}. The error is: {type(error_msg).__name__}: {error_msg} in "

                        f"method: {method_name}.")

    @staticmethod

    def load_config(config_path):

        assert os.path.exists(config_path), "Cannot find " + config_path

        with open(config_path) as json_file:

            config = json.load(json_file)

        assert 1 >= config["tissue_coverage"] >= 0, "Tissue coverage must be between 1 and 0"

        assert config["blocked_threads"] >= 0

        assert config["patches_per_tile"] >= 1, "Patches per tile must be >= 1"

        assert 0 <= config["overlap"] < 1, "Overlap must be between 1 and 0"

        assert config["annotation_overlap"] >= 0 and config["overlap"] < 1, "Annotation overlap must be between 1 and 0"

        return config

    def load_slide(self, slide_path):

        self.slide = openslide.OpenSlide(slide_path)

        self.total_width = self.slide.dimensions[0]

        self.total_height = self.slide.dimensions[1]

        self.levels = self.slide.level_count - 1

        processing_level = self.config["processing_level"]

        if self.levels < self.config["processing_level"]:

            print("###############################################")

            print(

                "WARNING: Processing level above highest available slide level. Maximum slide level is "

                + str(self.levels)

                + ", processing level is "

                + str(self.config["processing_level"])

                + ". Setting processing level to "

                + str(self.levels)

            )

            print("###############################################")

            processing_level = self.levels

        return processing_level

    def load_annotation(self, annotation_path):

        annotation_dict = {}

        file_format = Path(annotation_path).suffix

        # QuPath exports

        if file_format == ".geojson" or file_format == ".txt":

            with open(annotation_path) as annotation_file:

                annotations = json.load(annotation_file)

            for polygon_nb in range(len(annotations["features"])):

                if annotations["features"][polygon_nb]["geometry"]["type"] == "Polygon":

                    if (annotations["features"][polygon_nb]["properties"]["classification"]["name"] in

                            self.config["label_dict"].keys()):

                        annotation_dict.update({polygon_nb: {

                            "coordinates": annotations["features"][polygon_nb]["geometry"]["coordinates"][0],

                            "tissue_type": annotations["features"][polygon_nb]["properties"]["classification"][

                                "name"]}})

                    else:

                        warnings.warn(f'Unknown annotation type in file {annotation_file.name}: The annotation label '

                                      f'"{annotations["features"][polygon_nb]["properties"]["classification"]["name"]}"'

                                      f' is not part of the provided label dictionary '

                                      f'(keys: {list(self.config["label_dict"].keys())}. Skipping.')

                else:

                    warnings.warn(f'Not implemented warning in file {annotation_file.name}: The handling of the QuPath '

                                  f'annotation type {annotations["features"][polygon_nb]["geometry"]["type"]} '

                                  f'(id:{annotations["features"][polygon_nb]["id"]}) has not been implemented, yet. '

                                  f'Skipping.')

        # xml for CAMELYON17

        elif file_format == ".xml":

            tree = ET.parse(annotation_path)

            root = tree.getroot()

            for elem in root:

                polygon_nb = 0

                for subelem in elem:

                    items = subelem.attrib

                    if "Type" in items.keys():

                        if items["Type"] == "Polygon":

                            polygon_list = []

                            for coordinates in subelem:

                                for coord in coordinates:

                                    polygon_list.append([float(coord.attrib["X"]), float(coord.attrib["Y"])])

                            # all annotationy in CAMELYON17 are tumor, so this is a pseudo label

                            annotation_dict.update({polygon_nb: {"coordinates": polygon_list, "tissue_type": "Tumor"}})

                            polygon_nb += 1

        else:

            return None

        return annotation_dict

    def get_img(self, level=None, show=False):

        if level is None:

            level = self.levels

        dims = self.slide.level_dimensions[level]

        image = np.array(self.slide.read_region((0, 0), level, dims))

        if show:

            # Katja: fix for Wayland issue on my Ubuntu:

            # run 'export QT_QPA_PLATFORM=xcb' before opening pycharm (in the same terminal)

            plt.imshow(image)

            plt.title("Slide image")

            plt.show()

        return image, level

    def apply_tissue_detection(self, level=None, show=False):

        if level is not None:

            image, level = self.get_img(level, show)

        else:

            image, level = self.get_img(show=show)

        tissue_mask = tissue_detection.tissue_detection(image, remove_top_percentage=0)

        if show:

            plt.imshow(tissue_mask)

            plt.title("Tissue Mask")

            plt.show()

        return tissue_mask, level

    def determine_tile_size(self, level):

        if self.config["calibration"]["use_non_pixel_lengths"]:

            tile_size_0 = (self.config["calibration"]["patch_size_microns"] / self.res_x) * self.config[

                "patches_per_tile"

            ]

        else:

            tile_size_0 = self.config["patches_per_tile"] * self.config["patch_size"]

        downscale_factor = int(self.slide.level_downsamples[level])

        tile_size = int(tile_size_0 / downscale_factor)

        assert self.config["patches_per_tile"] >= 1, "Patches per tile must be greater than 1."

        return tile_size

    def get_relevant_tiles(self, tissue_mask, tile_size, min_coverage, level, show=False):

        rows, row_residue = divmod(tissue_mask.shape[0], tile_size)

        cols, col_residue = divmod(tissue_mask.shape[1], tile_size)

        if row_residue:

            rows += 1

        if col_residue:

            cols += 1

        if self.config["use_tissue_detection"]:

            colored = cv2.cvtColor(tissue_mask, cv2.COLOR_GRAY2RGB)

        if self.annotation_dict is not None:

            annotation_mask = np.zeros(shape=(tissue_mask.shape[0], tissue_mask.shape[1]))

            scaling_factor = self.slide.level_downsamples[level]

            scaled_list = [

                [[point[0] / scaling_factor, point[1] / scaling_factor]

                 for point in self.annotation_dict[polygon]["coordinates"]]

                for polygon in self.annotation_dict

            ]

            for polygon in scaled_list:

                cv2.fillPoly(annotation_mask, [np.array(polygon).astype(np.int32)], 1)

        relevant_tiles_dict = {}

        tile_nb = 0

        # +1 to solve border issues

        for row in range(rows):

            for col in range(cols):

                tile = tissue_mask[

                       row * tile_size: row * tile_size + tile_size, col * tile_size: col * tile_size + tile_size

                       ]

                tissue_coverage = np.count_nonzero(tile) / tile.size

                annotated = False

                if self.annotation_dict is not None:

                    if (np.count_nonzero(annotation_mask[row * tile_size: row * tile_size + tile_size,

                                         col * tile_size: col * tile_size + tile_size, ]) > 0):

                        annotated = True

                if (tissue_coverage >= min_coverage or

                        (self.config["keep_annotated_tiles_despite_too_little_tissue_coverage"] and annotated)):

                    relevant_tiles_dict.update(

                        {

                            tile_nb: {

                                "x": col * tile_size,

                                "y": row * tile_size,

                                "size": tile_size,

                                "level": level,

                                "annotated": annotated,

                            }

                        }

                    )

                    if self.config["use_tissue_detection"]:

                        if annotated:

                            colored = cv2.rectangle(

                                colored,

                                (col * tile_size, row * tile_size),

                                (col * tile_size + tile_size, row * tile_size + tile_size),

                                (0, 255, 0),

                                3,

                            )

                        else:

                            colored = cv2.rectangle(

                                colored,

                                (col * tile_size, row * tile_size),

                                (col * tile_size + tile_size, row * tile_size + tile_size),

                                (255, 0, 0),

                                1,

                            )

                    tile_nb += 1

        if show and self.config["use_tissue_detection"]:

            plt.imshow(colored)

            plt.title("Tiled image")

            plt.show()

        return relevant_tiles_dict

    @staticmethod

    def tissue_percentage_over_threshold(label, label_dict, percentage):

        if label_dict[label]["type"] == "==":

            if label_dict[label]["threshold"] == percentage:

                return label, percentage

        elif label_dict[label]["type"] == ">=":

            if percentage >= label_dict[label]["threshold"]:

                return label, percentage

        elif label_dict[label]["type"] == ">":

            if percentage > label_dict[label]["threshold"]:

                return label, percentage

        elif label_dict[label]["type"] == "<=":

            if percentage <= percentage[label]["threshold"]:

                return label, percentage

        elif label_dict[label]["type"] == "<":

            if percentage < label_dict[label]["threshold"]:

                return label, percentage

        return None, None

    @staticmethod

    def check_tissue_percentage_over_threshold(label, label_dict, percentage):

        if label_dict[label]["type"] == "==":

            if label_dict[label]["threshold"] == percentage:

                return True

        elif label_dict[label]["type"] == ">=":

            if percentage >= label_dict[label]["threshold"]:

                return True

        elif label_dict[label]["type"] == ">":

            if percentage > label_dict[label]["threshold"]:

                return True

        elif label_dict[label]["type"] == "<=":

            if percentage <= percentage[label]["threshold"]:

                return True

        elif label_dict[label]["type"] == "<":

            if percentage < label_dict[label]["threshold"]:

                return True

        return False

    @staticmethod

    def get_unique_nonzero_entries(ndarray):

        return np.unique(ndarray[np.nonzero(ndarray)]).astype(int)

    def get_possible_labels(self, annotation_mask):

        if self.get_unique_nonzero_entries(annotation_mask).size >= 1:

            return self.get_unique_nonzero_entries(annotation_mask).tolist()

        else:

            return [0]  # completely unlabeled patch -> only non-tumor

    @staticmethod

    def is_non_tumor(label_ids):  # non-tumor tissue is unannotated tissue that's left after tissue detection

        return label_ids[0] == 0

    def calculate_label_percentages(self, label_ids, annotation_mask):

        if self.is_non_tumor(label_ids):

            label_percentages = [(np.count_nonzero(np.max(annotation_mask, axis=(-1)) == 0) /

                                  annotation_mask[:, :, 0].size)]

        else:

            label_percentages = []

            for label_id in label_ids:

                label_percentages.append((np.count_nonzero(annotation_mask[:, :, label_id] == label_id) /

                                          annotation_mask[:, :, label_id].size))

        return label_percentages

    def get_labels_with_enough_tissue_annotated(self, label_dict, annotation_mask):

        label_ids = self.get_possible_labels(annotation_mask)

        label_percentages = self.calculate_label_percentages(label_ids, annotation_mask)

        labels_with_enough_tissue_including_non_tumor = []

        for (label_id, label_percentage) in zip(label_ids, label_percentages):

            label = list(label_dict)[label_id]

            if self.check_tissue_percentage_over_threshold(label, label_dict, label_percentage):

                labels_with_enough_tissue_including_non_tumor.append(label)

        return labels_with_enough_tissue_including_non_tumor

    def update_overlapping_annotations_file(self, slide_name, verbose):

        with open(os.path.join(self.config["output_path"],

                               "overlapping_annotations_present_in_slides.json"), "r") as file:

            overlapping_annotations_present = json.load(file)

        if not overlapping_annotations_present[slide_name]:

            overlapping_annotations_present[slide_name] = True

            with (open(os.path.join(self.config["output_path"],

                                    "overlapping_annotations_present_in_slides.json"), "w")

                  as file):

                json.dump(overlapping_annotations_present, file, indent=4)

            if verbose:

                print(f"There are overlapping annotations in slide {slide_name}.")

    @staticmethod

    def normalize_to_tile_size_px(point, tile_size_px):

        if point < 0:

            return 0

        elif point >= tile_size_px:

            return tile_size_px - 1.0

        else:

            return point

    def translate_world_coordinates_to_tile_coordinates(self, point, tile_x, tile_y, tile_size_px):

        # the shrinkage of the coordinates to tile size is necessary as cv2.fillPoly only works if the annotation is

        # completely within the tile, so I set any points larger than the tile coordinates to the closest (valid)

        # tile coordinates

        return [self.normalize_to_tile_size_px(point[0] - tile_x, tile_size_px),

                self.normalize_to_tile_size_px(point[1] - tile_y, tile_size_px)]

    def extract_calibrated_patches(

            self,

            tile_dict,

            level,

            annotations,

            label_dict,

            overlap=0,

            annotation_overlap=0,

            slide_name=None,

            output_format="png",

    ):

        scaling_factor = int(self.slide.level_downsamples[level])

        patch_dict = {}

        patch_nb = 0

        for tile_key in tile_dict:

            tile_x = tile_dict[tile_key]["x"] * scaling_factor

            tile_y = tile_dict[tile_key]["y"] * scaling_factor

            tile_size_px = tile_dict[tile_key]["size"] * scaling_factor

            patch_size_px_x = int(np.round(self.config["calibration"]["patch_size_microns"] / self.res_x))

            patch_size_px_y = int(np.round(self.config["calibration"]["patch_size_microns"] / self.res_y))

            tile = np.array(self.slide.read_region((tile_x, tile_y), level=0, size=(tile_size_px, tile_size_px)))

            tile = tile[:, :, 0:3]

            if tile_dict[tile_key]["annotated"]:

                px_overlap_x = int(patch_size_px_x * annotation_overlap)

                px_overlap_y = int(patch_size_px_y * annotation_overlap)

            else:

                px_overlap_x = int(patch_size_px_x * overlap)

                px_overlap_y = int(patch_size_px_y * overlap)

            rows = int(np.ceil(tile_size_px / (patch_size_px_y - px_overlap_y)))

            cols = int(np.ceil(tile_size_px / (patch_size_px_x - px_overlap_x)))

            # create annotation mask

            if annotations is not None:

                # Translate from world coordinates to tile coordinates

                tile_annotation_list = [

                    [self.translate_world_coordinates_to_tile_coordinates(point, tile_x, tile_y, tile_size_px)

                     for point in annotations[polygon]["coordinates"]] for polygon in annotations]

                tile_annotation_list = list(zip(tile_annotation_list, [annotations[polygon]["tissue_type"]

                                                                       for polygon in annotations]))

                # Create mask from polygons

                tile_annotation_mask = np.zeros(shape=(tile_size_px, tile_size_px, len(self.config["label_dict"])))

                annotated_tissue_types = {}

                tissue_type_number = 1

                for tissue_type, tissue_details in label_dict.items():

                    if tissue_details["annotated"]:

                        annotated_tissue_types.update({tissue_type: tissue_type_number})

                        tissue_type_number += 1

                for polygon in tile_annotation_list:

                    # note: the casting to a contiguous array is due to OpenCV requiring C-order (row major) for

                    # implementation purposes, compare the answer by vvolhejn here

                    # https://stackoverflow.com/questions/23830618/python-opencv-typeerror-layout-of-the-output-array-incompatible-with-cvmat

                    # basically: many (all?) copy operations in numpy do this, ascontiguousarray is one of the more

                    # verbose ones

                    tile_annotation_mask[:, :, annotated_tissue_types[polygon[1]]] = (

                        cv2.fillPoly(np.ascontiguousarray(

                            tile_annotation_mask[:, :, annotated_tissue_types[polygon[1]]]),

                            [np.array(polygon[0]).astype(np.int32)], annotated_tissue_types[polygon[1]]))

            stop_y = False

            for row in range(rows):

                stop_x = False

                for col in range(cols):

                    # Calculate patch coordinates

                    patch_x = int(col * (patch_size_px_x - px_overlap_x))

                    patch_y = int(row * (patch_size_px_y - px_overlap_y))

                    if patch_y + patch_size_px_y >= tile_size_px:

                        stop_y = True

                        patch_y = tile_size_px - patch_size_px_y

                    if patch_x + patch_size_px_x >= tile_size_px:

                        stop_x = True

                        patch_x = tile_size_px - patch_size_px_x

                    global_x = patch_x + tile_x

                    global_y = patch_y + tile_y

                    patch = tile[patch_y: patch_y + patch_size_px_y, patch_x: patch_x + patch_size_px_x, :]

                    if np.sum(patch) == 0:

                        break

                    # check if the patch is annotated

                    annotated = False

                    if annotations is not None:

                        patch_mask = tile_annotation_mask[patch_y: patch_y + patch_size_px_y,

                                     patch_x: patch_x + patch_size_px_x, :]

                        labels = self.get_labels_with_enough_tissue_annotated(label_dict, patch_mask)

                        if labels is not None:

                            if len(labels) > 1:

                                self.update_overlapping_annotations_file(

                                    slide_name, verbose=self.config["overlapping_annotations_verbose"])

                            for label in labels:

                                # this check is done to ensure that non-tumor tissue (unannotated) is handled properly

                                if self.config["label_dict"][label]["annotated"]:

                                    annotated = True

                    else:

                        labels = "unlabeled"

                    if labels is not None:

                        if self.annotated_only and annotated or not self.annotated_only:

                            file_name = slide_name + "_" + str(global_x) + "_" + str(global_y) + "." + output_format

                            if self.config["calibration"]["resize"]:

                                patch = cv2.resize(patch, (self.config["patch_size"], self.config["patch_size"]))

                            patch = Image.fromarray(patch)

                            for label in labels:

                                patch.save(os.path.join(self.output_path, label, file_name), format=output_format)

                                patch_dict.update(

                                    {

                                        patch_nb: {

                                            "slide_name": slide_name,

                                            "patch_path": os.path.join(label, file_name),

                                            "label": label,

                                            "x_pos": global_x,

                                            "y_pos": global_y,

                                            "patch_size": patch_size_px_x,

                                            "resized": self.config["calibration"]["resize"],

                                        }

                                    }

                                )

                                patch_nb += 1

                    if stop_x:

                        break

                if stop_y:

                    break

        return patch_dict

    def make_dirs(self, output_path, slide_name, label_dict, annotated):

        try:

            slide_path = os.path.join(output_path, slide_name)

            if not os.path.exists(slide_path):

                os.makedirs(slide_path)

            if not annotated:

                unlabeled_path = os.path.join(slide_path, "unlabeled")

                if not os.path.exists(unlabeled_path):

                    os.makedirs(unlabeled_path)

            else:

                for label in label_dict:

                    sub_path = os.path.join(slide_path, label)

                    if not os.path.exists(sub_path):

                        os.makedirs(sub_path)

                    for patch in os.listdir(sub_path):

                        os.remove(os.path.join(sub_path, patch))

            self.output_path = slide_path

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "make_dirs")

    def extract_patches(

            self,

            tile_dict,

            level,

            annotations,

            label_dict,

            overlap=0,

            annotation_overlap=0,

            patch_size=256,

            slide_name=None,

            output_format="png",

    ):

        patch_dict = {}

        scaling_factor = int(self.slide.level_downsamples[level])

        patch_nb = 0

        for tile_key in tile_dict:

            # skip unannotated tiles in case only annotated patches should be saved

            if self.annotated_only and not tile_dict[tile_key]["annotated"]:

                pass

            else:

                # ToDo: rows and cols arent calculated correctly, instead a quick fix by using breaks was applied

                tile_x = tile_dict[tile_key]["x"] * scaling_factor

                tile_y = tile_dict[tile_key]["y"] * scaling_factor

                tile_size = tile_dict[tile_key]["size"] * scaling_factor

                tile = np.array(self.slide.read_region((tile_x, tile_y), level=0, size=(tile_size, tile_size)))

                tile = tile[:, :, 0:3]

                # overlap separately  for annotated and unannotated patches

                if tile_dict[tile_key]["annotated"]:

                    px_overlap = int(patch_size * annotation_overlap)

                    rows = int(np.ceil(tile_size / (patch_size - px_overlap)))

                    cols = int(np.ceil(tile_size / (patch_size - px_overlap)))

                else:

                    px_overlap = int(patch_size * overlap)

                    rows = int(np.ceil(tile_size / (patch_size - px_overlap)))

                    cols = int(np.ceil(tile_size / (patch_size - px_overlap)))

                # create annotation mask

                if annotations is not None:

                    # Translate from world coordinates to tile coordinates

                    tile_annotation_list = [

                        [self.translate_world_coordinates_to_tile_coordinates(point, tile_x, tile_y, tile_size)

                         for point in annotations[polygon]["coordinates"]] for polygon in annotations]

                    tile_annotation_list = list(zip(tile_annotation_list, [annotations[polygon]["tissue_type"]

                                                                           for polygon in annotations]))

                    # Create mask from polygons

                    tile_annotation_mask = np.zeros(shape=(tile_size, tile_size, len(self.config["label_dict"])))

                    annotated_tissue_types = {}

                    tissue_type_number = 1

                    for tissue_type, tissue_details in label_dict.items():

                        if tissue_details["annotated"]:

                            annotated_tissue_types.update({tissue_type: tissue_type_number})

                            tissue_type_number += 1

                    for polygon in tile_annotation_list:

                        # note: the casting to a contiguous array is due to OpenCV requiring C-order (row major) for

                        # implementation purposes, compare the answer by vvolhejn here

                        # https://stackoverflow.com/questions/23830618/python-opencv-typeerror-layout-of-the-output-array-incompatible-with-cvmat

                        # basically: many (all?) copy operations in numpy do this, ascontiguousarray is one of the more

                        # verbose ones

                        tile_annotation_mask[:, :, annotated_tissue_types[polygon[1]]] = (

                            cv2.fillPoly(np.ascontiguousarray(

                                tile_annotation_mask[:, :, annotated_tissue_types[polygon[1]]]),

                                [np.array(polygon[0]).astype(np.int32)], annotated_tissue_types[polygon[1]]))

                stop_y = False

                for row in range(rows):

                    stop_x = False

                    for col in range(cols):

                        # Calculate patch coordinates

                        patch_x = int(col * (patch_size - px_overlap))

                        patch_y = int(row * (patch_size - px_overlap))

                        if patch_y + patch_size >= tile_size:

                            stop_y = True

                            patch_y = tile_size - patch_size

                        if patch_x + patch_size >= tile_size:

                            stop_x = True

                            patch_x = tile_size - patch_size

                        global_x = patch_x + tile_x

                        global_y = patch_y + tile_y

                        patch = tile[patch_y: patch_y + patch_size, patch_x: patch_x + patch_size, :]

                        if np.sum(patch) == 0:

                            break

                        # check if the patch is annotated

                        annotated = False

                        if annotations is not None:

                            patch_mask = tile_annotation_mask[

                                         patch_y: patch_y + patch_size, patch_x: patch_x + patch_size

                                         ]

                            labels = self.get_labels_with_enough_tissue_annotated(label_dict, patch_mask)

                            if labels is not None:

                                if len(labels) > 1:

                                    self.update_overlapping_annotations_file(

                                        slide_name, verbose=self.config["overlapping_annotations_verbose"])

                                for label in labels:

                                    # this check is done to ensure that non-tumor tissue (unannotated) is handled

                                    # properly

                                    if self.config["label_dict"][label]["annotated"]:

                                        annotated = True

                        else:

                            labels = "unlabeled"

                        if labels is not None:

                            if self.annotated_only and annotated or not self.annotated_only:

                                if slide_name is not None:

                                    file_name = (

                                            slide_name + "_" + str(global_x) + "_" + str(global_y) + "." + output_format

                                    )

                                else:

                                    file_name = (

                                            str(patch_nb) + "_" + str(global_x) + "_" + str(global_y) + "." +

                                            output_format

                                    )

                                patch = Image.fromarray(patch)

                                for label in labels:

                                    patch.save(os.path.join(self.output_path, label, file_name), format=output_format)

                                    patch_dict.update(

                                        {

                                            patch_nb: {

                                                "slide_name": slide_name,

                                                "patch_path": os.path.join(label, file_name),

                                                "label": label,

                                                "x_pos": global_x,

                                                "y_pos": global_y,

                                                "patch_size": patch_size,

                                            }

                                        }

                                    )

                                    patch_nb += 1

                        if stop_x:

                            break

                    if stop_y:

                        break

        return patch_dict

    def export_dict(self, dictionary, metadata_format, filename):

        if metadata_format == "json":

            file = os.path.join(self.output_path, filename + ".json")

            with open(file, "w") as json_file:

                json.dump(dictionary, json_file, indent=4)

        elif metadata_format == "csv":

            df = pd.DataFrame(dictionary.values())

            file = os.path.join(self.output_path, filename + ".csv")

            df.to_csv(file, index=False)

        else:

            print("Could not write metadata. Metadata format has to be json or csv")

    def save_thumbnail(self, mask, slide_name, level, output_format="png"):

        remap_color = ((0, 0, 0), (255, 255, 255))

        process_level = level

        img = self.slide.read_region([0, 0], process_level, self.slide.level_dimensions[process_level])

        # Remove Alpha

        img = np.array(img)[:, :, 0:3]

        if remap_color is not None:

            indizes = np.all(img == remap_color[0], axis=2)

            img[indizes] = remap_color[1]

            copy_img = img[mask.astype(bool), :]

            median_filtered_img = cv2.medianBlur(img, 11)

            median_filtered_img[mask.astype(bool)] = copy_img

            img = median_filtered_img

        file_name = os.path.join(self.config["output_path"], slide_name, "thumbnail." + output_format)

        plt.imsave(file_name, img, format=output_format)

    def init_generic_tiff(self):

        unit_dict = {"milimeter": 1000, "centimeter": 10000, "meter": 1000000}

        self.scanner = "generic-tiff"

        assert self.slide.properties["tiff.ResolutionUnit"] in unit_dict.keys(), (

                "Unknown unit " + self.slide.properties["tiff.ResolutionUnit"]

        )

        factor = unit_dict[self.slide.properties["tiff.ResolutionUnit"]]

        # convert to mpp

        self.res_x = factor / float(self.slide.properties["tiff.XResolution"])

        self.res_y = factor / float(self.slide.properties["tiff.YResolution"])

    def init_aperio(self):

        self.scanner = "aperio"

        self.res_x = float(self.slide.properties["openslide.mpp-x"])

        self.res_y = float(self.slide.properties["openslide.mpp-y"])

    def init_mirax(self):

        self.scanner = "mirax"

        self.res_x = float(self.slide.properties["openslide.mpp-x"])

        self.res_y = float(self.slide.properties["openslide.mpp-y"])

    def init_unknown(self):

        try:

            self.scanner = self.slide.properties["openslide.vendor"]

            self.res_x = float(self.slide.properties["openslide.mpp-x"])

            self.res_y = float(self.slide.properties["openslide.mpp-y"])

        except Exception as e:

            print(e)

    def init_patch_calibration(self):

        # check scanner type

        if self.slide.properties["openslide.vendor"] == "aperio":

            self.init_aperio()

        elif self.slide.properties["openslide.vendor"] == "generic-tiff":

            self.init_generic_tiff()

        elif self.slide.properties["openslide.vendor"] == "mirax":

            self.init_mirax()

        else:

            self.init_unknown()

        # future vendors

        # elif ...

        assert self.scanner, "Not integrated scanner type, aborting"

    def process_slide(self, slide):

        slide_name = os.path.basename(slide)

        slide_name = os.path.splitext(slide_name)[0]

        print("Processing", slide_name, "process id is", os.getpid())

        try:

            annotation_path = os.path.join(

                self.config["annotation_dir"], slide_name + "." + self.config["annotation_file_format"]

            )

            if os.path.exists(annotation_path):

                annotated = True

                self.annotation_dict = self.load_annotation(annotation_path)

            else:

                annotated = False

                self.annotation_dict = None

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "process_slide - load_annotations")

            return 0

        self.make_dirs(

            output_path=self.config["output_path"],

            slide_name=slide_name,

            label_dict=self.config["label_dict"],

            annotated=annotated,

        )

        slide_path = os.path.join(self.config["slides_dir"], slide)

        try:

            level = self.load_slide(slide_path)

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "load_slide")

            return 0

        if self.config["calibration"]["use_non_pixel_lengths"]:

            try:

                self.init_patch_calibration()

            except Exception as e:

                self.print_and_log_slide_error(slide_name, e, "init_patch_calibration")

                return 0

        if self.config["use_tissue_detection"]:

            mask, level = self.apply_tissue_detection(level=level, show=self.config["show_mode"])

        else:

            mask = np.ones(shape=self.slide.level_dimensions[level]).transpose()

        try:

            tile_size = self.determine_tile_size(level)

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "determine_tile_size")

            return 0

        try:

            tile_dict = self.get_relevant_tiles(

                mask,

                tile_size=tile_size,

                min_coverage=self.config["tissue_coverage"],

                level=level,

                show=self.config["show_mode"],

            )

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "get_relevant_tiles")

            return 0

        # Calibrated or non calibrated patch sizes

        if self.config["calibration"]["use_non_pixel_lengths"]:

            try:

                patch_dict = self.extract_calibrated_patches(

                    tile_dict,

                    level,

                    self.annotation_dict,

                    self.config["label_dict"],

                    overlap=self.config["overlap"],

                    annotation_overlap=self.config["annotation_overlap"],

                    slide_name=slide_name,

                    output_format=self.config["output_format"]

                )

            except Exception as e:

                self.print_and_log_slide_error(slide_name, e, "extract_calibrated_patches")

                return 0

        else:

            try:

                patch_dict = self.extract_patches(

                    tile_dict,

                    level,

                    self.annotation_dict,

                    self.config["label_dict"],

                    overlap=self.config["overlap"],

                    annotation_overlap=self.config["annotation_overlap"],

                    patch_size=self.config["patch_size"],

                    slide_name=slide_name,

                    output_format=self.config["output_format"],

                )

            except Exception as e:

                self.print_and_log_slide_error(slide_name, e, "extract_patches")

                return 0

        self.export_dict(patch_dict, self.config["metadata_format"], "tile_information")

        try:

            self.save_thumbnail(mask, level=level, slide_name=slide_name,

                                output_format=self.config["output_format"])

            print("Finished slide ", slide_name)

        except Exception as e:

            self.print_and_log_slide_error(slide_name, e, "save_thumbnail")

            return 0

    @staticmethod

    def read_slide_file(slide_file_path, ext_list):

        slide_list = []

        with open(slide_file_path) as file:

            lines = file.read().splitlines()

        for line in lines:

            if os.path.isdir(line):

                for ext in ext_list:

                    for file in Path(line).resolve().glob("**/*" + ext):

                        slide = str(file)

            else:

                slide = line

            slide_list.append(slide)

        return slide_list

    @staticmethod

    def init(l):

        global lock

        lock = l

    @staticmethod

    def get_slide_name_from_slide_path(slide_path):

        return os.path.splitext(os.path.basename(slide_path))[0]

    def slides2patches(self):

        l = multiprocessing.Lock()

        extensions = [".tif", ".svs", ".mrxs"]

        slide_list = []

        if self.config["slides_file"] is not None:

            print("Using slide file: " + self.config["slides_file"])

            slide_list = self.read_slide_file(self.config["slides_file"], extensions)

        else:

            for extension in extensions:

                for file in Path(self.config["slides_dir"]).resolve().glob("**/*" + extension):

                    slide_list.append(file)

        self.annotation_list = []

        if os.path.exists(self.config["annotation_dir"]):

            annotation_list = os.listdir(self.config["annotation_dir"])

            self.annotation_list = [os.path.splitext(annotation)[0] for annotation in annotation_list]

        missing_annotations = []

        annotated_slides = [

            name

            if os.path.splitext(os.path.basename(name))[0] in self.annotation_list

            else missing_annotations.append(os.path.splitext(os.path.basename(name))[0])

            for name in slide_list

        ]

        annotated_slides = list(filter(lambda slide: True if slide is not None else False, annotated_slides))

        print("###############################################")

        print("Found", len(annotated_slides), "annotated slides")

        print("###############################################")

        print("Found", len(missing_annotations), "unannotated slides")

        print("###############################################")

        if not self.config["use_tissue_detection"]:

            print("Tissue detection deactivated")

            print("###############################################")

        if self.config["skip_unlabeled_slides"]:

            slide_list = annotated_slides

            print("Processing annotated slides only")

        if not os.path.exists(self.config["output_path"]):

            os.makedirs(self.config["output_path"])

        with open(os.path.join(self.config["output_path"], "error_log.txt"), "w"):

            pass

        if not len(slide_list) == 0:

            slide_list = sorted(slide_list)

            # writing this to file to work somewhat elegantly around multiprocessing without much restructuring of the

            # existing code

            with (open(os.path.join(self.config["output_path"], "overlapping_annotations_present_in_slides.json"), "w")

                  as file):

                json.dump(dict.fromkeys(list(map(self.get_slide_name_from_slide_path, slide_list)), False),

                          file, indent=4)

            if _MULTIPROCESS:

                available_threads = multiprocessing.cpu_count() - self.config["blocked_threads"]

                pool = multiprocessing.Pool(processes=available_threads, initializer=self.init, initargs=(l,))

                pool.map(self.process_slide, slide_list)

            else:

                for slide in slide_list:

                    self.process_slide(slide)

            slide_dict = {}

            if len(annotated_slides) == 0:

                for i in range(len(slide_list)):

                    slide = slide_list[i]

                    slide_name = self.get_slide_name_from_slide_path(slide)

                    slide_dict.update({i: {"slide_name": slide_name,

                                           "slide_path": slide,

                                           }

                                       })

                self.output_path = self.config["output_path"]

                self.export_dict(slide_dict, self.config["metadata_format"], "slide_information")

            else:

                # Save label proportion per slide

                labels = list(self.config["label_dict"].keys())

                with (open(os.path.join(self.config["output_path"],

                                        "overlapping_annotations_present_in_slides.json"), "r") as file):

                    overlapping_annotations_present_in_slide = json.load(file)

                for i in range(len(annotated_slides)):

                    slide = slide_list[i]

                    slide_name = self.get_slide_name_from_slide_path(slide)

                    slide_path = os.path.join(self.config["output_path"], slide_name)

                    n_labeled_tiles = 0

                    n_labels = {}

                    for label in labels:

                        n_label = len(os.listdir(os.path.join(slide_path, label)))

                        n_labels.update({label: n_label})

                        n_labeled_tiles += n_label

                    slide_dict_entry = {}

                    slide_dict_entry.update({"slide_name": slide_name,

                                             "slide_contains_overlapping_annotations":

                                                 overlapping_annotations_present_in_slide[slide_name]})

                    fracs = {}

                    for label, n_label in n_labels.items():

                        slide_dict_entry.update({label: n_label})

                        fracs.update({label: n_label / n_labeled_tiles * 100})

                    slide_dict_entry.update({"total": n_labeled_tiles, "frac": fracs})

                    slide_dict.update({i: slide_dict_entry})

                    self.output_path = self.config["output_path"]

                    self.export_dict(slide_dict, self.config["metadata_format"], "slide_information")

            # Save used config file

            file = os.path.join(self.config["output_path"], "config.json")

            with open(file, "w") as json_file:

                json.dump(self.config, json_file, indent=4)

            print("Finished tiling process!")

        else:

            print("###############################################")

            print("WARNING: No slides processed!")

            print("###############################################")

if __name__ == "__main__":

    parser = ArgumentParser()

    parser.add_argument("--config_path", default=script_dir + "/resources/config.json")

    args = parser.parse_args()

    slide_handler = WSIHandler(config_path=args.config_path)

    slide_handler.slides2patches()