#!/usr/bin/python

# get opencv dependency if needed. We do it here for imageDiff 
# because we spawn multiple processes so we would have a race condition with each one trying to check and download opencv
import subprocess
import os.path
import pathlib
import sys
from collections import defaultdict

if not "NO_VENV" in os.environ.keys():
    from venv import create
    # create venv here and then install package
    VENV_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".pyenv"))
    if sys.platform.startswith('win32'):
        PYTHON = os.path.join(VENV_DIR, "Scripts", "python.exe")
    else:
        PYTHON = os.path.join(VENV_DIR, "bin", "python")

    if not os.path.exists(VENV_DIR):
        create(VENV_DIR, with_pip=True)
        subprocess.check_call([PYTHON, "-m", "pip", "install", "opencv-python-headless"])
else:
    PYTHON = os.path.realpath(sys.executable)

TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), "template")

from genericpath import exists
import argparse
import glob
import csv
import threading
import queue
from xml.etree import ElementTree as ET
from typing import TypeVar
import shutil
import json

parser = argparse.ArgumentParser(description="Compare two directories of images")
parser.add_argument("--goldens", "-g", required=True, help="INPUT directory of correct images")
parser.add_argument("--candidates", "-c", required=True, help="INPUT directory of candidate images")
parser.add_argument("--output", "-o", required=True, help="OUTPUT directory to store differences")
parser.add_argument("--verbose", "-v", action='store_true', help="enable verbose output")
parser.add_argument("--build", "-b", default='release', choices=['debug', 'release'], help="build configuration")
parser.add_argument("-j", "--jobs", default=1, type=int, help="number of jobs to run in parallel")
parser.add_argument("-r", "--recursive", action='store_true', help="recursively diffs images in \"--candidates\" sub folders against \"--goldens\"")
parser.add_argument("-p", "--pack", action='store_true', help="copy candidates and goldens into output folder along with results")
parser.add_argument("-H", "--histogram_compare", action='store_true', help="Use histogram compare method to determine if candidate matches gold")
parser.add_argument("-t", "--threshold", default=0.06, type=float, help="if histogram_compare is set, then threshold used for histogram pass result otherwise the threshold for pixel diff pass result")

clean_mode = parser.add_mutually_exclusive_group(required=False)
clean_mode.add_argument("-x", "--clean", action='store_true', help="delete golden and candidate images that are identical, also dont add identical images to index.html")
clean_mode.add_argument("-f", "--fails_only", action='store_true', help="delete images of all tests except for fails, also only adds failing tests to index.html, acts the same as -x if histogram_compare is false")

args = parser.parse_args()

status_filename_base = "_imagediff_status"
status_filename_pattern = f"{status_filename_base}_%i_*.txt" % os.getpid()
show_commands = False

class TestEntry(object):
    pass_entry_template:str = None
    error_entry_template:str = None
    identical_entry_template:str = None
    missing_file_entry_template:str = None

    @classmethod
    def load_templates(cls, path):
        with open(os.path.join(path, "error_entry.html")) as t:
            cls.error_entry_template = t.read()
        with open(os.path.join(path, "pass_entry.html")) as t:
            cls.pass_entry_template = t.read()
        with open(os.path.join(path, "identical_entry.html")) as t:
            cls.identical_entry_template = t.read()
        with open(os.path.join(path, "missing_file_entry.html")) as t:
            cls.missing_file_entry_template = t.read()

    def __init__(self, words, candidates_path, golden_path, output_path, device_name=None, browserstack_details=None):
        self.diff0_path_abs = None
        self.diff1_path_abs = None
        self.device = device_name
        self.browserstack_details = browserstack_details
        self.name = words[0]
        self.candidates_path_abs = os.path.join(candidates_path, f"{self.name}.png")
        if args.pack and device_name is not None:
            self.candidates_path = os.path.join(device_name, f"{self.name}.png")
            self.golden_path = os.path.join("golden", f"{self.name}.png")
        elif args.recursive:
            self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"),  pathlib.Path(output_path).parent.absolute())).as_posix()
            self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), pathlib.Path(output_path).parent.absolute())).as_posix()
        else:
            self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"), output_path)).as_posix()
            self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), output_path)).as_posix()

        if len(words) == 2:
            self.avg = None
            self.histogram = None
            self.type = words[1]
        else:    
            self.max_diff = int(words[1])
            self.avg = float(words[2])
            self.total_diff_count = int(words[3])
            self.total_pixels = int(words[4])
            self.diff0_path_abs = os.path.join(output_path, f"{self.name}.diff0.png")
            self.diff1_path_abs = os.path.join(output_path, f"{self.name}.diff1.png")
            if device_name is not None:
                self.diff0_path = os.path.join(device_name, f"{self.name}.diff0.png")
                self.diff1_path = os.path.join(device_name, f"{self.name}.diff1.png")
            else:
                self.diff0_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff0.png"), output_path)
                self.diff1_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff1.png"), output_path)
            if len(words) == 6:
                self.histogram = float(words[5])
                if self.histogram < (1.0-args.threshold):
                    self.type = "failed"
                else:
                    self.type = "pass"
            else:
                self.histogram = None
                if self.max_diff > args.threshold:
                    self.type = "failed"
                else:
                    self.type = "pass"

    # this is equivalent of implementing == we are comparing by name for when we check against the correct golds to delete
    def __eq__(self, other):
        return self.name == other.name

    # hash by name so that when we create a set out of entry list we condense it down to the number of goldens since we only care about them
    def __hash__(self):
        return hash(self.name)

    # this is equivalent of implementing < operator. We use this for sorted and sort functions
    def __lt__(self, other):
        # Always sort by avg first. Histogram is a good heuristic to divide into
        # "pass/fail" buckets, but it's helpful to then see the fail bucked
        # sorted by avg, which is more sensitive to differences.
        if (self.avg == other.avg and
            self.histogram is not None and
            other.histogram is not None):
            # LOWER histogram values mean worse matches. Sort the bad matches first.
            return self.histogram > other.histogram
        else:
            # HIGHER avg values mean worse matches. Sort the bad matches first.
            return self.avg < other.avg

    def __str__(self):
        vals = dict()
        vals['name'] = f"{self.name} ({self.device})" if self.device is not None else self.name
        vals['url'] = self.browserstack_details['browser_url'] if self.browserstack_details is not None else ' '
        if self.type == 'missing_golden':
            # show candidate, since golden is missing
            vals['image'] = self.candidates_path
            return self.missing_file_entry_template.format_map(vals)
        
        elif self.type == 'missing_candidate':
            # show golden, since candidate is missing
            vals['image'] = self.golden_path
            return self.missing_file_entry_template.format_map(vals)

        vals['golden'] = self.golden_path
        vals['candidate'] = self.candidates_path
        
        if self.type == "pass" or self.type == "failed":
            vals['max'] = self.max_diff
            vals['avg'] = self.avg
            vals['total_diff'] = self.total_diff_count
            vals['percent'] = float(self.total_diff_count) / float(self.total_pixels)
            vals['histogram'] = self.histogram if self.histogram is not None else 'None'
            vals['diff0'] = self.diff0_path
            vals['diff1'] = self.diff1_path

            if self.type == 'pass':
                return self.pass_entry_template.format_map(vals)
            else:
                return self.error_entry_template.format_map(vals)

        if self.type == 'identical':
            return self.identical_entry_template.format_map(vals)
        
        return ''

    def clean(self):
        if args.verbose:
            print(f"cleaning TestEntry {self.name} - {self.device}")
        # removes image files to save space, this is done for clean and fails only arguments
        os.remove(self.candidates_path_abs)
        # if we are packing the files, also delete the packed ones
        if args.pack:
            os.remove(os.path.join(args.output, self.candidates_path))
        # if we have diff files, delete those too
        if self.diff0_path_abs is not None:
            os.remove(self.diff0_path_abs)
        # this should always be true if diff0 is not none but just to be safe we do a separate check
        if self.diff1_path_abs is not None:
            os.remove(self.diff1_path_abs)


    @property
    def success(self):
        return self.type == "pass" or self.type == "identical"
    
    @property
    def csv_dict(self):
        val = dict()

        val['file_name'] = self.name
        val['original'] = self.golden_path
        val['candidate'] = self.candidates_path
        if self.type == "pass" or self.type == "failed":
            val['max_rgb'] = str(self.max_diff)
            val['avg_rgb'] = str(self.avg)
            val['pixel_diff_count'] = str(self.total_diff_count)
            val['pixel_diff_percent'] = '100'
            if self.histogram is not None:
                val['hist_result'] = str(self.histogram)
            val['color_diff'] = self.diff0_path
            val['pixel_diff'] = self.diff1_path
        elif self.type == 'identical':
            val['max_rgb'] = '0'
            val['avg_rgb'] = '0'
            val['pixel_diff_count'] = '0'
            val['pixel_diff_percent'] = '100'
            if args.histogram_compare:
                val['hist_result'] = '1.0'
            val['color_diff'] = ''
            val['pixel_diff'] = ''

        return val
    

def shallow_copy_images(src, dest):
    file_names = [file for file in os.scandir(src) if file.is_file() and '.png' in file.name]
    for file in file_names:
        shutil.copyfile(file.path, os.path.join(dest, file.name))

def write_csv(entries, origpath, candidatepath, diffpath, missing_candidates):
    origpath = os.path.relpath(origpath, diffpath)
    candidatepath = os.path.relpath(candidatepath, diffpath)

    height = 256

    with open(os.path.join(diffpath, "data.csv"), "w") as csvfile:
        fieldnames = ['file_name','original', 'candidate',  'max_rgb', 'avg_rgb', 'pixel_diff_count', 'pixel_diff_percent', 'color_diff', 'pixel_diff']
        if args.histogram_compare:
            fieldnames.extend(['hist_result'])
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        for name in missing_candidates:
            if args.histogram_compare:
                writer.writerow({
                    'file_name': name.split('.')[0],
                    'original': os.path.join(origpath, name), 
                    'candidate': '',
                    'max_rgb':255,
                    'avg_rgb':255,
                    # i guess its all of em, not 1 of em, but whatever
                    'pixel_diff_count': 1,
                    'pixel_diff_percent': '100',
                    'hist_result' : 1.0,
                    'color_diff': '',
                    'pixel_diff': ''
                })
            else:
                writer.writerow({
                    'file_name': name.split('.')[0],
                    'original': os.path.join(origpath, name), 
                    'candidate': '',
                    'max_rgb':255,
                    'avg_rgb':255,
                    # i guess its all of em, not 1 of em, but whatever
                    'pixel_diff_count': 1,
                    'pixel_diff_percent': '100',
                    'color_diff': '',
                    'pixel_diff': ''
                })

        for entry in entries:
            writer.writerow(entry.csv_dict)
            

def write_min_csv(total_passing, total_failing, total_missing_candidates, total_missing_goldens, total_identical, total_entries, csv_path):
    # delete and old data
    if os.path.exists(csv_path):
        os.remove(csv_path)

    with open(csv_path, 'w', newline='') as csv_file:
        csv_writer = csv.DictWriter(csv_file, fieldnames=['type', 'number'])
        csv_writer.writerow({'type':'failed', 'number' : str(total_failing)})
        csv_writer.writerow({'type':'pass', 'number' : str(total_passing)})
        csv_writer.writerow({'type':'missing_candidates', 'number' : str(total_missing_candidates)})
        csv_writer.writerow({'type':'missing_goldens', 'number' : str(total_missing_goldens)})
        csv_writer.writerow({'type':'identical', 'number' : str(total_identical)})
        csv_writer.writerow({'type':'total', 'number' : str(total_entries)})

def diff_worker(index, work_queue, golden, candidate, output, parent_pid):
    # One long-lived image_diff.py process that pulls filenames off the shared
    # work_queue one at a time. Keeping the process alive means cv2 is imported
    # only once (that import + process spawn dominate on Windows).
    cmd = [PYTHON, "image_diff.py", "--serve",
           "-g", golden,
           "-c", candidate,
           # Each process writes its own status file in order to avoid race conditions.
           "-s", "%s_%i_%i.txt" % (status_filename_base, parent_pid, index)]
    if output is not None:
        cmd.extend(["-o", output])
    if args.verbose:
        cmd.extend(["-v", "-l"])
    if args.histogram_compare:
        cmd.extend(["-H"])

    if show_commands:
        print(" ".join(cmd))

    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                            text=True)
    try:
        while True:
            try:
                filename = work_queue.get_nowait()
            except queue.Empty:
                break
            proc.stdin.write(filename + "\n")
            proc.stdin.flush()
            # Block until the worker signals it finished this image. If the pipe
            # closed early the worker died; stop feeding it (the missing status
            # line is caught by the line-count check in diff_directory_shallow).
            if proc.stdout.readline().strip() != "done":
                print("image_diff.py worker %i died processing %s" % (index, filename))
                break
    finally:
        proc.stdin.close()
        proc.wait()

def parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details):
    total_lines = 0
    test_entries = []
    success = True
    status_files = glob.glob(status_filename_pattern)
    if not status_files:
        print('Not a single status file got written, are you just starting new?')
    for status_filename in status_files:
        for line in open(status_filename, "r").readlines():
            total_lines += 1
            words = line.rstrip().split('\t')
            entry = TestEntry(words, candidates_path, golden_path, output_path, device_name, browserstack_details)
            test_entries.append(entry)
            if not entry.success:
                success = False

    return (total_lines, test_entries, success)

def diff_directory_shallow(candidates_path, output_path, golden_path, device_name=None, browserstack_details=None):
    original_filenames = set((file.name for file in os.scandir(candidates_path)
                              if file.is_file() and file.name.endswith('.png')))
    candidate_filenames = set(os.listdir(golden_path))
    intersect_filenames = original_filenames.intersection(candidate_filenames)

    missing = []
    for file in original_filenames.difference(candidate_filenames):
        print(f'Candidate file {file} missing in goldens.')
        missing.append(TestEntry([file.split('.')[0], 'missing_golden'], candidates_path, golden_path, output_path, device_name, browserstack_details))

    for file in candidate_filenames.difference(original_filenames):
        print(f'Golden file {file} missing in candidates.')
        missing.append(TestEntry([file.split('.')[0], 'missing_candidate'], candidates_path, golden_path, output_path, device_name, browserstack_details))

    if args.jobs > 1:
        print("Diffing %i candidates in %i processes..." % (len(intersect_filenames), args.jobs))
    else:
        print("Diffing %i candidates..." % len(intersect_filenames))
    sys.stdout.flush()

    # Fill a shared queue with every filename.
    work_queue = queue.Queue()
    for filename in intersect_filenames:
        work_queue.put(filename)

    # Generate the diffs (if any) and write to the status file(s). Start up to
    # `jobs` long-lived image_diff.py workers that each pull filenames off the
    # queue as fast as they can finish them.
    njobs = min(args.jobs, len(intersect_filenames))
    parent_pid = os.getpid()
    threads = []
    for index in range(njobs):
        t = threading.Thread(target=diff_worker,
                             args=(index, work_queue, golden_path,
                                   candidates_path, output_path, parent_pid))
        t.start()
        threads.append(t)
    for t in threads:
        t.join()

    (total_lines, entries, success) = parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details)
    
    entries.extend(missing)

    print(f'finished with Succes:{success} and {total_lines} lines')

    if total_lines != len(intersect_filenames):
        print(f"Internal failure: Got {total_lines} status lines. Expected {len(intersect_filenames)}.")
        success = False

    if original_filenames.symmetric_difference(candidate_filenames):
        print("golden and candidate directories do not have identical files.")
        success = False

#   cleanup our scratch files
    for status_filename in glob.iglob(status_filename_pattern):
        os.remove(status_filename)

    return (entries, missing, success)

# Sort descending total failure count (missing + failure), then by descending failure count, then by name
# (put the failed ones at the top)
def device_entry_sort_key(kv): 
    return (-(kv[1]["missing_candidate"] + kv[1]["failed"]), -(kv[1]["failed"]), kv[0])

# returns entries sorted into identical, passing and failing as well as html str list of each
# based on arguments passed, we may or may not return all of the string lists, but we always return the object lists
def sort_entries(entries):
    # we dont need an intermediate object list because we never sort ot clean these, so direct to html
    missing_golden_str = [str(entry) for entry in entries if entry.type == "missing_golden"]
    missing_candidate_str = [str(entry) for entry in entries if entry.type == "missing_candidate"]

    failed_entires = [entry for entry in entries if entry.type == "failed"]
    pass_entires = [entry for entry in entries if entry.type == "pass"]
    identical_entires = [entry for entry in entries if entry.type == "identical"]

    sorted_failed_entires = sorted(failed_entires, reverse=True)

    sorted_failed_str = [str(entry) for entry in sorted_failed_entires]

    # Build a list of stat counts by device name
    all_device_stats = dict()
    for entry in entries:
        if entry.device not in all_device_stats:
            all_device_stats[entry.device] = defaultdict(int)
            all_device_stats[entry.device]["url"] = entry.browserstack_details['browser_url'] if entry.browserstack_details is not None else ' '
        all_device_stats[entry.device][entry.type] += 1

    # Now build the device summary text using the template
    device_summary_str = ""
    with open(os.path.join(TEMPLATE_PATH, "device_summary_entry.html")) as t:
        device_summary_entry_template = t.read()

        for device_name, device_summary in sorted(all_device_stats.items(), key=device_entry_sort_key):
            none_succeeded = device_summary["pass"] == 0 and device_summary["identical"] == 0
            any_failed = device_summary["failed"] > 0
            any_missing = device_summary["missing_candidate"] > 0
            device_summary_str += device_summary_entry_template.format(
                name=device_name, 
                url=device_summary["url"],
                failed_count=device_summary["failed"] if any_failed else "-",
                failed_class="fail" if any_failed else "deemphasize",
                missing_count=device_summary["missing_candidate"] if any_missing else "-",
                missing_class="fail" if any_missing else "deemphasize",
                pass_count=device_summary["pass"],
                pass_class="fail" if none_succeeded else "success",
                identical_count=device_summary["identical"],
                identical_class="fail" if none_succeeded else "" if device_summary["identical"] == 0 else "success")

    # if we are only doing fails then only sort those and return empty html lists for "pass" and "identical" we still build and return
    # identical and pass object lists for cleaning, but we dont bother sorting them
    if args.fails_only:
        return (
            sorted_failed_entires,
            pass_entires,
            identical_entires,
            sorted_failed_str,
            [],
            [],
            missing_golden_str,
            missing_candidate_str,
            device_summary_str,
            len(all_device_stats))

    
    # now sort passed entires and build the html list
    sorted_passed_entires = sorted(pass_entires, reverse=True)
    sorted_passed_str = [str(entry) for entry in sorted_passed_entires]

    # if we are cleaning then return empty html list for identical. do everything else the same
    if args.clean:
         return (sorted_failed_entires, sorted_passed_entires, identical_entires, sorted_failed_str, sorted_passed_str, [], missing_golden_str, missing_candidate_str, device_summary_str, len(all_device_stats))

    # otherwise build identical html entry list and include it in the return
    identical_str = [str(entry) for entry in identical_entires]

    return (
        sorted_failed_entires,
        sorted_passed_entires,
        identical_entires,
        sorted_failed_str,
        sorted_passed_str,
        identical_str,
        missing_golden_str,
        missing_candidate_str,
        device_summary_str,
        len(all_device_stats))

def write_html(templates_path, failed_entries, passing_entries, identical_entries, missing_golden_entries, missing_candidate_entries, device_summary_str, device_number, output_path):
    with open(os.path.join(templates_path, "index.html")) as t:
        index_template = t.read()
    
    html = index_template.format(identical=" ".join(identical_entries), passing=" ".join(passing_entries),
                                 failed=" ".join(failed_entries), failed_number=len(failed_entries),
                                 passing_number=len(passing_entries), identical_number=len(identical_entries),
                                 missing_candidate=" ".join(missing_candidate_entries), missing_candidate_number=len(missing_candidate_entries),
                                 missing_golden=" ".join(missing_golden_entries), missing_golden_number=len(missing_golden_entries),
                                 device_summaries=device_summary_str, device_number=device_number,
                                 pass_hide_class="hidden" if args.fails_only else "",
                                 identical_hide_class="hidden" if args.fails_only or args.clean else "")

    with open(os.path.join(output_path, "index.html"), "w") as file:
        file.write(html)
    
    #copy our icon to the output folder
    shutil.copyfile(os.path.join(TEMPLATE_PATH, "favicon.ico"), os.path.join(output_path, "favicon.ico"))

def diff_directory_deep(candidates_path, output_path):
    golden_path = args.goldens
    if args.pack:
        new_golden_path = os.path.join(output_path, "golden")
        os.makedirs(new_golden_path, exist_ok=True)
        shallow_copy_images(args.goldens, new_golden_path)
        golden_path = new_golden_path
    
    all_entries = []

    for folder in os.scandir(candidates_path):
        if folder.is_dir():
            if folder.name[0] == '.':
                continue
            output = os.path.join(output_path ,folder.name)
            os.makedirs(output, exist_ok=True)
            browserstack_details = None
            browserstack_details_path = os.path.join(folder.path, "session_details")
            if os.path.exists(browserstack_details_path):
                with open(browserstack_details_path, 'rt') as file:
                    browserstack_details = json.load(file)
                os.remove(browserstack_details_path)
            
            (entries, _, _) = diff_directory_shallow(folder.path, output, golden_path, folder.name, browserstack_details)
            
            all_entries.extend(entries)

            if args.pack:
                shallow_copy_images(folder.path, output)

    (failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number) = sort_entries(all_entries)

    to_clean = []
    to_check = []
    # choose who to clean and who to check against
    if args.clean:
        to_clean = identical
        to_check = failed + passed

    if args.fails_only:
        to_clean = identical + passed
        to_check = failed

    # clean them
    for obj in to_clean:
        obj.clean()

    # only remove goldens that are in to_clean and not to_check so that we keep goldens that are still used
    # this part is why we needed __eq__ and __hash__ in our TestEntry class
    for obj in set(to_clean) - set(to_check):
        golden_file = os.path.join(golden_path, f"{obj.name}.png")
        if args.verbose:
            print("deleting orphaned golden " + golden_file)
        os.remove(golden_file)
        if args.pack:
            # remember to remove the original if we packed it
            os.remove(os.path.join(args.goldens, f"{obj.name}.png"))


    write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number, output_path)

    print(f"total entries {len(all_entries)}")
    write_min_csv(len(passed), len(failed), len(missing_candidate_str), len(missing_golden_str), len(identical), len(all_entries), output_path + "/issues.csv")

def main(argv=None):    
    if not os.path.exists(args.goldens):
        print("Can't find goldens " + args.goldens)
        return -1
    if not os.path.exists(args.candidates):
        print("Can't find candidates " + args.candidates)
        return -1

    # delete output dir if exists
    shutil.rmtree(args.output, ignore_errors=True)
    # remake output dir, this will make it correctly
    # even if it requires creating mulltiple directories
    os.makedirs(args.output, exist_ok=True)

#   reset our scratch files
    for status_filename in glob.iglob(status_filename_pattern):
        os.remove(status_filename)

    TestEntry.load_templates(TEMPLATE_PATH)

    if args.recursive:
        diff_directory_deep(args.candidates, args.output)
    else:
        (entries, missing, success) = diff_directory_shallow(args.candidates, args.output, args.goldens)
        if len(entries) > 0:
            (failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number) = sort_entries(entries)
            assert(len(failed) + len(passed) + len(identical) + len(missing_candidate_str) + len(missing_golden_str) == len(entries))
            write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number, args.output)
            # note could add these to the html output but w/e
            missing_candidates = [os.path.basename(entry.candidates_path_abs) for entry in missing if entry.type == 'missing_candidate']
            write_csv(entries, args.goldens, args.candidates, args.output, missing_candidates)
            print("Found", len(entries) - len(identical), "differences,",
                  len(failed), "failing.")

            # here we have to do a lot less work than the diff_directory_deep since we know goldens are not shared with other TestEntries
            if args.fails_only:
                for obj in identical+passed:
                    obj.clean()
                    golden_path = os.path.join(args.goldens, f"{obj.name}.png")
                    if args.verbose:
                        print(f"deleting orphaned golden {golden_path}")
                    os.remove(golden_path)
                    
            elif args.clean:
                for obj in identical:
                    obj.clean()
                    golden_path = os.path.join(args.goldens, f"{obj.name}.png")
                    if args.verbose:
                        print(f"deleting orphaned golden {golden_path}")
                    os.remove(golden_path)
                    
            
        # if we are in fail only mode than make it succesful when there are only "passing" entries
        if args.fails_only:
            if failed:
                # if there were diffs, its gotta fail
                print("FAILED.")
                return -1
        # otherwise fail like normal
        elif not success:
            # if there were diffs, its gotta fail
            print("FAILED.")
            return -1
        
    return 0
    
if __name__ == "__main__":
    sys.exit(main())