| #!/usr/bin/python | |
| # get opencv dependency if needed. We do it here for imageDiff | |
| # because we spawn multiple processes so we would have a race condition with each one trying to check and download opencv | |
| import subprocess | |
| import os.path | |
| import pathlib | |
| import sys | |
| from collections import defaultdict | |
| if not "NO_VENV" in os.environ.keys(): | |
| from venv import create | |
| # create venv here and then install package | |
| VENV_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), ".pyenv")) | |
| if sys.platform.startswith('win32'): | |
| PYTHON = os.path.join(VENV_DIR, "Scripts", "python.exe") | |
| else: | |
| PYTHON = os.path.join(VENV_DIR, "bin", "python") | |
| if not os.path.exists(VENV_DIR): | |
| create(VENV_DIR, with_pip=True) | |
| subprocess.check_call([PYTHON, "-m", "pip", "install", "opencv-python-headless"]) | |
| else: | |
| PYTHON = os.path.realpath(sys.executable) | |
| TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), "template") | |
| from genericpath import exists | |
| import argparse | |
| import glob | |
| import csv | |
| import threading | |
| import queue | |
| from xml.etree import ElementTree as ET | |
| from typing import TypeVar | |
| import shutil | |
| import json | |
| parser = argparse.ArgumentParser(description="Compare two directories of images") | |
| parser.add_argument("--goldens", "-g", required=True, help="INPUT directory of correct images") | |
| parser.add_argument("--candidates", "-c", required=True, help="INPUT directory of candidate images") | |
| parser.add_argument("--output", "-o", required=True, help="OUTPUT directory to store differences") | |
| parser.add_argument("--verbose", "-v", action='store_true', help="enable verbose output") | |
| parser.add_argument("--build", "-b", default='release', choices=['debug', 'release'], help="build configuration") | |
| parser.add_argument("-j", "--jobs", default=1, type=int, help="number of jobs to run in parallel") | |
| parser.add_argument("-r", "--recursive", action='store_true', help="recursively diffs images in \"--candidates\" sub folders against \"--goldens\"") | |
| parser.add_argument("-p", "--pack", action='store_true', help="copy candidates and goldens into output folder along with results") | |
| parser.add_argument("-H", "--histogram_compare", action='store_true', help="Use histogram compare method to determine if candidate matches gold") | |
| parser.add_argument("-t", "--threshold", default=0.06, type=float, help="if histogram_compare is set, then threshold used for histogram pass result otherwise the threshold for pixel diff pass result") | |
| clean_mode = parser.add_mutually_exclusive_group(required=False) | |
| clean_mode.add_argument("-x", "--clean", action='store_true', help="delete golden and candidate images that are identical, also dont add identical images to index.html") | |
| clean_mode.add_argument("-f", "--fails_only", action='store_true', help="delete images of all tests except for fails, also only adds failing tests to index.html, acts the same as -x if histogram_compare is false") | |
| args = parser.parse_args() | |
| status_filename_base = "_imagediff_status" | |
| status_filename_pattern = f"{status_filename_base}_%i_*.txt" % os.getpid() | |
| show_commands = False | |
| class TestEntry(object): | |
| pass_entry_template:str = None | |
| error_entry_template:str = None | |
| identical_entry_template:str = None | |
| missing_file_entry_template:str = None | |
| @classmethod | |
| def load_templates(cls, path): | |
| with open(os.path.join(path, "error_entry.html")) as t: | |
| cls.error_entry_template = t.read() | |
| with open(os.path.join(path, "pass_entry.html")) as t: | |
| cls.pass_entry_template = t.read() | |
| with open(os.path.join(path, "identical_entry.html")) as t: | |
| cls.identical_entry_template = t.read() | |
| with open(os.path.join(path, "missing_file_entry.html")) as t: | |
| cls.missing_file_entry_template = t.read() | |
| def __init__(self, words, candidates_path, golden_path, output_path, device_name=None, browserstack_details=None): | |
| self.diff0_path_abs = None | |
| self.diff1_path_abs = None | |
| self.device = device_name | |
| self.browserstack_details = browserstack_details | |
| self.name = words[0] | |
| self.candidates_path_abs = os.path.join(candidates_path, f"{self.name}.png") | |
| if args.pack and device_name is not None: | |
| self.candidates_path = os.path.join(device_name, f"{self.name}.png") | |
| self.golden_path = os.path.join("golden", f"{self.name}.png") | |
| elif args.recursive: | |
| self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"), pathlib.Path(output_path).parent.absolute())).as_posix() | |
| self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), pathlib.Path(output_path).parent.absolute())).as_posix() | |
| else: | |
| self.candidates_path = pathlib.Path(os.path.relpath(os.path.join(candidates_path, f"{self.name}.png"), output_path)).as_posix() | |
| self.golden_path = pathlib.Path(os.path.relpath(os.path.join(golden_path, f"{self.name}.png"), output_path)).as_posix() | |
| # image_diff.py writes small thumbnails next to the diff images in the | |
| # output directory. The landing page uses these for its <img src> so it | |
| # doesn't download every full resolution image at once; the full image is | |
| # still linked via the surrounding <a href>. Thumbnails always live in | |
| # the output directory regardless of --pack/--recursive, so their layout | |
| # mirrors the diff images (device sub-folder when there is one). | |
| thumb_dir = device_name if device_name is not None else "" | |
| self.golden_thumb = pathlib.Path(os.path.join(thumb_dir, f"{self.name}.golden.thumb.png")).as_posix() | |
| self.candidate_thumb = pathlib.Path(os.path.join(thumb_dir, f"{self.name}.candidate.thumb.png")).as_posix() | |
| self.diff0_thumb = pathlib.Path(os.path.join(thumb_dir, f"{self.name}.diff0.thumb.png")).as_posix() | |
| self.diff1_thumb = pathlib.Path(os.path.join(thumb_dir, f"{self.name}.diff1.thumb.png")).as_posix() | |
| if len(words) == 2: | |
| self.avg = None | |
| self.histogram = None | |
| self.type = words[1] | |
| else: | |
| self.max_diff = int(words[1]) | |
| self.avg = float(words[2]) | |
| self.total_diff_count = int(words[3]) | |
| self.total_pixels = int(words[4]) | |
| self.diff0_path_abs = os.path.join(output_path, f"{self.name}.diff0.png") | |
| self.diff1_path_abs = os.path.join(output_path, f"{self.name}.diff1.png") | |
| if device_name is not None: | |
| self.diff0_path = os.path.join(device_name, f"{self.name}.diff0.png") | |
| self.diff1_path = os.path.join(device_name, f"{self.name}.diff1.png") | |
| else: | |
| self.diff0_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff0.png"), output_path) | |
| self.diff1_path = os.path.relpath(os.path.join(output_path, f"{self.name}.diff1.png"), output_path) | |
| if len(words) == 6: | |
| self.histogram = float(words[5]) | |
| if self.histogram < (1.0-args.threshold): | |
| self.type = "failed" | |
| else: | |
| self.type = "pass" | |
| else: | |
| self.histogram = None | |
| if self.max_diff > args.threshold: | |
| self.type = "failed" | |
| else: | |
| self.type = "pass" | |
| # this is equivalent of implementing == we are comparing by name for when we check against the correct golds to delete | |
| def __eq__(self, other): | |
| return self.name == other.name | |
| # hash by name so that when we create a set out of entry list we condense it down to the number of goldens since we only care about them | |
| def __hash__(self): | |
| return hash(self.name) | |
| # this is equivalent of implementing < operator. We use this for sorted and sort functions | |
| def __lt__(self, other): | |
| # Always sort by avg first. Histogram is a good heuristic to divide into | |
| # "pass/fail" buckets, but it's helpful to then see the fail bucked | |
| # sorted by avg, which is more sensitive to differences. | |
| if (self.avg == other.avg and | |
| self.histogram is not None and | |
| other.histogram is not None): | |
| # LOWER histogram values mean worse matches. Sort the bad matches first. | |
| return self.histogram > other.histogram | |
| else: | |
| # HIGHER avg values mean worse matches. Sort the bad matches first. | |
| return self.avg < other.avg | |
| def __str__(self): | |
| vals = dict() | |
| vals['name'] = f"{self.name} ({self.device})" if self.device is not None else self.name | |
| vals['url'] = self.browserstack_details['browser_url'] if self.browserstack_details is not None else ' ' | |
| if self.type == 'missing_golden': | |
| # show candidate, since golden is missing | |
| vals['image'] = self.candidates_path | |
| vals['image_thumb'] = self.candidate_thumb | |
| return self.missing_file_entry_template.format_map(vals) | |
| elif self.type == 'missing_candidate': | |
| # show golden, since candidate is missing | |
| vals['image'] = self.golden_path | |
| vals['image_thumb'] = self.golden_thumb | |
| return self.missing_file_entry_template.format_map(vals) | |
| vals['golden'] = self.golden_path | |
| vals['candidate'] = self.candidates_path | |
| vals['golden_thumb'] = self.golden_thumb | |
| vals['candidate_thumb'] = self.candidate_thumb | |
| if self.type == "pass" or self.type == "failed": | |
| vals['max'] = self.max_diff | |
| vals['avg'] = self.avg | |
| vals['total_diff'] = self.total_diff_count | |
| vals['percent'] = float(self.total_diff_count) / float(self.total_pixels) | |
| vals['histogram'] = self.histogram if self.histogram is not None else 'None' | |
| vals['diff0'] = self.diff0_path | |
| vals['diff1'] = self.diff1_path | |
| vals['diff0_thumb'] = self.diff0_thumb | |
| vals['diff1_thumb'] = self.diff1_thumb | |
| if self.type == 'pass': | |
| return self.pass_entry_template.format_map(vals) | |
| else: | |
| return self.error_entry_template.format_map(vals) | |
| if self.type == 'identical': | |
| return self.identical_entry_template.format_map(vals) | |
| return '' | |
| def clean(self): | |
| if args.verbose: | |
| print(f"cleaning TestEntry {self.name} - {self.device}") | |
| # removes image files to save space, this is done for clean and fails only arguments | |
| os.remove(self.candidates_path_abs) | |
| # if we are packing the files, also delete the packed ones | |
| if args.pack: | |
| os.remove(os.path.join(args.output, self.candidates_path)) | |
| # if we have diff files, delete those too | |
| if self.diff0_path_abs is not None: | |
| os.remove(self.diff0_path_abs) | |
| # this should always be true if diff0 is not none but just to be safe we do a separate check | |
| if self.diff1_path_abs is not None: | |
| os.remove(self.diff1_path_abs) | |
| @property | |
| def success(self): | |
| return self.type == "pass" or self.type == "identical" | |
| @property | |
| def csv_dict(self): | |
| val = dict() | |
| val['file_name'] = self.name | |
| val['original'] = self.golden_path | |
| val['candidate'] = self.candidates_path | |
| if self.type == "pass" or self.type == "failed": | |
| val['max_rgb'] = str(self.max_diff) | |
| val['avg_rgb'] = str(self.avg) | |
| val['pixel_diff_count'] = str(self.total_diff_count) | |
| val['pixel_diff_percent'] = '100' | |
| if self.histogram is not None: | |
| val['hist_result'] = str(self.histogram) | |
| val['color_diff'] = self.diff0_path | |
| val['pixel_diff'] = self.diff1_path | |
| elif self.type == 'identical': | |
| val['max_rgb'] = '0' | |
| val['avg_rgb'] = '0' | |
| val['pixel_diff_count'] = '0' | |
| val['pixel_diff_percent'] = '100' | |
| if args.histogram_compare: | |
| val['hist_result'] = '1.0' | |
| val['color_diff'] = '' | |
| val['pixel_diff'] = '' | |
| return val | |
| def shallow_copy_images(src, dest): | |
| file_names = [file for file in os.scandir(src) if file.is_file() and '.png' in file.name] | |
| for file in file_names: | |
| shutil.copyfile(file.path, os.path.join(dest, file.name)) | |
| def write_csv(entries, origpath, candidatepath, diffpath, missing_candidates): | |
| origpath = os.path.relpath(origpath, diffpath) | |
| candidatepath = os.path.relpath(candidatepath, diffpath) | |
| height = 256 | |
| with open(os.path.join(diffpath, "data.csv"), "w") as csvfile: | |
| fieldnames = ['file_name','original', 'candidate', 'max_rgb', 'avg_rgb', 'pixel_diff_count', 'pixel_diff_percent', 'color_diff', 'pixel_diff'] | |
| if args.histogram_compare: | |
| fieldnames.extend(['hist_result']) | |
| writer = csv.DictWriter(csvfile, fieldnames=fieldnames) | |
| writer.writeheader() | |
| for name in missing_candidates: | |
| if args.histogram_compare: | |
| writer.writerow({ | |
| 'file_name': name.split('.')[0], | |
| 'original': os.path.join(origpath, name), | |
| 'candidate': '', | |
| 'max_rgb':255, | |
| 'avg_rgb':255, | |
| # i guess its all of em, not 1 of em, but whatever | |
| 'pixel_diff_count': 1, | |
| 'pixel_diff_percent': '100', | |
| 'hist_result' : 1.0, | |
| 'color_diff': '', | |
| 'pixel_diff': '' | |
| }) | |
| else: | |
| writer.writerow({ | |
| 'file_name': name.split('.')[0], | |
| 'original': os.path.join(origpath, name), | |
| 'candidate': '', | |
| 'max_rgb':255, | |
| 'avg_rgb':255, | |
| # i guess its all of em, not 1 of em, but whatever | |
| 'pixel_diff_count': 1, | |
| 'pixel_diff_percent': '100', | |
| 'color_diff': '', | |
| 'pixel_diff': '' | |
| }) | |
| for entry in entries: | |
| writer.writerow(entry.csv_dict) | |
| def write_min_csv(total_passing, total_failing, total_missing_candidates, total_missing_goldens, total_identical, total_entries, csv_path): | |
| # delete and old data | |
| if os.path.exists(csv_path): | |
| os.remove(csv_path) | |
| with open(csv_path, 'w', newline='') as csv_file: | |
| csv_writer = csv.DictWriter(csv_file, fieldnames=['type', 'number']) | |
| csv_writer.writerow({'type':'failed', 'number' : str(total_failing)}) | |
| csv_writer.writerow({'type':'pass', 'number' : str(total_passing)}) | |
| csv_writer.writerow({'type':'missing_candidates', 'number' : str(total_missing_candidates)}) | |
| csv_writer.writerow({'type':'missing_goldens', 'number' : str(total_missing_goldens)}) | |
| csv_writer.writerow({'type':'identical', 'number' : str(total_identical)}) | |
| csv_writer.writerow({'type':'total', 'number' : str(total_entries)}) | |
| def diff_worker(index, work_queue, golden, candidate, output, parent_pid): | |
| # One long-lived image_diff.py process that pulls filenames off the shared | |
| # work_queue one at a time. Keeping the process alive means cv2 is imported | |
| # only once (that import + process spawn dominate on Windows). | |
| cmd = [PYTHON, "image_diff.py", "--serve", | |
| "-g", golden, | |
| "-c", candidate, | |
| # Each process writes its own status file in order to avoid race conditions. | |
| "-s", "%s_%i_%i.txt" % (status_filename_base, parent_pid, index)] | |
| if output is not None: | |
| cmd.extend(["-o", output]) | |
| if args.verbose: | |
| cmd.extend(["-v", "-l"]) | |
| if args.histogram_compare: | |
| cmd.extend(["-H"]) | |
| if show_commands: | |
| print(" ".join(cmd)) | |
| proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, | |
| text=True) | |
| try: | |
| while True: | |
| try: | |
| filename = work_queue.get_nowait() | |
| except queue.Empty: | |
| break | |
| proc.stdin.write(filename + "\n") | |
| proc.stdin.flush() | |
| # Block until the worker signals it finished this image. If the pipe | |
| # closed early the worker died; stop feeding it (the missing status | |
| # line is caught by the line-count check in diff_directory_shallow). | |
| if proc.stdout.readline().strip() != "done": | |
| print("image_diff.py worker %i died processing %s" % (index, filename)) | |
| break | |
| finally: | |
| proc.stdin.close() | |
| proc.wait() | |
| def parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details): | |
| total_lines = 0 | |
| test_entries = [] | |
| success = True | |
| status_files = glob.glob(status_filename_pattern) | |
| if not status_files: | |
| print('Not a single status file got written, are you just starting new?') | |
| for status_filename in status_files: | |
| for line in open(status_filename, "r").readlines(): | |
| total_lines += 1 | |
| words = line.rstrip().split('\t') | |
| entry = TestEntry(words, candidates_path, golden_path, output_path, device_name, browserstack_details) | |
| test_entries.append(entry) | |
| if not entry.success: | |
| success = False | |
| return (total_lines, test_entries, success) | |
| def diff_directory_shallow(candidates_path, output_path, golden_path, device_name=None, browserstack_details=None): | |
| candidate_filenames = set((file.name for file in os.scandir(candidates_path) | |
| if file.is_file() and file.name.endswith('.png'))) | |
| golden_filenames = set((file.name for file in os.scandir(golden_path) | |
| if file.is_file() and file.name.endswith('.png'))) | |
| # Diff every file that appears in either directory. image_diff.py reports a | |
| # missing_candidate / missing_golden status for files that only exist on one | |
| # side and writes a thumbnail of the single image that does exist, so the | |
| # landing page can use thumbnails for missing entries too (rather than | |
| # eagerly loading the full resolution image). | |
| all_filenames = candidate_filenames | golden_filenames | |
| # Log which files are missing where; image_diff.py records them as entries. | |
| for file in candidate_filenames.difference(golden_filenames): | |
| print(f'Candidate file {file} missing in goldens.') | |
| for file in golden_filenames.difference(candidate_filenames): | |
| print(f'Golden file {file} missing in candidates.') | |
| if args.jobs > 1: | |
| print("Diffing %i candidates in %i processes..." % (len(all_filenames), args.jobs)) | |
| else: | |
| print("Diffing %i candidates..." % len(all_filenames)) | |
| sys.stdout.flush() | |
| # Fill a shared queue with every filename. | |
| work_queue = queue.Queue() | |
| for filename in all_filenames: | |
| work_queue.put(filename) | |
| # Generate the diffs (if any) and write to the status file(s). Start up to | |
| # `jobs` long-lived image_diff.py workers that each pull filenames off the | |
| # queue as fast as they can finish them. | |
| njobs = min(args.jobs, len(all_filenames)) | |
| parent_pid = os.getpid() | |
| threads = [] | |
| for index in range(njobs): | |
| t = threading.Thread(target=diff_worker, | |
| args=(index, work_queue, golden_path, | |
| candidates_path, output_path, parent_pid)) | |
| t.start() | |
| threads.append(t) | |
| for t in threads: | |
| t.join() | |
| (total_lines, entries, success) = parse_status(candidates_path, golden_path, output_path, device_name, browserstack_details) | |
| print(f'finished with Succes:{success} and {total_lines} lines') | |
| if total_lines != len(all_filenames): | |
| print(f"Internal failure: Got {total_lines} status lines. Expected {len(all_filenames)}.") | |
| success = False | |
| if candidate_filenames.symmetric_difference(golden_filenames): | |
| print("golden and candidate directories do not have identical files.") | |
| success = False | |
| # cleanup our scratch files | |
| for status_filename in glob.iglob(status_filename_pattern): | |
| os.remove(status_filename) | |
| missing = [entry for entry in entries | |
| if entry.type in ('missing_golden', 'missing_candidate')] | |
| return (entries, missing, success) | |
| # Sort descending total failure count (missing + failure), then by descending failure count, then by name | |
| # (put the failed ones at the top) | |
| def device_entry_sort_key(kv): | |
| return (-(kv[1]["missing_candidate"] + kv[1]["failed"]), -(kv[1]["failed"]), kv[0]) | |
| # returns entries sorted into identical, passing and failing as well as html str list of each | |
| # based on arguments passed, we may or may not return all of the string lists, but we always return the object lists | |
| def sort_entries(entries): | |
| # we dont need an intermediate object list because we never sort ot clean these, so direct to html | |
| missing_golden_str = [str(entry) for entry in entries if entry.type == "missing_golden"] | |
| missing_candidate_str = [str(entry) for entry in entries if entry.type == "missing_candidate"] | |
| failed_entires = [entry for entry in entries if entry.type == "failed"] | |
| pass_entires = [entry for entry in entries if entry.type == "pass"] | |
| identical_entires = [entry for entry in entries if entry.type == "identical"] | |
| sorted_failed_entires = sorted(failed_entires, reverse=True) | |
| sorted_failed_str = [str(entry) for entry in sorted_failed_entires] | |
| # Build a list of stat counts by device name | |
| all_device_stats = dict() | |
| for entry in entries: | |
| if entry.device not in all_device_stats: | |
| all_device_stats[entry.device] = defaultdict(int) | |
| all_device_stats[entry.device]["url"] = entry.browserstack_details['browser_url'] if entry.browserstack_details is not None else ' ' | |
| all_device_stats[entry.device][entry.type] += 1 | |
| # Now build the device summary text using the template | |
| device_summary_str = "" | |
| with open(os.path.join(TEMPLATE_PATH, "device_summary_entry.html")) as t: | |
| device_summary_entry_template = t.read() | |
| for device_name, device_summary in sorted(all_device_stats.items(), key=device_entry_sort_key): | |
| none_succeeded = device_summary["pass"] == 0 and device_summary["identical"] == 0 | |
| any_failed = device_summary["failed"] > 0 | |
| any_missing = device_summary["missing_candidate"] > 0 | |
| device_summary_str += device_summary_entry_template.format( | |
| name=device_name, | |
| url=device_summary["url"], | |
| failed_count=device_summary["failed"] if any_failed else "-", | |
| failed_class="fail" if any_failed else "deemphasize", | |
| missing_count=device_summary["missing_candidate"] if any_missing else "-", | |
| missing_class="fail" if any_missing else "deemphasize", | |
| pass_count=device_summary["pass"], | |
| pass_class="fail" if none_succeeded else "success", | |
| identical_count=device_summary["identical"], | |
| identical_class="fail" if none_succeeded else "" if device_summary["identical"] == 0 else "success") | |
| # if we are only doing fails then only sort those and return empty html lists for "pass" and "identical" we still build and return | |
| # identical and pass object lists for cleaning, but we dont bother sorting them | |
| if args.fails_only: | |
| return ( | |
| sorted_failed_entires, | |
| pass_entires, | |
| identical_entires, | |
| sorted_failed_str, | |
| [], | |
| [], | |
| missing_golden_str, | |
| missing_candidate_str, | |
| device_summary_str, | |
| len(all_device_stats)) | |
| # now sort passed entires and build the html list | |
| sorted_passed_entires = sorted(pass_entires, reverse=True) | |
| sorted_passed_str = [str(entry) for entry in sorted_passed_entires] | |
| # if we are cleaning then return empty html list for identical. do everything else the same | |
| if args.clean: | |
| return (sorted_failed_entires, sorted_passed_entires, identical_entires, sorted_failed_str, sorted_passed_str, [], missing_golden_str, missing_candidate_str, device_summary_str, len(all_device_stats)) | |
| # otherwise build identical html entry list and include it in the return | |
| identical_str = [str(entry) for entry in identical_entires] | |
| return ( | |
| sorted_failed_entires, | |
| sorted_passed_entires, | |
| identical_entires, | |
| sorted_failed_str, | |
| sorted_passed_str, | |
| identical_str, | |
| missing_golden_str, | |
| missing_candidate_str, | |
| device_summary_str, | |
| len(all_device_stats)) | |
| def write_html(templates_path, failed_entries, passing_entries, identical_entries, missing_golden_entries, missing_candidate_entries, device_summary_str, device_number, output_path): | |
| with open(os.path.join(templates_path, "index.html")) as t: | |
| index_template = t.read() | |
| html = index_template.format(identical=" ".join(identical_entries), passing=" ".join(passing_entries), | |
| failed=" ".join(failed_entries), failed_number=len(failed_entries), | |
| passing_number=len(passing_entries), identical_number=len(identical_entries), | |
| missing_candidate=" ".join(missing_candidate_entries), missing_candidate_number=len(missing_candidate_entries), | |
| missing_golden=" ".join(missing_golden_entries), missing_golden_number=len(missing_golden_entries), | |
| device_summaries=device_summary_str, device_number=device_number, | |
| pass_hide_class="hidden" if args.fails_only else "", | |
| identical_hide_class="hidden" if args.fails_only or args.clean else "") | |
| with open(os.path.join(output_path, "index.html"), "w") as file: | |
| file.write(html) | |
| #copy our icon to the output folder | |
| shutil.copyfile(os.path.join(TEMPLATE_PATH, "favicon.ico"), os.path.join(output_path, "favicon.ico")) | |
| def diff_directory_deep(candidates_path, output_path): | |
| golden_path = args.goldens | |
| if args.pack: | |
| new_golden_path = os.path.join(output_path, "golden") | |
| os.makedirs(new_golden_path, exist_ok=True) | |
| shallow_copy_images(args.goldens, new_golden_path) | |
| golden_path = new_golden_path | |
| all_entries = [] | |
| for folder in os.scandir(candidates_path): | |
| if folder.is_dir(): | |
| if folder.name[0] == '.': | |
| continue | |
| output = os.path.join(output_path ,folder.name) | |
| os.makedirs(output, exist_ok=True) | |
| browserstack_details = None | |
| browserstack_details_path = os.path.join(folder.path, "session_details") | |
| if os.path.exists(browserstack_details_path): | |
| with open(browserstack_details_path, 'rt') as file: | |
| browserstack_details = json.load(file) | |
| os.remove(browserstack_details_path) | |
| (entries, _, _) = diff_directory_shallow(folder.path, output, golden_path, folder.name, browserstack_details) | |
| all_entries.extend(entries) | |
| if args.pack: | |
| shallow_copy_images(folder.path, output) | |
| (failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number) = sort_entries(all_entries) | |
| to_clean = [] | |
| to_check = [] | |
| # choose who to clean and who to check against | |
| if args.clean: | |
| to_clean = identical | |
| to_check = failed + passed | |
| if args.fails_only: | |
| to_clean = identical + passed | |
| to_check = failed | |
| # clean them | |
| for obj in to_clean: | |
| obj.clean() | |
| # only remove goldens that are in to_clean and not to_check so that we keep goldens that are still used | |
| # this part is why we needed __eq__ and __hash__ in our TestEntry class | |
| for obj in set(to_clean) - set(to_check): | |
| golden_file = os.path.join(golden_path, f"{obj.name}.png") | |
| if args.verbose: | |
| print("deleting orphaned golden " + golden_file) | |
| os.remove(golden_file) | |
| if args.pack: | |
| # remember to remove the original if we packed it | |
| os.remove(os.path.join(args.goldens, f"{obj.name}.png")) | |
| write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number, output_path) | |
| print(f"total entries {len(all_entries)}") | |
| write_min_csv(len(passed), len(failed), len(missing_candidate_str), len(missing_golden_str), len(identical), len(all_entries), output_path + "/issues.csv") | |
| def main(argv=None): | |
| if not os.path.exists(args.goldens): | |
| print("Can't find goldens " + args.goldens) | |
| return -1 | |
| if not os.path.exists(args.candidates): | |
| print("Can't find candidates " + args.candidates) | |
| return -1 | |
| # delete output dir if exists | |
| shutil.rmtree(args.output, ignore_errors=True) | |
| # remake output dir, this will make it correctly | |
| # even if it requires creating mulltiple directories | |
| os.makedirs(args.output, exist_ok=True) | |
| # reset our scratch files | |
| for status_filename in glob.iglob(status_filename_pattern): | |
| os.remove(status_filename) | |
| TestEntry.load_templates(TEMPLATE_PATH) | |
| if args.recursive: | |
| diff_directory_deep(args.candidates, args.output) | |
| else: | |
| (entries, missing, success) = diff_directory_shallow(args.candidates, args.output, args.goldens) | |
| if len(entries) > 0: | |
| (failed, passed, identical, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number) = sort_entries(entries) | |
| assert(len(failed) + len(passed) + len(identical) + len(missing_candidate_str) + len(missing_golden_str) == len(entries)) | |
| write_html(TEMPLATE_PATH, failed_str, passed_str, identical_str, missing_golden_str, missing_candidate_str, device_summary_str, device_number, args.output) | |
| # note could add these to the html output but w/e | |
| missing_candidates = [os.path.basename(entry.candidates_path_abs) for entry in missing if entry.type == 'missing_candidate'] | |
| write_csv(entries, args.goldens, args.candidates, args.output, missing_candidates) | |
| print("Found", len(entries) - len(identical), "differences,", | |
| len(failed), "failing.") | |
| # here we have to do a lot less work than the diff_directory_deep since we know goldens are not shared with other TestEntries | |
| if args.fails_only: | |
| for obj in identical+passed: | |
| obj.clean() | |
| golden_path = os.path.join(args.goldens, f"{obj.name}.png") | |
| if args.verbose: | |
| print(f"deleting orphaned golden {golden_path}") | |
| os.remove(golden_path) | |
| elif args.clean: | |
| for obj in identical: | |
| obj.clean() | |
| golden_path = os.path.join(args.goldens, f"{obj.name}.png") | |
| if args.verbose: | |
| print(f"deleting orphaned golden {golden_path}") | |
| os.remove(golden_path) | |
| # if we are in fail only mode than make it succesful when there are only "passing" entries | |
| if args.fails_only: | |
| if failed: | |
| # if there were diffs, its gotta fail | |
| print("FAILED.") | |
| return -1 | |
| # otherwise fail like normal | |
| elif not success: | |
| # if there were diffs, its gotta fail | |
| print("FAILED.") | |
| return -1 | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) |