icu4c/source/python/icutools/databuilder/__main__.py - external/github.com/unicode-org/icu - Git at Google

 # Copyright (C) 2018 and later: Unicode, Inc. and others.
 # License & terms of use: http://www.unicode.org/copyright.html

 # Python 2/3 Compatibility (ICU-20299)
 # TODO(ICU-20301): Remove this.
 from __future__ import print_function

 import argparse
 import glob as pyglob
 import io as pyio
 import json
 import os
 import sys

 from . import *
 from .comment_stripper import CommentStripper
 from .request_types import CopyRequest
 from .renderers import makefile, common_exec
 from . import filtration, utils

 flag_parser = argparse.ArgumentParser(
     description = """Generates rules for building ICU binary data files from text
 and other input files in source control.

 Use the --mode option to declare how to execute those rules, either exporting
 the rules to a Makefile or spawning child processes to run them immediately:

   --mode=gnumake prints a Makefile to standard out.
   --mode=unix-exec spawns child processes in a Unix-like environment.
   --mode=windows-exec spawns child processes in a Windows-like environment.

 Tips for --mode=unix-exec
 =========================

 Create two empty directories for out_dir and tmp_dir. They will get filled
 with a lot of intermediate files.

 Set LD_LIBRARY_PATH to include the lib directory. e.g., from icu4c/source:

   $ LD_LIBRARY_PATH=lib PYTHONPATH=python python3 -m icutools.databuilder ...

 Once icutools.databuilder finishes, you have compiled the data, but you have
 not packaged it into a .dat or .so file. This is done by the separate pkgdata
 tool in bin. Read the docs of pkgdata:

   $ LD_LIBRARY_PATH=lib ./bin/pkgdata --help

 Example command line to call pkgdata:

   $ LD_LIBRARY_PATH=lib ./bin/pkgdata -m common -p icudt63l -c \\
       -O data/icupkg.inc -s $OUTDIR -d $TMPDIR $TMPDIR/icudata.lst

 where $OUTDIR and $TMPDIR are your out and tmp directories, respectively.
 The above command will create icudt63l.dat in the tmpdir.

 Command-Line Arguments
 ======================
 """,
     formatter_class = argparse.RawDescriptionHelpFormatter
 )

 arg_group_required = flag_parser.add_argument_group("required arguments")
 arg_group_required.add_argument(
     "--mode",
     help = "What to do with the generated rules.",
     choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"],
     required = True
 )

 flag_parser.add_argument(
     "--src_dir",
     help = "Path to data source folder (icu4c/source/data).",
     default = "."
 )
 flag_parser.add_argument(
     "--filter_file",
     metavar = "PATH",
     help = "Path to an ICU data filter JSON file.",
     default = None
 )
 flag_parser.add_argument(
     "--include_uni_core_data",
     help = "Include the full Unicode core data in the dat file.",
     default = False,
     action = "store_true"
 )
 flag_parser.add_argument(
     "--seqmode",
     help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
     choices = ["sequential", "parallel"],
     default = "sequential"
 )
 flag_parser.add_argument(
     "--verbose",
     help = "Print more verbose output (default false).",
     default = False,
     action = "store_true"
 )

 arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes")
 arg_group_exec.add_argument(
     "--out_dir",
     help = "Path to where to save output data files.",
     default = "icudata"
 )
 arg_group_exec.add_argument(
     "--tmp_dir",
     help = "Path to where to save temporary files.",
     default = "icutmp"
 )
 arg_group_exec.add_argument(
     "--tool_dir",
     help = "Path to where to find binary tools (genrb, etc).",
     default = "../bin"
 )
 arg_group_exec.add_argument(
     "--tool_cfg",
     help = "The build configuration of the tools. Used in 'windows-exec' mode only.",
     default = "x86/Debug"
 )


 class Config(object):

     def __init__(self, args):
         # Process arguments
         self.max_parallel = (args.seqmode == "parallel")

         # Boolean: Whether to include core Unicode data files in the .dat file
         self.include_uni_core_data = args.include_uni_core_data

         # Default fields before processing filter file
         self.filters_json_data = {}
         self.filter_dir = "ERROR_NO_FILTER_FILE"

         # Process filter file
         if args.filter_file:
             try:
                 with open(args.filter_file, "r") as f:
                     print("Note: Applying filters from %s." % args.filter_file, file=sys.stderr)
                     self._parse_filter_file(f)
             except IOError:
                 print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr)
                 exit(1)
             self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file))

         # Either "unihan" or "implicithan"
         self.coll_han_type = "unihan"
         if "collationUCAData" in self.filters_json_data:
             self.coll_han_type = self.filters_json_data["collationUCAData"]

         # Either "additive" or "subtractive"
         self.strategy = "subtractive"
         if "strategy" in self.filters_json_data:
             self.strategy = self.filters_json_data["strategy"]

         # True or False (could be extended later to support enum/list)
         self.use_pool_bundle = True
         if "usePoolBundle" in self.filters_json_data:
             self.use_pool_bundle = self.filters_json_data["usePoolBundle"]

     def _parse_filter_file(self, f):
         # Use the Hjson parser if it is available; otherwise, use vanilla JSON.
         try:
             import hjson
             self.filters_json_data = hjson.load(f)
         except ImportError:
             self.filters_json_data = json.load(CommentStripper(f))

         # Optionally pre-validate the JSON schema before further processing.
         # Some schema errors will be caught later, but this step ensures
         # maximal validity.
         try:
             import jsonschema
             schema_path = os.path.join(os.path.dirname(__file__), "filtration_schema.json")
             with open(schema_path) as schema_f:
                 schema = json.load(CommentStripper(schema_f))
             validator = jsonschema.Draft4Validator(schema)
             for error in validator.iter_errors(self.filters_json_data, schema):
                 print("WARNING: ICU data filter JSON file:", error.message,
                     "at", "".join(
                         "[%d]" % part if isinstance(part, int) else ".%s" % part
                         for part in error.absolute_path
                     ),
                     file=sys.stderr)
         except ImportError:
             print("Tip: to validate your filter file, install the Pip package 'jsonschema'", file=sys.stderr)
             pass


 def add_copy_input_requests(requests, config, common_vars):
     files_to_copy = set()
     for request in requests:
         request_files = request.all_input_files()
         # Also add known dependency txt files as possible inputs.
         # This is required for translit rule files.
         if hasattr(request, "dep_targets"):
             request_files += [
                 f for f in request.dep_targets if isinstance(f, InFile)
             ]
         for f in request_files:
             if isinstance(f, InFile):
                 files_to_copy.add(f)

     result = []
     id = 0

     json_data = config.filters_json_data["fileReplacements"]
     dirname = json_data["directory"]
     for directive in json_data["replacements"]:
         if type(directive) == str:
             input_file = LocalFile(dirname, directive)
             output_file = InFile(directive)
         else:
             input_file = LocalFile(dirname, directive["src"])
             output_file = InFile(directive["dest"])
         result += [
             CopyRequest(
                 name = "input_copy_%d" % id,
                 input_file = input_file,
                 output_file = output_file
             )
         ]
         files_to_copy.remove(output_file)
         id += 1

     for f in files_to_copy:
         result += [
             CopyRequest(
                 name = "input_copy_%d" % id,
                 input_file = SrcFile(f.filename),
                 output_file = f
             )
         ]
         id += 1

     result += requests
     return result


 class IO(object):
     """I/O operations required when computing the build actions"""

     def __init__(self, src_dir):
         self.src_dir = src_dir

     def glob(self, pattern):
         absolute_paths = pyglob.glob(os.path.join(self.src_dir, pattern))
         # Strip off the absolute path suffix so we are left with a relative path.
         relative_paths = [v[len(self.src_dir)+1:] for v in sorted(absolute_paths)]
         # For the purposes of icutools.databuilder, force Unix-style directory separators.
         # Within the Python code, including BUILDRULES.py and user-provided config files,
         # directory separators are normalized to '/', including on Windows platforms.
         return [v.replace("\\", "/") for v in relative_paths]

     def read_locale_deps(self, tree):
         return self._read_json("%s/LOCALE_DEPS.json" % tree)

     def _read_json(self, filename):
         with pyio.open(os.path.join(self.src_dir, filename), "r", encoding="utf-8-sig") as f:
             return json.load(CommentStripper(f))


 def main(argv):
     args = flag_parser.parse_args(argv)
     config = Config(args)

     if args.mode == "gnumake":
         makefile_vars = {
             "SRC_DIR": "$(srcdir)",
             "IN_DIR": "$(srcdir)",
             "INDEX_NAME": "res_index"
         }
         makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR"]
         common = {
             key: "$(%s)" % key
             for key in list(makefile_vars.keys()) + makefile_env
         }
         common["FILTERS_DIR"] = config.filter_dir
         common["CWD_DIR"] = os.getcwd()
     else:
         makefile_vars = None
         common = {
             "SRC_DIR": args.src_dir,
             "IN_DIR": args.src_dir,
             "OUT_DIR": args.out_dir,
             "TMP_DIR": args.tmp_dir,
             "FILTERS_DIR": config.filter_dir,
             "CWD_DIR": os.getcwd(),
             "INDEX_NAME": "res_index",
             # TODO: Pull this from configure script:
             "ICUDATA_CHAR": "l"
         }

     # Automatically load BUILDRULES from the src_dir
     sys.path.append(args.src_dir)
     try:
         import BUILDRULES
     except ImportError:
         print("Cannot find BUILDRULES! Did you set your --src_dir?", file=sys.stderr)
         sys.exit(1)

     io = IO(args.src_dir)
     requests = BUILDRULES.generate(config, io, common)

     if "fileReplacements" in config.filters_json_data:
         tmp_in_dir = "{TMP_DIR}/in".format(**common)
         if makefile_vars:
             makefile_vars["IN_DIR"] = tmp_in_dir
         else:
             common["IN_DIR"] = tmp_in_dir
         requests = add_copy_input_requests(requests, config, common)

     requests = filtration.apply_filters(requests, config, io)
     requests = utils.flatten_requests(requests, config, common)

     build_dirs = utils.compute_directories(requests)

     if args.mode == "gnumake":
         print(makefile.get_gnumake_rules(
             build_dirs,
             requests,
             makefile_vars,
             common_vars = common
         ))
     elif args.mode == "windows-exec":
         return common_exec.run(
             platform = "windows",
             build_dirs = build_dirs,
             requests = requests,
             common_vars = common,
             tool_dir = args.tool_dir,
             tool_cfg = args.tool_cfg,
             verbose = args.verbose,
         )
     elif args.mode == "unix-exec":
         return common_exec.run(
             platform = "unix",
             build_dirs = build_dirs,
             requests = requests,
             common_vars = common,
             tool_dir = args.tool_dir,
             verbose = args.verbose,
         )
     elif args.mode == "bazel-exec":
         return common_exec.run(
             platform = "bazel",
             build_dirs = build_dirs,
             requests = requests,
             common_vars = common,
             tool_dir = args.tool_dir,
             verbose = args.verbose,
         )
     else:
         print("Mode not supported: %s" % args.mode)
         return 1
     return 0

 if __name__ == "__main__":
     exit(main(sys.argv[1:]))
	# Copyright (C) 2018 and later: Unicode, Inc. and others.
	# License & terms of use: http://www.unicode.org/copyright.html

	# Python 2/3 Compatibility (ICU-20299)
	# TODO(ICU-20301): Remove this.
	from __future__ import print_function

	import argparse
	import glob as pyglob
	import io as pyio
	import json
	import os
	import sys

	from . import *
	from .comment_stripper import CommentStripper
	from .request_types import CopyRequest
	from .renderers import makefile, common_exec
	from . import filtration, utils

	flag_parser = argparse.ArgumentParser(
	description = """Generates rules for building ICU binary data files from text
	and other input files in source control.

	Use the --mode option to declare how to execute those rules, either exporting
	the rules to a Makefile or spawning child processes to run them immediately:

	--mode=gnumake prints a Makefile to standard out.
	--mode=unix-exec spawns child processes in a Unix-like environment.
	--mode=windows-exec spawns child processes in a Windows-like environment.

	Tips for --mode=unix-exec
	=========================

	Create two empty directories for out_dir and tmp_dir. They will get filled
	with a lot of intermediate files.

	Set LD_LIBRARY_PATH to include the lib directory. e.g., from icu4c/source:

	$ LD_LIBRARY_PATH=lib PYTHONPATH=python python3 -m icutools.databuilder ...

	Once icutools.databuilder finishes, you have compiled the data, but you have
	not packaged it into a .dat or .so file. This is done by the separate pkgdata
	tool in bin. Read the docs of pkgdata:

	$ LD_LIBRARY_PATH=lib ./bin/pkgdata --help

	Example command line to call pkgdata:

	$ LD_LIBRARY_PATH=lib ./bin/pkgdata -m common -p icudt63l -c \\
	-O data/icupkg.inc -s $OUTDIR -d $TMPDIR $TMPDIR/icudata.lst

	where $OUTDIR and $TMPDIR are your out and tmp directories, respectively.
	The above command will create icudt63l.dat in the tmpdir.

	Command-Line Arguments
	======================
	""",
	formatter_class = argparse.RawDescriptionHelpFormatter
	)

	arg_group_required = flag_parser.add_argument_group("required arguments")
	arg_group_required.add_argument(
	"--mode",
	help = "What to do with the generated rules.",
	choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"],
	required = True
	)

	flag_parser.add_argument(
	"--src_dir",
	help = "Path to data source folder (icu4c/source/data).",
	default = "."
	)
	flag_parser.add_argument(
	"--filter_file",
	metavar = "PATH",
	help = "Path to an ICU data filter JSON file.",
	default = None
	)
	flag_parser.add_argument(
	"--include_uni_core_data",
	help = "Include the full Unicode core data in the dat file.",
	default = False,
	action = "store_true"
	)
	flag_parser.add_argument(
	"--seqmode",
	help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
	choices = ["sequential", "parallel"],
	default = "sequential"
	)
	flag_parser.add_argument(
	"--verbose",
	help = "Print more verbose output (default false).",
	default = False,
	action = "store_true"
	)

	arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes")
	arg_group_exec.add_argument(
	"--out_dir",
	help = "Path to where to save output data files.",
	default = "icudata"
	)
	arg_group_exec.add_argument(
	"--tmp_dir",
	help = "Path to where to save temporary files.",
	default = "icutmp"
	)
	arg_group_exec.add_argument(
	"--tool_dir",
	help = "Path to where to find binary tools (genrb, etc).",
	default = "../bin"
	)
	arg_group_exec.add_argument(
	"--tool_cfg",
	help = "The build configuration of the tools. Used in 'windows-exec' mode only.",
	default = "x86/Debug"
	)


	class Config(object):

	def __init__(self, args):
	# Process arguments
	self.max_parallel = (args.seqmode == "parallel")

	# Boolean: Whether to include core Unicode data files in the .dat file
	self.include_uni_core_data = args.include_uni_core_data

	# Default fields before processing filter file
	self.filters_json_data = {}
	self.filter_dir = "ERROR_NO_FILTER_FILE"

	# Process filter file
	if args.filter_file:
	try:
	with open(args.filter_file, "r") as f:
	print("Note: Applying filters from %s." % args.filter_file, file=sys.stderr)
	self._parse_filter_file(f)
	except IOError:
	print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr)
	exit(1)
	self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file))

	# Either "unihan" or "implicithan"
	self.coll_han_type = "unihan"
	if "collationUCAData" in self.filters_json_data:
	self.coll_han_type = self.filters_json_data["collationUCAData"]

	# Either "additive" or "subtractive"
	self.strategy = "subtractive"
	if "strategy" in self.filters_json_data:
	self.strategy = self.filters_json_data["strategy"]

	# True or False (could be extended later to support enum/list)
	self.use_pool_bundle = True
	if "usePoolBundle" in self.filters_json_data:
	self.use_pool_bundle = self.filters_json_data["usePoolBundle"]

	def _parse_filter_file(self, f):
	# Use the Hjson parser if it is available; otherwise, use vanilla JSON.
	try:
	import hjson
	self.filters_json_data = hjson.load(f)
	except ImportError:
	self.filters_json_data = json.load(CommentStripper(f))

	# Optionally pre-validate the JSON schema before further processing.
	# Some schema errors will be caught later, but this step ensures
	# maximal validity.
	try:
	import jsonschema
	schema_path = os.path.join(os.path.dirname(__file__), "filtration_schema.json")
	with open(schema_path) as schema_f:
	schema = json.load(CommentStripper(schema_f))
	validator = jsonschema.Draft4Validator(schema)
	for error in validator.iter_errors(self.filters_json_data, schema):
	print("WARNING: ICU data filter JSON file:", error.message,
	"at", "".join(
	"[%d]" % part if isinstance(part, int) else ".%s" % part
	for part in error.absolute_path
	),
	file=sys.stderr)
	except ImportError:
	print("Tip: to validate your filter file, install the Pip package 'jsonschema'", file=sys.stderr)
	pass


	def add_copy_input_requests(requests, config, common_vars):
	files_to_copy = set()
	for request in requests:
	request_files = request.all_input_files()
	# Also add known dependency txt files as possible inputs.
	# This is required for translit rule files.
	if hasattr(request, "dep_targets"):
	request_files += [
	f for f in request.dep_targets if isinstance(f, InFile)
	]
	for f in request_files:
	if isinstance(f, InFile):
	files_to_copy.add(f)

	result = []
	id = 0

	json_data = config.filters_json_data["fileReplacements"]
	dirname = json_data["directory"]
	for directive in json_data["replacements"]:
	if type(directive) == str:
	input_file = LocalFile(dirname, directive)
	output_file = InFile(directive)
	else:
	input_file = LocalFile(dirname, directive["src"])
	output_file = InFile(directive["dest"])
	result += [
	CopyRequest(
	name = "input_copy_%d" % id,
	input_file = input_file,
	output_file = output_file
	)
	]
	files_to_copy.remove(output_file)
	id += 1

	for f in files_to_copy:
	result += [
	CopyRequest(
	name = "input_copy_%d" % id,
	input_file = SrcFile(f.filename),
	output_file = f
	)
	]
	id += 1

	result += requests
	return result


	class IO(object):
	"""I/O operations required when computing the build actions"""

	def __init__(self, src_dir):
	self.src_dir = src_dir

	def glob(self, pattern):
	absolute_paths = pyglob.glob(os.path.join(self.src_dir, pattern))
	# Strip off the absolute path suffix so we are left with a relative path.
	relative_paths = [v[len(self.src_dir)+1:] for v in sorted(absolute_paths)]
	# For the purposes of icutools.databuilder, force Unix-style directory separators.
	# Within the Python code, including BUILDRULES.py and user-provided config files,
	# directory separators are normalized to '/', including on Windows platforms.
	return [v.replace("\\", "/") for v in relative_paths]

	def read_locale_deps(self, tree):
	return self._read_json("%s/LOCALE_DEPS.json" % tree)

	def _read_json(self, filename):
	with pyio.open(os.path.join(self.src_dir, filename), "r", encoding="utf-8-sig") as f:
	return json.load(CommentStripper(f))


	def main(argv):
	args = flag_parser.parse_args(argv)
	config = Config(args)

	if args.mode == "gnumake":
	makefile_vars = {
	"SRC_DIR": "$(srcdir)",
	"IN_DIR": "$(srcdir)",
	"INDEX_NAME": "res_index"
	}
	makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR"]
	common = {
	key: "$(%s)" % key
	for key in list(makefile_vars.keys()) + makefile_env
	}
	common["FILTERS_DIR"] = config.filter_dir
	common["CWD_DIR"] = os.getcwd()
	else:
	makefile_vars = None
	common = {
	"SRC_DIR": args.src_dir,
	"IN_DIR": args.src_dir,
	"OUT_DIR": args.out_dir,
	"TMP_DIR": args.tmp_dir,
	"FILTERS_DIR": config.filter_dir,
	"CWD_DIR": os.getcwd(),
	"INDEX_NAME": "res_index",
	# TODO: Pull this from configure script:
	"ICUDATA_CHAR": "l"
	}

	# Automatically load BUILDRULES from the src_dir
	sys.path.append(args.src_dir)
	try:
	import BUILDRULES
	except ImportError:
	print("Cannot find BUILDRULES! Did you set your --src_dir?", file=sys.stderr)
	sys.exit(1)

	io = IO(args.src_dir)
	requests = BUILDRULES.generate(config, io, common)

	if "fileReplacements" in config.filters_json_data:
	tmp_in_dir = "{TMP_DIR}/in".format(**common)
	if makefile_vars:
	makefile_vars["IN_DIR"] = tmp_in_dir
	else:
	common["IN_DIR"] = tmp_in_dir
	requests = add_copy_input_requests(requests, config, common)

	requests = filtration.apply_filters(requests, config, io)
	requests = utils.flatten_requests(requests, config, common)

	build_dirs = utils.compute_directories(requests)

	if args.mode == "gnumake":
	print(makefile.get_gnumake_rules(
	build_dirs,
	requests,
	makefile_vars,
	common_vars = common
	))
	elif args.mode == "windows-exec":
	return common_exec.run(
	platform = "windows",
	build_dirs = build_dirs,
	requests = requests,
	common_vars = common,
	tool_dir = args.tool_dir,
	tool_cfg = args.tool_cfg,
	verbose = args.verbose,
	)
	elif args.mode == "unix-exec":
	return common_exec.run(
	platform = "unix",
	build_dirs = build_dirs,
	requests = requests,
	common_vars = common,
	tool_dir = args.tool_dir,
	verbose = args.verbose,
	)
	elif args.mode == "bazel-exec":
	return common_exec.run(
	platform = "bazel",
	build_dirs = build_dirs,
	requests = requests,
	common_vars = common,
	tool_dir = args.tool_dir,
	verbose = args.verbose,
	)
	else:
	print("Mode not supported: %s" % args.mode)
	return 1
	return 0

	if __name__ == "__main__":
	exit(main(sys.argv[1:]))