ICU-10923 Adding file replacement mechanism to buildtool.
diff --git a/icu4c/source/configure b/icu4c/source/configure
index 91c3513..274f361 100755
--- a/icu4c/source/configure
+++ b/icu4c/source/configure
@@ -9131,7 +9131,7 @@
PYTHONPATH="$srcdir/data" $PYTHON -m buildtool \
--mode gnumake \
--seqmode parallel \
- --in_dir "$srcdir/data" \
+ --src_dir "$srcdir/data" \
--filter_file "$ICU_DATA_FILTER_FILE" \
> data/rules.mk
if test "$?" != "0"; then
@@ -9141,7 +9141,7 @@
PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON -m buildtool \
--mode gnumake \
--seqmode parallel \
- --in_dir "$srcdir/test/testdata" \
+ --src_dir "$srcdir/test/testdata" \
> test/testdata/rules.mk
if test "$?" != "0"; then
as_fn_error $? "Python failed to run; see above error." "$LINENO" 5
diff --git a/icu4c/source/configure.ac b/icu4c/source/configure.ac
index e5feb4c..ef3e72f 100644
--- a/icu4c/source/configure.ac
+++ b/icu4c/source/configure.ac
@@ -1395,7 +1395,7 @@
PYTHONPATH="$srcdir/data" $PYTHON -m buildtool \
--mode gnumake \
--seqmode parallel \
- --in_dir "$srcdir/data" \
+ --src_dir "$srcdir/data" \
--filter_file "$ICU_DATA_FILTER_FILE" \
> data/rules.mk
if test "$?" != "0"; then
@@ -1405,7 +1405,7 @@
PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON -m buildtool \
--mode gnumake \
--seqmode parallel \
- --in_dir "$srcdir/test/testdata" \
+ --src_dir "$srcdir/test/testdata" \
> test/testdata/rules.mk
if test "$?" != "0"; then
AC_MSG_ERROR(Python failed to run; see above error.)
diff --git a/icu4c/source/data/BUILDRULES.py b/icu4c/source/data/BUILDRULES.py
index 12e2089..bdcd781 100644
--- a/icu4c/source/data/BUILDRULES.py
+++ b/icu4c/source/data/BUILDRULES.py
@@ -18,7 +18,7 @@
requests = []
if len(glob("misc/*")) == 0:
- print("Error: Cannot find data directory; please specify --in_dir", file=sys.stderr)
+ print("Error: Cannot find data directory; please specify --src_dir", file=sys.stderr)
exit(1)
requests += generate_cnvalias(config, glob, common_vars)
@@ -101,7 +101,7 @@
# Depends on timezoneTypes.res and keyTypeData.res.
# TODO: We should not need this dependency to build collation.
# TODO: Bake keyTypeData.res into the common library?
- [DepTarget("coll_ucadata"), DepTarget("misc_res")])
+ [DepTarget("coll_ucadata"), DepTarget("misc_res"), InFile("unidata/UCARules.txt")])
requests += generate_tree(config, glob, common_vars,
"brkitr",
@@ -225,7 +225,7 @@
RepeatedExecutionRequest(
name = "stringprep",
category = "stringprep",
- dep_targets = [],
+ dep_targets = [InFile("unidata/NormalizationCorrections.txt")],
input_files = input_files,
output_files = output_files,
tool = IcuTool("gensprep"),
@@ -398,6 +398,9 @@
InFile("translit/en.txt"),
InFile("translit/el.txt")
]
+ dep_files = set(InFile(filename) for filename in glob("translit/*.txt"))
+ dep_files -= set(input_files)
+ dep_files = list(dep_files)
input_basenames = [v.filename[9:] for v in input_files]
output_files = [
OutFile("translit/%s.res" % v[:-4])
@@ -407,7 +410,7 @@
RepeatedOrSingleExecutionRequest(
name = "translit_res",
category = "translit",
- dep_targets = [],
+ dep_targets = dep_files,
input_files = input_files,
output_files = output_files,
tool = IcuTool("genrb"),
diff --git a/icu4c/source/data/buildtool/__init__.py b/icu4c/source/data/buildtool/__init__.py
index 04a8c34..be93616 100644
--- a/icu4c/source/data/buildtool/__init__.py
+++ b/icu4c/source/data/buildtool/__init__.py
@@ -3,6 +3,8 @@
from collections import namedtuple
+LocalFile = namedtuple("LocalFile", ["dirname", "filename"])
+SrcFile = namedtuple("SrcFile", ["filename"])
InFile = namedtuple("InFile", ["filename"])
TmpFile = namedtuple("TmpFile", ["filename"])
OutFile = namedtuple("OutFile", ["filename"])
diff --git a/icu4c/source/data/buildtool/__main__.py b/icu4c/source/data/buildtool/__main__.py
index 40c33a8..6bc2ef9a 100644
--- a/icu4c/source/data/buildtool/__main__.py
+++ b/icu4c/source/data/buildtool/__main__.py
@@ -13,6 +13,7 @@
from . import *
from .comment_stripper import CommentStripper
+from .request_types import CopyRequest
from .renderers import makefile, unix_exec, windows_exec
from . import filtration, utils
import BUILDRULES
@@ -67,8 +68,8 @@
)
flag_parser.add_argument(
- "--in_dir",
- help = "Path to data input folder (icu4c/source/data).",
+ "--src_dir",
+ help = "Path to data source folder (icu4c/source/data).",
default = "."
)
flag_parser.add_argument(
@@ -163,12 +164,52 @@
pass
+def add_copy_input_requests(requests, config, common_vars):
+ files_to_copy = set()
+ for request in requests:
+ for f in request.all_input_files():
+ if isinstance(f, InFile):
+ files_to_copy.add(f)
+
+ result = []
+ id = 0
+
+ json_data = config.filters_json_data["fileReplacements"]
+ dirname = json_data["directory"]
+ for directive in json_data["replacements"]:
+ input_file = LocalFile(dirname, directive["src"])
+ output_file = InFile(directive["dest"])
+ result += [
+ CopyRequest(
+ name = "input_copy_%d" % id,
+ input_file = input_file,
+ output_file = output_file
+ )
+ ]
+ files_to_copy.remove(output_file)
+ id += 1
+
+ for f in files_to_copy:
+ result += [
+ CopyRequest(
+ name = "input_copy_%d" % id,
+ input_file = SrcFile(f.filename),
+ output_file = f
+ )
+ ]
+ id += 1
+
+ result += requests
+ return result
+
+
def main():
args = flag_parser.parse_args()
config = Config(args)
if args.mode == "gnumake":
makefile_vars = {
+ "SRC_DIR": "$(srcdir)",
"IN_DIR": "$(srcdir)",
"INDEX_NAME": "res_index"
}
@@ -177,13 +218,14 @@
key: "$(%s)" % key
for key in list(makefile_vars.keys()) + makefile_env
}
- common["GLOB_DIR"] = args.in_dir
+ common["GLOB_DIR"] = args.src_dir
else:
common = {
# GLOB_DIR is used now, whereas IN_DIR is used during execution phase.
# There is no useful distinction in unix-exec or windows-exec mode.
- "GLOB_DIR": args.in_dir,
- "IN_DIR": args.in_dir,
+ "GLOB_DIR": args.src_dir,
+ "SRC_DIR": args.src_dir,
+ "IN_DIR": args.src_dir,
"OUT_DIR": args.out_dir,
"TMP_DIR": args.tmp_dir,
"INDEX_NAME": "res_index",
@@ -192,17 +234,25 @@
}
def glob(pattern):
- result_paths = pyglob.glob("{IN_DIR}/{PATTERN}".format(
- IN_DIR = args.in_dir,
+ result_paths = pyglob.glob("{GLOB_DIR}/{PATTERN}".format(
+ GLOB_DIR = args.src_dir,
PATTERN = pattern
))
# For the purposes of buildtool, force Unix-style directory separators.
- return [v.replace("\\", "/")[len(args.in_dir)+1:] for v in sorted(result_paths)]
+ return [v.replace("\\", "/")[len(args.src_dir)+1:] for v in sorted(result_paths)]
requests = BUILDRULES.generate(config, glob, common)
requests = filtration.apply_filters(requests, config)
requests = utils.flatten_requests(requests, config, common)
+ if "fileReplacements" in config.filters_json_data:
+ tmp_in_dir = "{TMP_DIR}/in".format(**common)
+ if makefile_vars:
+ makefile_vars["IN_DIR"] = tmp_in_dir
+ else:
+ common["IN_DIR"] = tmp_in_dir
+ requests = add_copy_input_requests(requests, config, common)
+
build_dirs = utils.compute_directories(requests)
if args.mode == "gnumake":
diff --git a/icu4c/source/data/buildtool/filtration_schema.json b/icu4c/source/data/buildtool/filtration_schema.json
index 48e674e..f305fdc 100644
--- a/icu4c/source/data/buildtool/filtration_schema.json
+++ b/icu4c/source/data/buildtool/filtration_schema.json
@@ -33,6 +33,26 @@
"required": ["categories", "rules"],
"additionalProperties": false
}
+ },
+ "fileReplacements": {
+ "type": "object",
+ "properties": {
+ "directory": { "type": "string" },
+ "replacements": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "src": { "type": "string" },
+ "dest": { "type": "string" }
+ },
+ "additionalProperties": false,
+ "required": ["src", "dest"]
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": ["directory", "replacements"]
}
},
"additionalProperties": false,
diff --git a/icu4c/source/data/buildtool/request_types.py b/icu4c/source/data/buildtool/request_types.py
index 1890dd3..d5230c4 100644
--- a/icu4c/source/data/buildtool/request_types.py
+++ b/icu4c/source/data/buildtool/request_types.py
@@ -104,6 +104,9 @@
if not self.dep_targets:
return
for dep_target in self.dep_targets:
+ if isinstance(dep_target, InFile):
+ self.dep_files.append(dep_target)
+ continue
for request in all_requests:
if request.name == dep_target.name:
self.dep_files += request.all_output_files()
diff --git a/icu4c/source/data/buildtool/utils.py b/icu4c/source/data/buildtool/utils.py
index d072428..7fef140 100644
--- a/icu4c/source/data/buildtool/utils.py
+++ b/icu4c/source/data/buildtool/utils.py
@@ -11,6 +11,10 @@
def dir_for(file):
+ if isinstance(file, LocalFile):
+ return file.dirname
+ if isinstance(file, SrcFile):
+ return "{SRC_DIR}"
if isinstance(file, InFile):
return "{IN_DIR}"
if isinstance(file, TmpFile):
diff --git a/icu4c/source/data/makedata.mak b/icu4c/source/data/makedata.mak
index 06fcc72..c78ac6b 100644
--- a/icu4c/source/data/makedata.mak
+++ b/icu4c/source/data/makedata.mak
@@ -235,7 +235,7 @@
@cd "$(ICUSRCDATA)"
py -3 -m buildtool \
--mode windows-exec \
- --in_dir "$(ICUSRCDATA)" \
+ --src_dir "$(ICUSRCDATA)" \
--tool_dir "$(ICUTOOLS)" \
--tool_cfg "$(CFG)" \
--out_dir "$(ICUBLD_PKG)" \
diff --git a/icu4c/source/test/testdata/testdata.mak b/icu4c/source/test/testdata/testdata.mak
index 9d799f0..a74ba08 100644
--- a/icu4c/source/test/testdata/testdata.mak
+++ b/icu4c/source/test/testdata/testdata.mak
@@ -39,7 +39,7 @@
--mode windows-exec \
--tool_dir "$(ICUTOOLS)" \
--tool_cfg "$(CFG)" \
- --in_dir "$(TESTDATA)" \
+ --src_dir "$(TESTDATA)" \
--tmp_dir "$(TESTDATATMP)" \
--out_dir "$(TESTDATABLD)"
"$(ICUPBIN)\pkgdata" -f -v -m common -c -p"$(TESTPKG)" -d "$(TESTDATAOUT)" -T "$(TESTDATABLD)" -s "$(TESTDATABLD)" $(TESTDATATMP)\testdata.lst
