build: i18n: Autodownload ICU, Intl small-icu by default!

* make `--with-intl=small-icu` the default
 * Download, verify (md5), unpack ICU's zip if not there
 * update docs

This is to support joyent#7676, Intl by default.

There's a "list" of URLs being used, but right now only the
first is picked up. The logic works something like this:

* if there is no directory `deps/icu`,
 * if no zip file (currently `icu4c-54_1-src.zip`),
  * download zip file (icu-project.org -> sf.net)
 * verify the MD5 sum of the zipfile
  * if bad, print error and exit
 * unpack the zipfile into `deps/icu`
* if `deps/icu` now exists, use it, else fail with help text

Also:
* refactor some code into tools/configure.d/nodedownload.py
* add `intl-none` option for `vcbuild.bat`

Author: srl295

.gitignore

@@ -46,6 +46,8 @@ ipch/
 email.md
 deps/v8-*
 deps/icu
+deps/icu*.zip
+deps/icu*.tgz
 ./node_modules
 .svn/
 

README.md

@@ -83,18 +83,41 @@ make doc
 man doc/node.1
 ```
 
-### To build `Intl` (ECMA-402) support:
+### `Intl` (ECMA-402) support:
 
-*Note:* more docs, including how to reduce disk footprint, are on
+The `small-icu` mode is enabled by default. It will build
+with English-only data. You can add full data at runtime.
+
+*Note:* more docs are on
 [the wiki](https://github.com/joyent/node/wiki/Intl).
 
+#### Build with full ICU support (all locales supported by ICU):
+
+Unix/Macintosh:
+
+```sh
+./configure --with-intl=full-icu
+```
+
+Windows:
+
+```sh
+vcbuild full-icu
+```
+
+#### Build with no Intl support `:-(`
+
+```sh
+./configure --with-intl=none
+```
+
 #### Use existing installed ICU (Unix/Macintosh only):
 
 ```sh
 pkg-config --modversion icu-i18n && ./configure --with-intl=system-icu
 ```
 
-#### Build ICU from source:
+#### Build with a specific ICU:
 
 First: Unpack latest ICU
   [icu4c-**##.#**-src.tgz](http://icu-project.org/download) (or `.zip`)

configure

@@ -6,13 +6,20 @@ import re
 import shlex
 import subprocess
 import sys
+import urllib
+import zipfile
+import hashlib
 
 CC = os.environ.get('CC', 'cc')
 
 root_dir = os.path.dirname(__file__)
 sys.path.insert(0, os.path.join(root_dir, 'tools', 'gyp', 'pylib'))
 from gyp.common import GetFlavor
 
+# imports in tools/configure.d
+sys.path.insert(0, os.path.join(root_dir, 'tools', 'configure.d'))
+import nodedownload
+
 # parse our options
 parser = optparse.OptionParser()
 
@@ -244,7 +251,7 @@ parser.add_option('--with-icu-path',
 parser.add_option('--with-intl',
     action='store',
     dest='with_intl',
-    help='Intl mode: none, full-icu, small-icu (default is none)')
+    help='Intl mode: none, full-icu, small-icu (default is small-icu)')
 
 parser.add_option('--with-perfctr',
     action='store_true',
@@ -712,6 +719,58 @@ def glob_to_var(dir_base, dir_sub):
   return list
 
 def configure_intl(o):
+  icus = [
+    {
+      'url': 'http://download.icu-project.org/files/icu4c/54.1/icu4c-54_1-src.zip',
+      # from https://ssl.icu-project.org/files/icu4c/54.1/icu4c-src-54_1.md5:
+      'md5': '6b89d60e2f0e140898ae4d7f72323bca',
+    },
+  ]
+  class ConfigOpener(urllib.FancyURLopener):
+    # append to existing version (UA)
+    version = '%s (node.js/configure)' % urllib.URLopener.version
+  def icu_download(path):
+    # download ICU, if needed
+    def fmtMb(amt):
+      return "{:.1f}".format(amt / 1024000.)
+    def reporthook(count, size, total):
+      sys.stdout.write(' ICU: %c %sMB total, %sMB downloaded   \r' %
+                       (nodedownload.spin(count),
+                        nodedownload.formatSize(total),
+                        nodedownload.formatSize(count*size)))
+    for icu in icus:
+      url = icu['url']
+      md5 = icu['md5']
+      local = url.split('/')[-1]
+      targetfile = os.path.join(root_dir, 'deps', local)
+      if not os.path.isfile(targetfile):
+        try:
+          sys.stdout.write(' <%s>\nConnecting...\r' % url)
+          sys.stdout.flush()
+          msg = urllib.urlretrieve(url, targetfile, reporthook=reporthook)
+          print ''  # clear the line
+        except:
+          print ' ** Error occurred while downloading\n <%s>' % url
+          raise
+      else:
+        print ' Re-using existing %s' % targetfile
+      if os.path.isfile(targetfile):
+        digest = hashlib.md5()
+        count = 0
+        sys.stdout.write(' Checking file integrity with MD5:\r')
+        with open(targetfile, 'rb') as f:
+          chunk = f.read(1024)
+          while chunk !=  "":
+            digest.update(chunk)
+            chunk = f.read(1024)
+        gotmd5 = digest.hexdigest()
+        print ' MD5:      %s  %s' % (gotmd5, targetfile)
+        if (md5 == gotmd5):
+          return targetfile
+        else:
+          print ' Expected: %s      *MISMATCH*' % md5
+          print '\n ** Corrupted ZIP? Delete %s to retry download.\n' % targetfile
+    return None
   icu_config = {
     'variables': {}
   }
@@ -739,6 +798,8 @@ def configure_intl(o):
     o['variables']['icu_gyp_path'] = options.with_icu_path
     return
   # --with-intl=<with_intl>
+  if with_intl is None:
+    with_intl = 'small-icu'  # The default mode
   if with_intl == 'none' or with_intl is None:
     o['variables']['v8_enable_i18n_support'] = 0
     return  # no Intl
@@ -773,16 +834,28 @@ def configure_intl(o):
   byteorder = sys.byteorder
   o['variables']['icu_gyp_path'] = 'tools/icu/icu-generic.gyp'
   # ICU source dir relative to root
-  icu_full_path = os.path.join(root_dir, 'deps/icu')
+  icu_parent_path = os.path.join(root_dir, 'deps')
+  icu_full_path = os.path.join(icu_parent_path, 'icu')
   o['variables']['icu_path'] = icu_full_path
   if not os.path.isdir(icu_full_path):
-    print 'Error: ICU path is not a directory: %s' % (icu_full_path)
+    print '* ECMA-402 (Intl) support didn\'t find ICU in %s..' % (icu_full_path)
+    # can we download (or find) a zipfile?
+    localzip = icu_download(icu_full_path)
+    if localzip:
+      with zipfile.ZipFile(localzip, 'r') as icuzip:
+        print ' Extracting ICU source zip: %s' % localzip
+        icuzip.extractall(icu_parent_path)
+  if not os.path.isdir(icu_full_path):
+    print ' Cannot build Intl without ICU in %s.' % (icu_full_path)
+    print ' (Fix, or disable with "--with-intl=none" )'
     sys.exit(1)
+  else:
+    print '* Using ICU in %s' % (icu_full_path)
   # Now, what version of ICU is it? We just need the "major", such as 54.
   # uvernum.h contains it as a #define.
   uvernum_h = os.path.join(icu_full_path, 'source/common/unicode/uvernum.h')
   if not os.path.isfile(uvernum_h):
-    print 'Error: could not load %s - is ICU installed?' % uvernum_h
+    print ' Error: could not load %s - is ICU installed?' % uvernum_h
     sys.exit(1)
   icu_ver_major = None
   matchVerExp = r'^\s*#define\s+U_ICU_VERSION_SHORT\s+"([^"]*)".*'
@@ -792,7 +865,7 @@ def configure_intl(o):
     if m:
       icu_ver_major = m.group(1)
   if not icu_ver_major:
-    print 'Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h
+    print ' Could not read U_ICU_VERSION_SHORT version from %s' % uvernum_h
     sys.exit(1)
   icu_endianness = sys.byteorder[0];  # TODO(srl295): EBCDIC should be 'e'
   o['variables']['icu_ver_major'] = icu_ver_major
@@ -819,8 +892,8 @@ def configure_intl(o):
   # this is the icudt*.dat file which node will be using (platform endianness)
   o['variables']['icu_data_file'] = icu_data_file
   if not os.path.isfile(icu_data_path):
-    print 'Error: ICU prebuilt data file %s does not exist.' % icu_data_path
-    print 'See the README.md.'
+    print ' Error: ICU prebuilt data file %s does not exist.' % icu_data_path
+    print ' See the README.md.'
     # .. and we're not about to build it from .gyp!
     sys.exit(1)
   # map from variable name to subdirs

tools/configure.d/nodedownload.py

@@ -0,0 +1,11 @@
+# Moved some utilities here from ../../configure
+
+def formatSize(amt):
+    """Format a size as a string"""
+    return "{:.1f}".format(amt / 1024000.)
+
+def spin(c):
+    """print out a spinner based on 'c'"""
+#    spin = "\\|/-"
+    spin = ".:|'"
+    return (spin[c % len(spin)])

vcbuild.bat

@@ -65,6 +65,7 @@ if /i "%1"=="upload"        set upload=1&goto arg-ok
 if /i "%1"=="jslint"        set jslint=1&goto arg-ok
 if /i "%1"=="small-icu"     set i18n_arg=%1&goto arg-ok
 if /i "%1"=="full-icu"      set i18n_arg=%1&goto arg-ok
+if /i "%1"=="intl-none"     set i18n_arg=%1&goto arg-ok
 
 echo Warning: ignoring invalid command line option `%1`.
 
@@ -85,6 +86,7 @@ if defined noperfctr set noperfctr_arg=--without-perfctr& set noperfctr_msi_arg=
 
 if "%i18n_arg%"=="full-icu" set i18n_arg=--with-intl=full-icu
 if "%i18n_arg%"=="small-icu" set i18n_arg=--with-intl=small-icu
+if "%i18n_arg%"=="intl-none" set i18n_arg=--with-intl=none
 
 :project-gen
 @rem Skip project generation if requested.
@@ -232,7 +234,7 @@ python tools/closure_linter/closure_linter/gjslint.py --unix_mode --strict --noj
 goto exit
 
 :help
-echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [nobuild] [nosign] [x86/x64]
+echo vcbuild.bat [debug/release] [msi] [test-all/test-uv/test-internet/test-pummel/test-simple/test-message] [clean] [noprojgen] [small-icu/full-icu/intl-none] [nobuild] [nosign] [x86/x64]
 echo Examples:
 echo   vcbuild.bat                : builds release build
 echo   vcbuild.bat debug          : builds debug build